From 5176bb3c0398488d0c9669fa049e85b33fbacd54 Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Thu, 15 Feb 2024 14:12:04 -0500
Subject: [PATCH 001/157] `IncompleteNetworkInterface`: allow user-specified
 explicit `slot` (#5065)

This makes a several minor changes to plumb slots through:

* `IncompleteNetworkInterface` now stores an optional slot, just like it
stores an optional IP/MAC address
* In `network_interfaces::InsertQuery`, if the incoming slot is set, we
use it directly instead of running the `NextItem`-based subquery
* Adds a partial index to ensure uniqueness of a slot within a single
`parent_id` (I believe this is correct, but would love confirmation from
someone more familiar!)
* `IncompleteNetworkInterface::new_service()` now takes a _non-optional_
IP, MAC address, and slot. This matches how it was called in all
non-test code.
* Tweaked the Nexus internal API used for RSS handoff to include the
slot in the description of NICs.

This is a partial fix for #5056, and should produce correct behavior on
new systems that run through RSS, even without a fix for #5055 (because
we bypass `NextItem` altogether with this change). In particular, I
think this should unblock testing of #5045 on madrid / testbeds. It does
not address the already-recorded-NICs-with-incorrect-slots on systems
like dogfood; I'll take care of that in a subsequent PR.
---
 nexus/db-model/src/network_interface.rs       |  55 +++--
 nexus/db-model/src/schema.rs                  |   2 +-
 nexus/db-queries/src/db/datastore/rack.rs     |  19 +-
 .../src/db/queries/network_interface.rs       | 201 +++++++++++++++---
 nexus/src/app/mod.rs                          |   2 +-
 nexus/test-utils/src/lib.rs                   |   2 +
 nexus/types/src/internal_api/params.rs        |   1 +
 openapi/nexus-internal.json                   |   8 +-
 schema/crdb/34.0.0/up.sql                     |   6 +
 schema/crdb/dbinit.sql                        |  13 +-
 sled-agent/src/params.rs                      |   3 +
 sled-agent/src/sim/server.rs                  |   2 +
 12 files changed, 266 insertions(+), 48 deletions(-)
 create mode 100644 schema/crdb/34.0.0/up.sql

diff --git a/nexus/db-model/src/network_interface.rs b/nexus/db-model/src/network_interface.rs
index 3d3fabbe66..01317fc160 100644
--- a/nexus/db-model/src/network_interface.rs
+++ b/nexus/db-model/src/network_interface.rs
@@ -17,6 +17,13 @@ use nexus_types::identity::Resource;
 use omicron_common::api::external;
 use uuid::Uuid;
 
+/// The max number of interfaces that may be associated with a resource,
+/// e.g., instance or service.
+///
+/// RFD 135 caps instances at 8 interfaces and we use the same limit for
+/// all types of interfaces for simplicity.
+pub const MAX_NICS_PER_INSTANCE: usize = 8;
+
 impl_enum_type! {
     #[derive(SqlType, QueryId, Debug, Clone, Copy)]
     #[diesel(postgres_type(name = "network_interface_kind", schema = "public"))]
@@ -210,9 +217,11 @@ pub struct IncompleteNetworkInterface {
     pub subnet: VpcSubnet,
     pub ip: Option<std::net::IpAddr>,
     pub mac: Option<external::MacAddr>,
+    pub slot: Option<u8>,
 }
 
 impl IncompleteNetworkInterface {
+    #[allow(clippy::too_many_arguments)]
     fn new(
         interface_id: Uuid,
         kind: NetworkInterfaceKind,
@@ -221,24 +230,36 @@ impl IncompleteNetworkInterface {
         identity: external::IdentityMetadataCreateParams,
         ip: Option<std::net::IpAddr>,
         mac: Option<external::MacAddr>,
+        slot: Option<u8>,
     ) -> Result<Self, external::Error> {
         if let Some(ip) = ip {
             subnet.check_requestable_addr(ip)?;
         };
-        match (mac, kind) {
-            (Some(mac), NetworkInterfaceKind::Instance) if !mac.is_guest() => {
-                return Err(external::Error::invalid_request(&format!(
-                    "invalid MAC address {} for guest NIC",
-                    mac
-                )));
+        if let Some(mac) = mac {
+            match kind {
+                NetworkInterfaceKind::Instance => {
+                    if !mac.is_guest() {
+                        return Err(external::Error::invalid_request(format!(
+                            "invalid MAC address {mac} for guest NIC",
+                        )));
+                    }
+                }
+                NetworkInterfaceKind::Service => {
+                    if !mac.is_system() {
+                        return Err(external::Error::invalid_request(format!(
+                            "invalid MAC address {mac} for service NIC",
+                        )));
+                    }
+                }
             }
-            (Some(mac), NetworkInterfaceKind::Service) if !mac.is_system() => {
-                return Err(external::Error::invalid_request(&format!(
-                    "invalid MAC address {} for service NIC",
-                    mac
+        }
+        if let Some(slot) = slot {
+            if usize::from(slot) >= MAX_NICS_PER_INSTANCE {
+                return Err(external::Error::invalid_request(format!(
+                    "invalid slot {slot} for NIC (max slot = {})",
+                    MAX_NICS_PER_INSTANCE - 1,
                 )));
             }
-            _ => {}
         }
         let identity = NetworkInterfaceIdentity::new(interface_id, identity);
         Ok(IncompleteNetworkInterface {
@@ -248,6 +269,7 @@ impl IncompleteNetworkInterface {
             subnet,
             ip,
             mac,
+            slot,
         })
     }
 
@@ -266,6 +288,7 @@ impl IncompleteNetworkInterface {
             identity,
             ip,
             None,
+            None,
         )
     }
 
@@ -274,8 +297,9 @@ impl IncompleteNetworkInterface {
         service_id: Uuid,
         subnet: VpcSubnet,
         identity: external::IdentityMetadataCreateParams,
-        ip: Option<std::net::IpAddr>,
-        mac: Option<external::MacAddr>,
+        ip: std::net::IpAddr,
+        mac: external::MacAddr,
+        slot: u8,
     ) -> Result<Self, external::Error> {
         Self::new(
             interface_id,
@@ -283,8 +307,9 @@ impl IncompleteNetworkInterface {
             service_id,
             subnet,
             identity,
-            ip,
-            mac,
+            Some(ip),
+            Some(mac),
+            Some(slot),
         )
     }
 }
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index 7fc4f9ae45..9ebf7516cf 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(33, 0, 1);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(34, 0, 0);
 
 table! {
     disk (id) {
diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs
index 50bae03c2d..99ed71a073 100644
--- a/nexus/db-queries/src/db/datastore/rack.rs
+++ b/nexus/db-queries/src/db/datastore/rack.rs
@@ -482,8 +482,9 @@ impl DataStore {
                         name: nic.name.clone(),
                         description: format!("{} service vNIC", service.kind),
                     },
-                    Some(nic.ip),
-                    Some(nic.mac),
+                    nic.ip,
+                    nic.mac,
+                    nic.slot,
                 )
                 .map_err(|e| RackInitError::AddingNic(e))?;
                 Some((db_ip, db_nic))
@@ -504,8 +505,9 @@ impl DataStore {
                         name: nic.name.clone(),
                         description: format!("{} service vNIC", service.kind),
                     },
-                    Some(nic.ip),
-                    Some(nic.mac),
+                    nic.ip,
+                    nic.mac,
+                    nic.slot,
                 )
                 .map_err(|e| RackInitError::AddingNic(e))?;
                 Some((db_ip, db_nic))
@@ -1159,6 +1161,7 @@ mod test {
                         name: "external-dns".parse().unwrap(),
                         ip: external_dns_pip.into(),
                         mac: macs.next().unwrap(),
+                        slot: 0,
                     },
                 },
             },
@@ -1178,6 +1181,7 @@ mod test {
                         name: "ntp1".parse().unwrap(),
                         ip: ntp1_pip.into(),
                         mac: macs.next().unwrap(),
+                        slot: 0,
                     },
                 },
             },
@@ -1193,6 +1197,7 @@ mod test {
                         name: "nexus".parse().unwrap(),
                         ip: nexus_pip.into(),
                         mac: macs.next().unwrap(),
+                        slot: 0,
                     },
                 },
             },
@@ -1212,6 +1217,7 @@ mod test {
                         name: "ntp2".parse().unwrap(),
                         ip: ntp2_pip.into(),
                         mac: macs.next().unwrap(),
+                        slot: 0,
                     },
                 },
             },
@@ -1406,6 +1412,7 @@ mod test {
                         name: "nexus1".parse().unwrap(),
                         ip: nexus_pip1.into(),
                         mac: macs.next().unwrap(),
+                        slot: 0,
                     },
                 },
             },
@@ -1421,6 +1428,7 @@ mod test {
                         name: "nexus2".parse().unwrap(),
                         ip: nexus_pip2.into(),
                         mac: macs.next().unwrap(),
+                        slot: 0,
                     },
                 },
             },
@@ -1603,6 +1611,7 @@ mod test {
                     name: "nexus".parse().unwrap(),
                     ip: nexus_pip.into(),
                     mac: macs.next().unwrap(),
+                    slot: 0,
                 },
             },
         }];
@@ -1662,6 +1671,7 @@ mod test {
                         name: "external-dns".parse().unwrap(),
                         ip: external_dns_pip.into(),
                         mac: macs.next().unwrap(),
+                        slot: 0,
                     },
                 },
             },
@@ -1677,6 +1687,7 @@ mod test {
                         name: "nexus".parse().unwrap(),
                         ip: nexus_pip.into(),
                         mac: macs.next().unwrap(),
+                        slot: 0,
                     },
                 },
             },
diff --git a/nexus/db-queries/src/db/queries/network_interface.rs b/nexus/db-queries/src/db/queries/network_interface.rs
index ae3e2c8ead..0643089316 100644
--- a/nexus/db-queries/src/db/queries/network_interface.rs
+++ b/nexus/db-queries/src/db/queries/network_interface.rs
@@ -25,8 +25,8 @@ use diesel::QueryResult;
 use diesel::RunQueryDsl;
 use ipnetwork::IpNetwork;
 use ipnetwork::Ipv4Network;
-use nexus_db_model::NetworkInterfaceKind;
-use nexus_db_model::NetworkInterfaceKindEnum;
+use nexus_db_model::{NetworkInterfaceKind, MAX_NICS_PER_INSTANCE};
+use nexus_db_model::{NetworkInterfaceKindEnum, SqlU8};
 use omicron_common::api::external;
 use omicron_common::api::external::MacAddr;
 use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
@@ -34,13 +34,6 @@ use once_cell::sync::Lazy;
 use std::net::IpAddr;
 use uuid::Uuid;
 
-/// The max number of interfaces that may be associated with a resource,
-/// e.g., instance or service.
-///
-/// RFD 135 caps instances at 8 interfaces and we use the same limit for
-/// all types of interfaces for simplicity.
-pub(crate) const MAX_NICS: usize = 8;
-
 // These are sentinel values and other constants used to verify the state of the
 // system when operating on network interfaces
 
@@ -111,6 +104,8 @@ pub enum InsertError {
     IpAddressNotAvailable(std::net::IpAddr),
     /// An explicity-requested MAC address is already in use
     MacAddressNotAvailable(MacAddr),
+    /// An explicity-requested interface slot is already in use
+    SlotNotAvailable(u8),
     /// There are no slots available for a new interface
     NoSlotsAvailable,
     /// There are no MAC addresses available
@@ -176,21 +171,26 @@ impl InsertError {
                 ))
             }
             InsertError::IpAddressNotAvailable(ip) => {
-                external::Error::invalid_request(&format!(
+                external::Error::invalid_request(format!(
                     "The IP address '{}' is not available",
                     ip
                 ))
             }
             InsertError::MacAddressNotAvailable(mac) => {
-                external::Error::invalid_request(&format!(
+                external::Error::invalid_request(format!(
                     "The MAC address '{}' is not available",
                     mac
                 ))
             }
+            InsertError::SlotNotAvailable(slot) => {
+                external::Error::invalid_request(format!(
+                    "The interface slot '{slot}' is not available",
+                ))
+            }
             InsertError::NoSlotsAvailable => {
-                external::Error::invalid_request(&format!(
+                external::Error::invalid_request(format!(
                     "May not attach more than {} network interfaces",
-                    MAX_NICS
+                    MAX_NICS_PER_INSTANCE
                 ))
             }
             InsertError::NoMacAddrressesAvailable => {
@@ -260,6 +260,12 @@ fn decode_database_error(
     const MAC_NOT_AVAILABLE_CONSTRAINT: &str =
         "network_interface_vpc_id_mac_key";
 
+    // The name of the index whose uniqueness is violated if we try to assign a
+    // slot to an interface that is already allocated to another interface in
+    // the same instance or service.
+    const SLOT_NOT_AVAILABLE_CONSTRAINT: &str =
+        "network_interface_parent_id_slot_key";
+
     // The name of the index whose uniqueness is violated if we try to assign a
     // name to an interface that is already used for another interface on the
     // same resource.
@@ -386,6 +392,12 @@ fn decode_database_error(
                 let mac = interface.mac.unwrap_or_else(|| MacAddr::from_i64(0));
                 InsertError::MacAddressNotAvailable(mac)
             }
+            // Constraint violated if a user-requested slot has
+            // already been assigned within the same instance or service.
+            Some(constraint) if constraint == SLOT_NOT_AVAILABLE_CONSTRAINT => {
+                let slot = interface.slot.unwrap_or(0);
+                InsertError::SlotNotAvailable(slot)
+            }
             // Constraint violated if the user-requested name is already
             // assigned to an interface on this resource
             Some(constraint) if constraint == NAME_CONFLICT_CONSTRAINT => {
@@ -565,7 +577,7 @@ impl NextNicSlot {
     pub fn new(parent_id: Uuid) -> Self {
         let generator = DefaultShiftGenerator {
             base: 0,
-            max_shift: i64::try_from(MAX_NICS)
+            max_shift: i64::try_from(MAX_NICS_PER_INSTANCE)
                 .expect("Too many network interfaces"),
             min_shift: 0,
         };
@@ -984,6 +996,7 @@ pub struct InsertQuery {
     parent_id_str: String,
     ip_sql: Option<IpNetwork>,
     mac_sql: Option<db::model::MacAddr>,
+    slot_sql: Option<SqlU8>,
     next_mac_subquery: NextMacAddress,
     next_ipv4_address_subquery: NextIpv4Address,
     next_slot_subquery: NextNicSlot,
@@ -998,6 +1011,7 @@ impl InsertQuery {
         let parent_id_str = interface.parent_id.to_string();
         let ip_sql = interface.ip.map(|ip| ip.into());
         let mac_sql = interface.mac.map(|mac| mac.into());
+        let slot_sql = interface.slot.map(|slot| slot.into());
         let next_mac_subquery =
             NextMacAddress::new(interface.subnet.vpc_id, interface.kind);
         let next_ipv4_address_subquery = NextIpv4Address::new(
@@ -1015,6 +1029,7 @@ impl InsertQuery {
             parent_id_str,
             ip_sql,
             mac_sql,
+            slot_sql,
             next_mac_subquery,
             next_ipv4_address_subquery,
             next_slot_subquery,
@@ -1169,7 +1184,11 @@ impl QueryFragment<Pg> for InsertQuery {
         out.push_identifier(dsl::ip::NAME)?;
         out.push_sql(", ");
 
-        select_from_cte(out.reborrow(), dsl::slot::NAME)?;
+        if let Some(slot) = &self.slot_sql {
+            out.push_bind_param::<sql_types::Int2, SqlU8>(slot)?;
+        } else {
+            select_from_cte(out.reborrow(), dsl::slot::NAME)?;
+        }
         out.push_sql(", ");
         select_from_cte(out.reborrow(), dsl::is_primary::NAME)?;
 
@@ -1681,7 +1700,7 @@ mod tests {
     use super::first_available_address;
     use super::last_address_offset;
     use super::InsertError;
-    use super::MAX_NICS;
+    use super::MAX_NICS_PER_INSTANCE;
     use super::NUM_INITIAL_RESERVED_IP_ADDRESSES;
     use crate::authz;
     use crate::context::OpContext;
@@ -1713,6 +1732,7 @@ mod tests {
     use omicron_common::api::external::MacAddr;
     use omicron_test_utils::dev;
     use omicron_test_utils::dev::db::CockroachInstance;
+    use std::collections::HashSet;
     use std::convert::TryInto;
     use std::net::IpAddr;
     use std::net::Ipv4Addr;
@@ -2167,8 +2187,14 @@ mod tests {
     async fn test_insert_request_mac() {
         let context = TestContext::new("test_insert_request_mac", 1).await;
 
-        // Insert a service NIC with an explicit MAC address
+        // Ensure service NICs are recorded with the explicit requested MAC
+        // address
         let service_id = Uuid::new_v4();
+        let ip = context.net1.subnets[0]
+            .ipv4_block
+            .iter()
+            .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES)
+            .unwrap();
         let mac = MacAddr::random_system();
         let interface = IncompleteNetworkInterface::new_service(
             Uuid::new_v4(),
@@ -2178,8 +2204,9 @@ mod tests {
                 name: "service-nic".parse().unwrap(),
                 description: String::from("service nic"),
             },
-            None,
-            Some(mac),
+            ip.into(),
+            mac,
+            0,
         )
         .unwrap();
         let inserted_interface = context
@@ -2192,13 +2219,60 @@ mod tests {
         context.success().await;
     }
 
+    #[tokio::test]
+    async fn test_insert_request_slot() {
+        let context = TestContext::new("test_insert_request_slot", 1).await;
+
+        // Ensure service NICs are recorded with the explicit requested slot
+        let mut used_macs = HashSet::new();
+        let mut ips = context.net1.subnets[0]
+            .ipv4_block
+            .iter()
+            .skip(NUM_INITIAL_RESERVED_IP_ADDRESSES);
+        for slot in 0..u8::try_from(MAX_NICS_PER_INSTANCE).unwrap() {
+            let service_id = Uuid::new_v4();
+            let ip = ips.next().expect("exhausted test subnet");
+            let mut mac = MacAddr::random_system();
+            while !used_macs.insert(mac) {
+                mac = MacAddr::random_system();
+            }
+            let interface = IncompleteNetworkInterface::new_service(
+                Uuid::new_v4(),
+                service_id,
+                context.net1.subnets[0].clone(),
+                IdentityMetadataCreateParams {
+                    name: "service-nic".parse().unwrap(),
+                    description: String::from("service nic"),
+                },
+                ip.into(),
+                mac,
+                slot,
+            )
+            .unwrap();
+            let inserted_interface = context
+                .db_datastore
+                .service_create_network_interface_raw(&context.opctx, interface)
+                .await
+                .expect("Failed to insert interface");
+            assert_eq!(inserted_interface.slot, i16::from(slot));
+        }
+
+        context.success().await;
+    }
+
     #[tokio::test]
     async fn test_insert_request_same_mac_fails() {
         let context =
             TestContext::new("test_insert_request_same_mac_fails", 2).await;
 
+        let mut ips = context.net1.subnets[0]
+            .ipv4_block
+            .iter()
+            .skip(NUM_INITIAL_RESERVED_IP_ADDRESSES);
+
         // Insert a service NIC
         let service_id = Uuid::new_v4();
+        let mac = MacAddr::random_system();
         let interface = IncompleteNetworkInterface::new_service(
             Uuid::new_v4(),
             service_id,
@@ -2207,8 +2281,9 @@ mod tests {
                 name: "service-nic".parse().unwrap(),
                 description: String::from("service nic"),
             },
-            None,
-            None,
+            ips.next().expect("exhausted test subnet").into(),
+            mac,
+            0,
         )
         .unwrap();
         let inserted_interface = context
@@ -2216,6 +2291,7 @@ mod tests {
             .service_create_network_interface_raw(&context.opctx, interface)
             .await
             .expect("Failed to insert interface");
+        assert_eq!(inserted_interface.mac.0, mac);
 
         // Inserting an interface with the same MAC should fail, even if all
         // other parameters are valid.
@@ -2228,8 +2304,9 @@ mod tests {
                 name: "new-service-nic".parse().unwrap(),
                 description: String::from("new-service nic"),
             },
-            None,
-            Some(inserted_interface.mac.0),
+            ips.next().expect("exhausted test subnet").into(),
+            mac,
+            0,
         )
         .unwrap();
         let result = context
@@ -2243,6 +2320,80 @@ mod tests {
         context.success().await;
     }
 
+    #[tokio::test]
+    async fn test_insert_request_same_slot_fails() {
+        let context =
+            TestContext::new("test_insert_request_same_slot_fails", 2).await;
+
+        let ip0 = context.net1.subnets[0]
+            .ipv4_block
+            .iter()
+            .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES)
+            .unwrap();
+        let ip1 = context.net1.subnets[1]
+            .ipv4_block
+            .iter()
+            .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES)
+            .unwrap();
+
+        let mut next_mac = {
+            let mut used_macs = HashSet::new();
+            move || {
+                let mut mac = MacAddr::random_system();
+                while !used_macs.insert(mac) {
+                    mac = MacAddr::random_system();
+                }
+                mac
+            }
+        };
+
+        // Insert a service NIC
+        let service_id = Uuid::new_v4();
+        let interface = IncompleteNetworkInterface::new_service(
+            Uuid::new_v4(),
+            service_id,
+            context.net1.subnets[0].clone(),
+            IdentityMetadataCreateParams {
+                name: "service-nic".parse().unwrap(),
+                description: String::from("service nic"),
+            },
+            ip0.into(),
+            next_mac(),
+            0,
+        )
+        .unwrap();
+        let inserted_interface = context
+            .db_datastore
+            .service_create_network_interface_raw(&context.opctx, interface)
+            .await
+            .expect("Failed to insert interface");
+        assert_eq!(inserted_interface.slot, 0);
+
+        // Inserting an interface with the same slot on the same service should
+        let new_interface = IncompleteNetworkInterface::new_service(
+            Uuid::new_v4(),
+            service_id,
+            context.net1.subnets[1].clone(),
+            IdentityMetadataCreateParams {
+                name: "new-service-nic".parse().unwrap(),
+                description: String::from("new-service nic"),
+            },
+            ip1.into(),
+            next_mac(),
+            0,
+        )
+        .unwrap();
+        let result = context
+            .db_datastore
+            .service_create_network_interface_raw(&context.opctx, new_interface)
+            .await;
+        assert!(
+            matches!(result, Err(InsertError::SlotNotAvailable(0))),
+            "Requesting an interface with an existing slot should fail"
+        );
+        context.success().await;
+    }
+
     #[tokio::test]
     async fn test_insert_with_duplicate_name_fails() {
         let context =
@@ -2553,11 +2704,11 @@ mod tests {
     async fn test_limit_number_of_interfaces_per_instance_query() {
         let context = TestContext::new(
             "test_limit_number_of_interfaces_per_instance_query",
-            MAX_NICS as u8 + 1,
+            MAX_NICS_PER_INSTANCE as u8 + 1,
         )
         .await;
         let instance = context.create_stopped_instance().await;
-        for slot in 0..MAX_NICS {
+        for slot in 0..MAX_NICS_PER_INSTANCE {
             let subnet = &context.net1.subnets[slot];
             let interface = IncompleteNetworkInterface::new_instance(
                 Uuid::new_v4(),
diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs
index 7a9a26b05f..969320617f 100644
--- a/nexus/src/app/mod.rs
+++ b/nexus/src/app/mod.rs
@@ -83,7 +83,7 @@ pub(crate) mod sagas;
 
 pub(crate) use nexus_db_queries::db::queries::disk::MAX_DISKS_PER_INSTANCE;
 
-pub(crate) const MAX_NICS_PER_INSTANCE: usize = 8;
+pub(crate) use nexus_db_model::MAX_NICS_PER_INSTANCE;
 
 // XXX: Might want to recast as max *floating* IPs, we have at most one
 //      ephemeral (so bounded in saga by design).
diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs
index 7baacf97ce..2a37721bb0 100644
--- a/nexus/test-utils/src/lib.rs
+++ b/nexus/test-utils/src/lib.rs
@@ -608,6 +608,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
                         .unwrap()
                         .into(),
                     mac,
+                    slot: 0,
                 },
             },
             internal_dns::ServiceName::Nexus,
@@ -831,6 +832,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
                         .unwrap()
                         .into(),
                     mac,
+                    slot: 0,
                 },
             },
             internal_dns::ServiceName::ExternalDns,
diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs
index ab15ec26b7..987efbf6b7 100644
--- a/nexus/types/src/internal_api/params.rs
+++ b/nexus/types/src/internal_api/params.rs
@@ -163,6 +163,7 @@ pub struct ServiceNic {
     pub name: Name,
     pub ip: IpAddr,
     pub mac: MacAddr,
+    pub slot: u8,
 }
 
 /// Describes the purpose of the service.
diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json
index a1a78deb84..b8b45aa08e 100644
--- a/openapi/nexus-internal.json
+++ b/openapi/nexus-internal.json
@@ -6273,13 +6273,19 @@
           },
           "name": {
             "$ref": "#/components/schemas/Name"
+          },
+          "slot": {
+            "type": "integer",
+            "format": "uint8",
+            "minimum": 0
           }
         },
         "required": [
           "id",
           "ip",
           "mac",
-          "name"
+          "name",
+          "slot"
         ]
       },
       "ServicePutRequest": {
diff --git a/schema/crdb/34.0.0/up.sql b/schema/crdb/34.0.0/up.sql
new file mode 100644
index 0000000000..b8190b2246
--- /dev/null
+++ b/schema/crdb/34.0.0/up.sql
@@ -0,0 +1,6 @@
+CREATE UNIQUE INDEX IF NOT EXISTS network_interface_parent_id_slot_key ON omicron.public.network_interface (
+    parent_id,
+    slot
+)
+WHERE
+    time_deleted IS NULL;
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 18b1b82563..254080e92d 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -1462,6 +1462,17 @@ STORING (vpc_id, subnet_id, is_primary)
 WHERE
     time_deleted IS NULL;
 
+/*
+ * Index used to verify that all interfaces for a resource (e.g. Instance,
+ * Service) have unique slots.
+ */
+CREATE UNIQUE INDEX IF NOT EXISTS network_interface_parent_id_slot_key ON omicron.public.network_interface (
+    parent_id,
+    slot
+)
+WHERE
+    time_deleted IS NULL;
+
 CREATE TYPE IF NOT EXISTS omicron.public.vpc_firewall_rule_status AS ENUM (
     'disabled',
     'enabled'
@@ -3504,7 +3515,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '33.0.1', NULL)
+    ( TRUE, NOW(), NOW(), '34.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;
diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs
index 7ed1264d9c..fda952ca87 100644
--- a/sled-agent/src/params.rs
+++ b/sled-agent/src/params.rs
@@ -377,6 +377,7 @@ impl OmicronZoneConfig {
                         name: nic.name.clone(),
                         ip: nic.ip,
                         mac: nic.mac,
+                        slot: nic.slot,
                     },
                 },
             },
@@ -397,6 +398,7 @@ impl OmicronZoneConfig {
                         name: nic.name.clone(),
                         ip: nic.ip,
                         mac: nic.mac,
+                        slot: nic.slot,
                     },
                 },
             },
@@ -440,6 +442,7 @@ impl OmicronZoneConfig {
                             name: nic.name.clone(),
                             ip: nic.ip,
                             mac: nic.mac,
+                            slot: nic.slot,
                         },
                     },
                 }
diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs
index fd5995b8f1..8854aee05c 100644
--- a/sled-agent/src/sim/server.rs
+++ b/sled-agent/src/sim/server.rs
@@ -363,6 +363,7 @@ pub async fn run_standalone_server(
                         .unwrap()
                         .into(),
                     mac: macs.next().unwrap(),
+                    slot: 0,
                 },
             },
             service_id: Uuid::new_v4(),
@@ -396,6 +397,7 @@ pub async fn run_standalone_server(
                         .unwrap()
                         .into(),
                     mac: macs.next().unwrap(),
+                    slot: 0,
                 },
             },
             service_id: Uuid::new_v4(),

From 4db6c6c4983a68ec8717f4155d30f3489191d453 Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Thu, 15 Feb 2024 15:44:35 -0500
Subject: [PATCH 002/157] [omdb] Add `db network list-vnics` and expand output
 of `db network list-eips` (#5064)

This PR has some omdb commands I wanted during dev/debug of #5045. It
expands the output of `list-vnics` to include the parent's ID
(particularly useful when trying to determine the external IP of a
specific Nexus instance, for example):

```
 IP                PORTS        KIND       STATE     OWNER_KIND  OWNER_ID                              OWNER_DESCRIPTION
 10.1.1.3/32       0/65535      floating   Attached  instance    4e6fb33a-7ba2-4a5e-abc5-dc9b047c01e0  v6/some-vm2
 10.1.1.4/32       0/16383      SNAT       Attached  instance    4e6fb33a-7ba2-4a5e-abc5-dc9b047c01e0  v6/some-vm2
 10.1.1.5/32       0/65535      ephemeral  Attached  instance    4e6fb33a-7ba2-4a5e-abc5-dc9b047c01e0  v6/some-vm2
 172.20.26.1/32    0/65535      floating   Attached  service     edd99650-5df1-4241-815d-253e4ef2399c  ExternalDns
 172.20.26.2/32    0/65535      floating   Attached  service     f500d564-c40a-4eca-ac8a-a26b435f2037  ExternalDns
 172.20.26.3/32    0/65535      floating   Attached  service     65a11c18-7f59-41ac-b9e7-680627f996e7  Nexus
 172.20.26.4/32    0/65535      floating   Attached  service     20b100d0-84c3-4119-aa9b-0c632b0b6a3a  Nexus
 172.20.26.5/32    0/65535      floating   Attached  service     2898657e-4141-4c05-851b-147bffc6bbbd  Nexus
 172.20.26.6/32    0/16383      SNAT       Attached  service     c3ec3d1a-3172-4d36-bfd3-f54a04d5ba55  Ntp
```

and adds a `list-vnics` command to show allocated vnics:

```
 IP                 MAC                SLOT  PRIMARY  KIND      SUBNET           PARENT_ID                             DESCRIPTION
 172.30.0.5/32      A8:40:25:F8:A5:8C  1     true     instance  172.30.0.0/22    2a4afdda-e269-48bc-913f-01ad57c50543  default primary interface for p4
 172.30.0.5/32      A8:40:25:F5:AF:F0  1     true     instance  172.30.0.0/22    be705808-d507-4693-9a97-186c92970e7b  default primary interface for updateinst
 172.30.0.5/32      A8:40:25:F7:3B:00  1     true     instance  172.30.0.0/22    0ab1939f-af6e-4ea2-a155-71f210e937fc  a sample nic
 172.30.1.5/32      A8:40:25:FF:B0:9C  1     true     service   172.30.1.0/24    edd99650-5df1-4241-815d-253e4ef2399c  external_dns service vNIC
 172.30.1.6/32      A8:40:25:FF:D0:B4  1     true     service   172.30.1.0/24    f500d564-c40a-4eca-ac8a-a26b435f2037  external_dns service vNIC
 172.30.2.5/32      A8:40:25:FF:A6:83  1     true     service   172.30.2.0/24    65a11c18-7f59-41ac-b9e7-680627f996e7  nexus service vNIC
 172.30.2.6/32      A8:40:25:FF:B4:C1  1     true     service   172.30.2.0/24    20b100d0-84c3-4119-aa9b-0c632b0b6a3a  nexus service vNIC
 172.30.2.7/32      A8:40:25:FF:C6:59  0     true     service   172.30.2.0/24    2898657e-4141-4c05-851b-147bffc6bbbd  nexus service vNIC
 172.30.3.5/32      A8:40:25:FF:B2:52  1     true     service   172.30.3.0/24    c3ec3d1a-3172-4d36-bfd3-f54a04d5ba55  ntp service vNIC
 172.30.3.6/32      A8:40:25:FF:A0:F9  1     true     service   172.30.3.0/24    6ea2684c-115e-48a6-8453-ab52d1cecd73  ntp service vNIC
```

(This command immediately revealed issues with the slot number recording
on dogfood, which led to opening #5056.)
---
 dev-tools/omdb/src/bin/omdb/db.rs     | 317 ++++++++++++++++++++++----
 dev-tools/omdb/tests/usage_errors.out |   5 +-
 2 files changed, 270 insertions(+), 52 deletions(-)

diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs
index 9c41c25cc0..a51deacaf4 100644
--- a/dev-tools/omdb/src/bin/omdb/db.rs
+++ b/dev-tools/omdb/src/bin/omdb/db.rs
@@ -32,8 +32,10 @@ use diesel::BoolExpressionMethods;
 use diesel::ExpressionMethods;
 use diesel::JoinOnDsl;
 use diesel::NullableExpressionMethods;
+use diesel::OptionalExtension;
 use diesel::TextExpressionMethods;
 use gateway_client::types::SpType;
+use ipnetwork::IpNetwork;
 use nexus_db_model::Dataset;
 use nexus_db_model::Disk;
 use nexus_db_model::DnsGroup;
@@ -45,6 +47,9 @@ use nexus_db_model::HwBaseboardId;
 use nexus_db_model::Instance;
 use nexus_db_model::InvCollection;
 use nexus_db_model::IpAttachState;
+use nexus_db_model::IpKind;
+use nexus_db_model::NetworkInterface;
+use nexus_db_model::NetworkInterfaceKind;
 use nexus_db_model::Project;
 use nexus_db_model::Region;
 use nexus_db_model::RegionSnapshot;
@@ -55,6 +60,7 @@ use nexus_db_model::SwCaboose;
 use nexus_db_model::SwRotPage;
 use nexus_db_model::Vmm;
 use nexus_db_model::Volume;
+use nexus_db_model::VpcSubnet;
 use nexus_db_model::Zpool;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db;
@@ -75,6 +81,7 @@ use nexus_types::inventory::Collection;
 use nexus_types::inventory::RotPageWhich;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::Generation;
+use omicron_common::api::external::MacAddr;
 use omicron_common::postgres_config::PostgresConfigWithUrl;
 use sled_agent_client::types::VolumeConstructionRequest;
 use std::borrow::Cow;
@@ -331,6 +338,8 @@ struct NetworkArgs {
 enum NetworkCommands {
     /// List external IPs
     ListEips,
+    /// List virtual network interfaces
+    ListVnics,
 }
 
 #[derive(Debug, Args)]
@@ -488,6 +497,17 @@ impl DbArgs {
                 cmd_db_eips(&opctx, &datastore, &self.fetch_opts, *verbose)
                     .await
             }
+            DbCommands::Network(NetworkArgs {
+                command: NetworkCommands::ListVnics,
+                verbose,
+            }) => {
+                cmd_db_network_list_vnics(
+                    &datastore,
+                    &self.fetch_opts,
+                    *verbose,
+                )
+                .await
+            }
             DbCommands::Snapshots(SnapshotArgs {
                 command: SnapshotCommands::Info(uuid),
             }) => cmd_db_snapshot_info(&opctx, &datastore, uuid).await,
@@ -582,6 +602,42 @@ fn first_page<'a, T>(limit: NonZeroU32) -> DataPageParams<'a, T> {
     }
 }
 
+/// Helper function to looks up an instance with the given ID.
+async fn lookup_instance(
+    datastore: &DataStore,
+    instance_id: Uuid,
+) -> anyhow::Result<Option<Instance>> {
+    use db::schema::instance::dsl;
+
+    let conn = datastore.pool_connection_for_tests().await?;
+    dsl::instance
+        .filter(dsl::id.eq(instance_id))
+        .limit(1)
+        .select(Instance::as_select())
+        .get_result_async(&*conn)
+        .await
+        .optional()
+        .with_context(|| format!("loading instance {instance_id}"))
+}
+
+/// Helper function to looks up a project with the given ID.
+async fn lookup_project(
+    datastore: &DataStore,
+    project_id: Uuid,
+) -> anyhow::Result<Option<Project>> {
+    use db::schema::project::dsl;
+
+    let conn = datastore.pool_connection_for_tests().await?;
+    dsl::project
+        .filter(dsl::id.eq(project_id))
+        .limit(1)
+        .select(Project::as_select())
+        .get_result_async(&*conn)
+        .await
+        .optional()
+        .with_context(|| format!("loading project {project_id}"))
+}
+
 // Disks
 
 /// Run `omdb db disk list`.
@@ -1743,32 +1799,54 @@ async fn cmd_db_eips(
         }
     }
 
-    #[derive(Tabled)]
     enum Owner {
-        Instance { project: String, name: String },
-        Service { kind: String },
+        Instance { id: Uuid, project: String, name: String },
+        Service { id: Uuid, kind: String },
+        Project { id: Uuid, name: String },
         None,
     }
 
-    impl Display for Owner {
-        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    impl Owner {
+        fn kind(&self) -> &'static str {
+            match self {
+                Owner::Instance { .. } => "instance",
+                Owner::Service { .. } => "service",
+                Owner::Project { .. } => "project",
+                Owner::None => "none",
+            }
+        }
+
+        fn id(&self) -> String {
+            match self {
+                Owner::Instance { id, .. }
+                | Owner::Service { id, .. }
+                | Owner::Project { id, .. } => id.to_string(),
+                Owner::None => "none".to_string(),
+            }
+        }
+
+        fn name(&self) -> String {
             match self {
-                Self::Instance { project, name } => {
-                    write!(f, "Instance {project}/{name}")
+                Self::Instance { project, name, .. } => {
+                    format!("{project}/{name}")
                 }
-                Self::Service { kind } => write!(f, "Service {kind}"),
-                Self::None => write!(f, "None"),
+                Self::Service { kind, .. } => kind.to_string(),
+                Self::Project { name, .. } => name.to_string(),
+                Self::None => "none".to_string(),
             }
         }
     }
 
     #[derive(Tabled)]
+    #[tabled(rename_all = "SCREAMING_SNAKE_CASE")]
     struct IpRow {
         ip: ipnetwork::IpNetwork,
         ports: PortRange,
-        kind: String,
+        kind: IpKind,
         state: IpAttachState,
-        owner: Owner,
+        owner_kind: &'static str,
+        owner_id: String,
+        owner_name: String,
     }
 
     if verbose {
@@ -1798,50 +1876,58 @@ async fn cmd_db_eips(
                         continue;
                     }
                 };
-                Owner::Service { kind: format!("{:?}", service.1.kind) }
+                Owner::Service {
+                    id: owner_id,
+                    kind: format!("{:?}", service.1.kind),
+                }
             } else {
-                use db::schema::instance::dsl as instance_dsl;
-                let instance = match instance_dsl::instance
-                    .filter(instance_dsl::id.eq(owner_id))
-                    .limit(1)
-                    .select(Instance::as_select())
-                    .load_async(&*datastore.pool_connection_for_tests().await?)
-                    .await
-                    .context("loading requested instance")?
-                    .pop()
-                {
-                    Some(instance) => instance,
-                    None => {
-                        eprintln!("instance with id {owner_id} not found");
-                        continue;
-                    }
-                };
-
-                use db::schema::project::dsl as project_dsl;
-                let project = match project_dsl::project
-                    .filter(project_dsl::id.eq(instance.project_id))
-                    .limit(1)
-                    .select(Project::as_select())
-                    .load_async(&*datastore.pool_connection_for_tests().await?)
-                    .await
-                    .context("loading requested project")?
-                    .pop()
-                {
-                    Some(instance) => instance,
-                    None => {
-                        eprintln!(
-                            "project with id {} not found",
-                            instance.project_id
-                        );
-                        continue;
-                    }
-                };
+                let instance =
+                    match lookup_instance(datastore, owner_id).await? {
+                        Some(instance) => instance,
+                        None => {
+                            eprintln!("instance with id {owner_id} not found");
+                            continue;
+                        }
+                    };
+
+                let project =
+                    match lookup_project(datastore, instance.project_id).await?
+                    {
+                        Some(project) => project,
+                        None => {
+                            eprintln!(
+                                "project with id {} not found",
+                                instance.project_id
+                            );
+                            continue;
+                        }
+                    };
 
                 Owner::Instance {
+                    id: owner_id,
                     project: project.name().to_string(),
                     name: instance.name().to_string(),
                 }
             }
+        } else if let Some(project_id) = ip.project_id {
+            use db::schema::project::dsl as project_dsl;
+            let project = match project_dsl::project
+                .filter(project_dsl::id.eq(project_id))
+                .limit(1)
+                .select(Project::as_select())
+                .load_async(&*datastore.pool_connection_for_tests().await?)
+                .await
+                .context("loading requested project")?
+                .pop()
+            {
+                Some(project) => project,
+                None => {
+                    eprintln!("project with id {} not found", project_id);
+                    continue;
+                }
+            };
+
+            Owner::Project { id: project_id, name: project.name().to_string() }
         } else {
             Owner::None
         };
@@ -1853,8 +1939,139 @@ async fn cmd_db_eips(
                 last: ip.last_port.into(),
             },
             state: ip.state,
-            kind: format!("{:?}", ip.kind),
-            owner,
+            kind: ip.kind,
+            owner_kind: owner.kind(),
+            owner_id: owner.id(),
+            owner_name: owner.name(),
+        };
+        rows.push(row);
+    }
+
+    rows.sort_by(|a, b| a.ip.cmp(&b.ip));
+    let table = tabled::Table::new(rows)
+        .with(tabled::settings::Style::empty())
+        .to_string();
+
+    println!("{}", table);
+
+    Ok(())
+}
+
+async fn cmd_db_network_list_vnics(
+    datastore: &DataStore,
+    fetch_opts: &DbFetchOptions,
+    verbose: bool,
+) -> Result<(), anyhow::Error> {
+    #[derive(Tabled)]
+    #[tabled(rename_all = "SCREAMING_SNAKE_CASE")]
+    struct NicRow {
+        ip: IpNetwork,
+        mac: MacAddr,
+        slot: i16,
+        primary: bool,
+        kind: &'static str,
+        subnet: String,
+        parent_id: Uuid,
+        parent_name: String,
+    }
+    use db::schema::network_interface::dsl;
+    let mut query = dsl::network_interface.into_boxed();
+    if !fetch_opts.include_deleted {
+        query = query.filter(dsl::time_deleted.is_null());
+    }
+
+    let nics: Vec<NetworkInterface> = query
+        .select(NetworkInterface::as_select())
+        .limit(i64::from(u32::from(fetch_opts.fetch_limit)))
+        .get_results_async(&*datastore.pool_connection_for_tests().await?)
+        .await?;
+
+    check_limit(&nics, fetch_opts.fetch_limit, || {
+        String::from("listing network interfaces")
+    });
+
+    if verbose {
+        for nic in &nics {
+            if verbose {
+                println!("{nic:#?}");
+            }
+        }
+        return Ok(());
+    }
+
+    let mut rows = Vec::new();
+
+    for nic in &nics {
+        let (kind, parent_name) = match nic.kind {
+            NetworkInterfaceKind::Instance => {
+                match lookup_instance(datastore, nic.parent_id).await? {
+                    Some(instance) => {
+                        match lookup_project(datastore, instance.project_id)
+                            .await?
+                        {
+                            Some(project) => (
+                                "instance",
+                                format!(
+                                    "{}/{}",
+                                    project.name(),
+                                    instance.name()
+                                ),
+                            ),
+                            None => {
+                                eprintln!(
+                                    "project with id {} not found",
+                                    instance.project_id
+                                );
+                                continue;
+                            }
+                        }
+                    }
+                    None => {
+                        ("instance?", "parent instance not found".to_string())
+                    }
+                }
+            }
+            NetworkInterfaceKind::Service => {
+                // We create service NICs named after the service, so we can use
+                // the nic name instead of looking up the service.
+                ("service", nic.name().to_string())
+            }
+        };
+
+        let subnet = {
+            use db::schema::vpc_subnet::dsl;
+            let subnet = match dsl::vpc_subnet
+                .filter(dsl::id.eq(nic.subnet_id))
+                .limit(1)
+                .select(VpcSubnet::as_select())
+                .load_async(&*datastore.pool_connection_for_tests().await?)
+                .await
+                .context("loading requested subnet")?
+                .pop()
+            {
+                Some(subnet) => subnet,
+                None => {
+                    eprintln!("subnet with id {} not found", nic.subnet_id);
+                    continue;
+                }
+            };
+
+            if nic.ip.is_ipv4() {
+                subnet.ipv4_block.to_string()
+            } else {
+                subnet.ipv6_block.to_string()
+            }
+        };
+
+        let row = NicRow {
+            ip: nic.ip,
+            mac: *nic.mac,
+            slot: nic.slot,
+            primary: nic.primary,
+            kind,
+            subnet,
+            parent_id: nic.parent_id,
+            parent_name,
         };
         rows.push(row);
     }
diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out
index bb7da1be57..c10f95e23d 100644
--- a/dev-tools/omdb/tests/usage_errors.out
+++ b/dev-tools/omdb/tests/usage_errors.out
@@ -254,8 +254,9 @@ Print information about the network
 Usage: omdb db network [OPTIONS] <COMMAND>
 
 Commands:
-  list-eips  List external IPs
-  help       Print this message or the help of the given subcommand(s)
+  list-eips   List external IPs
+  list-vnics  List virtual network interfaces
+  help        Print this message or the help of the given subcommand(s)
 
 Options:
       --verbose  Print out raw data structures from the data store

From e4ba67bfece71a417201d844f89e3ff23d5176c8 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Thu, 15 Feb 2024 14:02:32 -0800
Subject: [PATCH 003/157] chore(deps): update rust crate num-integer to 0.1.46
 (#5076)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock                | 5 ++---
 Cargo.toml                | 2 +-
 workspace-hack/Cargo.toml | 4 ++--
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2f13d1e990..63d72bc117 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4699,11 +4699,10 @@ dependencies = [
 
 [[package]]
 name = "num-integer"
-version = "0.1.45"
+version = "0.1.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
 dependencies = [
- "autocfg",
  "num-traits",
 ]
 
diff --git a/Cargo.toml b/Cargo.toml
index 3dae9211e3..e5fa364305 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -264,7 +264,7 @@ nexus-test-interface = { path = "nexus/test-interface" }
 nexus-test-utils-macros = { path = "nexus/test-utils-macros" }
 nexus-test-utils = { path = "nexus/test-utils" }
 nexus-types = { path = "nexus/types" }
-num-integer = "0.1.45"
+num-integer = "0.1.46"
 num = { version = "0.4.1", default-features = false, features = [ "libm" ] }
 omicron-common = { path = "common" }
 omicron-gateway = { path = "gateway" }
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index a52f63fec1..d72b63356b 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -71,7 +71,7 @@ managed = { version = "0.8.0", default-features = false, features = ["alloc", "m
 memchr = { version = "2.6.3" }
 nom = { version = "7.1.3" }
 num-bigint = { version = "0.4.4", features = ["rand"] }
-num-integer = { version = "0.1.45", features = ["i128"] }
+num-integer = { version = "0.1.46", features = ["i128"] }
 num-iter = { version = "0.1.43", default-features = false, features = ["i128"] }
 num-traits = { version = "0.2.16", features = ["i128", "libm"] }
 openapiv3 = { version = "2.0.0", default-features = false, features = ["skip_serializing_defaults"] }
@@ -178,7 +178,7 @@ managed = { version = "0.8.0", default-features = false, features = ["alloc", "m
 memchr = { version = "2.6.3" }
 nom = { version = "7.1.3" }
 num-bigint = { version = "0.4.4", features = ["rand"] }
-num-integer = { version = "0.1.45", features = ["i128"] }
+num-integer = { version = "0.1.46", features = ["i128"] }
 num-iter = { version = "0.1.43", default-features = false, features = ["i128"] }
 num-traits = { version = "0.2.16", features = ["i128", "libm"] }
 openapiv3 = { version = "2.0.0", default-features = false, features = ["skip_serializing_defaults"] }

From eda05500978621d28df6e15cb50549abed9a60ca Mon Sep 17 00:00:00 2001
From: bnaecker <ben@oxide.computer>
Date: Thu, 15 Feb 2024 14:43:47 -0800
Subject: [PATCH 004/157] Update progenitor to bc0bb4b (#5071)

---
 Cargo.lock                            | 61 +++++++++++----------------
 clients/dns-service-client/src/lib.rs |  3 +-
 common/src/api/external/error.rs      |  3 +-
 sled-agent/src/instance.rs            |  3 +-
 workspace-hack/Cargo.toml             |  6 +--
 5 files changed, 33 insertions(+), 43 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 63d72bc117..e82fd6a60f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -655,7 +655,7 @@ dependencies = [
  "omicron-common",
  "omicron-workspace-hack",
  "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
- "regress 0.8.0",
+ "regress",
  "reqwest",
  "schemars",
  "serde",
@@ -1946,7 +1946,7 @@ dependencies = [
  "progenitor-client 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
  "quote",
  "rand 0.8.5",
- "regress 0.8.0",
+ "regress",
  "reqwest",
  "rustfmt-wrapper",
  "schemars",
@@ -3368,7 +3368,7 @@ dependencies = [
  "opte-ioctl",
  "oxide-vpc",
  "oxlog",
- "regress 0.8.0",
+ "regress",
  "schemars",
  "serde",
  "serde_json",
@@ -3504,7 +3504,7 @@ dependencies = [
  "installinator-common",
  "omicron-workspace-hack",
  "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
- "regress 0.8.0",
+ "regress",
  "reqwest",
  "schemars",
  "serde",
@@ -4302,7 +4302,7 @@ dependencies = [
  "omicron-passwords",
  "omicron-workspace-hack",
  "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
- "regress 0.8.0",
+ "regress",
  "reqwest",
  "schemars",
  "serde",
@@ -4871,7 +4871,7 @@ dependencies = [
  "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
  "proptest",
  "rand 0.8.5",
- "regress 0.8.0",
+ "regress",
  "reqwest",
  "schemars",
  "semver 1.0.21",
@@ -5365,7 +5365,6 @@ dependencies = [
  "generic-array",
  "getrandom 0.2.10",
  "group",
- "hashbrown 0.13.2",
  "hashbrown 0.14.3",
  "hex",
  "hmac",
@@ -5647,7 +5646,7 @@ dependencies = [
  "omicron-workspace-hack",
  "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
  "rand 0.8.5",
- "regress 0.8.0",
+ "regress",
  "reqwest",
  "serde",
  "serde_json",
@@ -6500,7 +6499,7 @@ dependencies = [
 [[package]]
 name = "progenitor"
 version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor?branch=main#86b60220b88a2ca3629fb87acf8f83ff35f63aaa"
+source = "git+https://github.com/oxidecomputer/progenitor?branch=main#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
 dependencies = [
  "progenitor-client 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
  "progenitor-impl 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
@@ -6511,7 +6510,7 @@ dependencies = [
 [[package]]
 name = "progenitor"
 version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor#86b60220b88a2ca3629fb87acf8f83ff35f63aaa"
+source = "git+https://github.com/oxidecomputer/progenitor#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
 dependencies = [
  "progenitor-client 0.5.0 (git+https://github.com/oxidecomputer/progenitor)",
  "progenitor-impl 0.5.0 (git+https://github.com/oxidecomputer/progenitor)",
@@ -6522,7 +6521,7 @@ dependencies = [
 [[package]]
 name = "progenitor-client"
 version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor?branch=main#86b60220b88a2ca3629fb87acf8f83ff35f63aaa"
+source = "git+https://github.com/oxidecomputer/progenitor?branch=main#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
 dependencies = [
  "bytes",
  "futures-core",
@@ -6536,7 +6535,7 @@ dependencies = [
 [[package]]
 name = "progenitor-client"
 version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor#86b60220b88a2ca3629fb87acf8f83ff35f63aaa"
+source = "git+https://github.com/oxidecomputer/progenitor#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
 dependencies = [
  "bytes",
  "futures-core",
@@ -6550,7 +6549,7 @@ dependencies = [
 [[package]]
 name = "progenitor-impl"
 version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor?branch=main#86b60220b88a2ca3629fb87acf8f83ff35f63aaa"
+source = "git+https://github.com/oxidecomputer/progenitor?branch=main#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
 dependencies = [
  "getopts",
  "heck 0.4.1",
@@ -6572,7 +6571,7 @@ dependencies = [
 [[package]]
 name = "progenitor-impl"
 version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor#86b60220b88a2ca3629fb87acf8f83ff35f63aaa"
+source = "git+https://github.com/oxidecomputer/progenitor#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
 dependencies = [
  "getopts",
  "heck 0.4.1",
@@ -6594,7 +6593,7 @@ dependencies = [
 [[package]]
 name = "progenitor-macro"
 version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor?branch=main#86b60220b88a2ca3629fb87acf8f83ff35f63aaa"
+source = "git+https://github.com/oxidecomputer/progenitor?branch=main#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
 dependencies = [
  "openapiv3",
  "proc-macro2",
@@ -6611,7 +6610,7 @@ dependencies = [
 [[package]]
 name = "progenitor-macro"
 version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor#86b60220b88a2ca3629fb87acf8f83ff35f63aaa"
+source = "git+https://github.com/oxidecomputer/progenitor#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
 dependencies = [
  "openapiv3",
  "proc-macro2",
@@ -7051,16 +7050,6 @@ version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
 
-[[package]]
-name = "regress"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ed9969cad8051328011596bf549629f1b800cf1731e7964b1eef8dfc480d2c2"
-dependencies = [
- "hashbrown 0.13.2",
- "memchr",
-]
-
 [[package]]
 name = "regress"
 version = "0.8.0"
@@ -8146,7 +8135,7 @@ dependencies = [
  "omicron-common",
  "omicron-workspace-hack",
  "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
- "regress 0.8.0",
+ "regress",
  "reqwest",
  "schemars",
  "serde",
@@ -9009,18 +8998,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.56"
+version = "1.0.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad"
+checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.56"
+version = "1.0.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471"
+checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -9716,7 +9705,7 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
 [[package]]
 name = "typify"
 version = "0.0.15"
-source = "git+https://github.com/oxidecomputer/typify#1f97f167923f001818d461b1286f8a5242abf8b1"
+source = "git+https://github.com/oxidecomputer/typify#ce009d6f83b620cbd0e3acdd9b9ea071018471d8"
 dependencies = [
  "typify-impl",
  "typify-macro",
@@ -9725,13 +9714,13 @@ dependencies = [
 [[package]]
 name = "typify-impl"
 version = "0.0.15"
-source = "git+https://github.com/oxidecomputer/typify#1f97f167923f001818d461b1286f8a5242abf8b1"
+source = "git+https://github.com/oxidecomputer/typify#ce009d6f83b620cbd0e3acdd9b9ea071018471d8"
 dependencies = [
  "heck 0.4.1",
  "log",
  "proc-macro2",
  "quote",
- "regress 0.7.1",
+ "regress",
  "schemars",
  "serde_json",
  "syn 2.0.48",
@@ -9742,7 +9731,7 @@ dependencies = [
 [[package]]
 name = "typify-macro"
 version = "0.0.15"
-source = "git+https://github.com/oxidecomputer/typify#1f97f167923f001818d461b1286f8a5242abf8b1"
+source = "git+https://github.com/oxidecomputer/typify#ce009d6f83b620cbd0e3acdd9b9ea071018471d8"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -10466,7 +10455,7 @@ dependencies = [
  "ipnetwork",
  "omicron-workspace-hack",
  "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
- "regress 0.8.0",
+ "regress",
  "reqwest",
  "schemars",
  "serde",
diff --git a/clients/dns-service-client/src/lib.rs b/clients/dns-service-client/src/lib.rs
index e437f1a7f6..52c2b8bcd2 100644
--- a/clients/dns-service-client/src/lib.rs
+++ b/clients/dns-service-client/src/lib.rs
@@ -32,7 +32,8 @@ pub fn is_retryable(error: &DnsConfigError<crate::types::Error>) -> bool {
         | DnsConfigError::InvalidResponsePayload(_, _)
         | DnsConfigError::UnexpectedResponse(_)
         | DnsConfigError::InvalidUpgrade(_)
-        | DnsConfigError::ResponseBodyError(_) => return false,
+        | DnsConfigError::ResponseBodyError(_)
+        | DnsConfigError::PreHookError(_) => return false,
         DnsConfigError::ErrorResponse(response_value) => response_value,
     };
 
diff --git a/common/src/api/external/error.rs b/common/src/api/external/error.rs
index d2e062f2e1..f7eb257e8f 100644
--- a/common/src/api/external/error.rs
+++ b/common/src/api/external/error.rs
@@ -496,7 +496,8 @@ impl<T: ClientError> From<progenitor::progenitor_client::Error<T>> for Error {
             )
             | progenitor::progenitor_client::Error::UnexpectedResponse(_)
             | progenitor::progenitor_client::Error::InvalidUpgrade(_)
-            | progenitor::progenitor_client::Error::ResponseBodyError(_) => {
+            | progenitor::progenitor_client::Error::ResponseBodyError(_)
+            | progenitor::progenitor_client::Error::PreHookError(_) => {
                 Error::internal_error(&e.to_string())
             }
             // This error represents an expected error from the remote service.
diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs
index 5b9bf54dd9..7a6033b4bb 100644
--- a/sled-agent/src/instance.rs
+++ b/sled-agent/src/instance.rs
@@ -504,7 +504,8 @@ impl InstanceRunner {
                             | nexus_client::Error::InvalidResponsePayload(..)
                             | nexus_client::Error::UnexpectedResponse(_)
                             | nexus_client::Error::InvalidUpgrade(_)
-                            | nexus_client::Error::ResponseBodyError(_) => {
+                            | nexus_client::Error::ResponseBodyError(_)
+                            | nexus_client::Error::PreHookError(_) => {
                                 BackoffError::permanent(Error::Notification(
                                     err,
                                 ))
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index d72b63356b..098be26460 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -54,8 +54,7 @@ gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-
 generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] }
 getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] }
 group = { version = "0.13.0", default-features = false, features = ["alloc"] }
-hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.3", features = ["raw"] }
-hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13.2" }
+hashbrown = { version = "0.14.3", features = ["raw"] }
 hex = { version = "0.4.3", features = ["serde"] }
 hmac = { version = "0.12.1", default-features = false, features = ["reset"] }
 hyper = { version = "0.14.27", features = ["full"] }
@@ -161,8 +160,7 @@ gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-
 generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] }
 getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] }
 group = { version = "0.13.0", default-features = false, features = ["alloc"] }
-hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.3", features = ["raw"] }
-hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13.2" }
+hashbrown = { version = "0.14.3", features = ["raw"] }
 hex = { version = "0.4.3", features = ["serde"] }
 hmac = { version = "0.12.1", default-features = false, features = ["reset"] }
 hyper = { version = "0.14.27", features = ["full"] }

From fedd8c87ccbf17af6251433a122652cf0962464e Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Thu, 15 Feb 2024 23:06:32 +0000
Subject: [PATCH 005/157] chore(deps): update rust crate serde_with to 3.6.1
 (#5077)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock | 9 +++++----
 Cargo.toml | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e82fd6a60f..c5f0192dbd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7931,9 +7931,9 @@ dependencies = [
 
 [[package]]
 name = "serde_with"
-version = "3.6.0"
+version = "3.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b0ed1662c5a68664f45b76d18deb0e234aff37207086803165c961eb695e981"
+checksum = "15d167997bd841ec232f5b2b8e0e26606df2e7caa4c31b95ea9ca52b200bd270"
 dependencies = [
  "base64",
  "chrono",
@@ -7941,6 +7941,7 @@ dependencies = [
  "indexmap 1.9.3",
  "indexmap 2.2.3",
  "serde",
+ "serde_derive",
  "serde_json",
  "serde_with_macros",
  "time",
@@ -7948,9 +7949,9 @@ dependencies = [
 
 [[package]]
 name = "serde_with_macros"
-version = "3.6.0"
+version = "3.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "568577ff0ef47b879f736cd66740e022f3672788cdf002a05a4e609ea5a6fb15"
+checksum = "865f9743393e638991566a8b7a479043c2c8da94a33e0a31f18214c9cae0a64d"
 dependencies = [
  "darling 0.20.3",
  "proc-macro2",
diff --git a/Cargo.toml b/Cargo.toml
index e5fa364305..bf9ebd9354 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -341,7 +341,7 @@ serde_json = "1.0.113"
 serde_path_to_error = "0.1.15"
 serde_tokenstream = "0.2"
 serde_urlencoded = "0.7.1"
-serde_with = "3.6.0"
+serde_with = "3.6.1"
 sha2 = "0.10.8"
 sha3 = "0.10.8"
 shell-words = "1.1.0"

From 1f48138840644e2573800b8f52dd00af35acc179 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Fri, 16 Feb 2024 01:32:15 +0000
Subject: [PATCH 006/157] chore(deps): update actions/setup-node action to
 v4.0.2 (#5075)

---
 .github/workflows/validate-openapi-spec.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/validate-openapi-spec.yml b/.github/workflows/validate-openapi-spec.yml
index a76567af2a..50aba2dc53 100644
--- a/.github/workflows/validate-openapi-spec.yml
+++ b/.github/workflows/validate-openapi-spec.yml
@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461
-      - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
+      - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
         with:
           node-version: '18'
       - name: Install our tools

From 235420ebd04831829ff3b2ce89fcd26f38c768f5 Mon Sep 17 00:00:00 2001
From: Rain <rain@oxide.computer>
Date: Thu, 15 Feb 2024 20:58:31 -0800
Subject: [PATCH 007/157] [nexus] basic scaffolding for typed UUIDs + a couple
 of simple examples (#4934)

This PR contains a basic implementation of typed UUIDs using the [newtype-uuid](https://crates.io/crates/newtype-uuid) crate, plus a simple conversion of two UUID types to this model: `LoopbackAddress` and `TufRepo`.

I picked these two types based on the fact that there were just a few places that used them.
---
 .config/hakari.toml                           |  11 +-
 Cargo.lock                                    |  36 +++
 Cargo.toml                                    |  28 +++
 common/Cargo.toml                             |   1 +
 common/src/api/external/error.rs              |   6 +
 nexus/authz-macros/Cargo.toml                 |   1 +
 nexus/authz-macros/outputs/rack.txt           |  75 ++++++
 nexus/authz-macros/src/lib.rs                 | 114 ++++++++-
 nexus/db-macros/Cargo.toml                    |   1 +
 .../outputs/asset_with_uuid_kind.txt          |  49 ++++
 nexus/db-macros/outputs/project.txt           |   6 +-
 .../outputs/resource_with_uuid_kind.txt       |  66 +++++
 nexus/db-macros/outputs/silo_user.txt         |   6 +-
 nexus/db-macros/outputs/sled.txt              | 182 ++++++++++++++
 nexus/db-macros/src/lib.rs                    | 229 +++++++++++++++---
 nexus/db-macros/src/lookup.rs                 | 111 +++++++--
 nexus/db-macros/src/test_helpers.rs           |  10 +
 nexus/db-model/Cargo.toml                     |   2 +
 nexus/db-model/src/lib.rs                     |   2 +
 nexus/db-model/src/switch_interface.rs        |   7 +-
 nexus/db-model/src/tuf_repo.rs                |  11 +-
 nexus/db-model/src/typed_uuid.rs              | 116 +++++++++
 nexus/db-queries/Cargo.toml                   |   3 +-
 nexus/db-queries/src/authz/api_resources.rs   |   4 +-
 .../src/authz/policy_test/resources.rs        |   5 +-
 .../src/db/datastore/switch_interface.rs      |   9 +-
 nexus/db-queries/src/db/datastore/update.rs   |  15 +-
 nexus/db-queries/src/db/lookup.rs             |   6 +-
 nexus/macros-common/Cargo.toml                |  11 +
 nexus/macros-common/src/lib.rs                | 109 +++++++++
 nexus/types/Cargo.toml                        |   1 +
 nexus/types/src/identity.rs                   |  13 +-
 uuid-kinds/Cargo.toml                         |  20 ++
 uuid-kinds/README.adoc                        |  73 ++++++
 uuid-kinds/src/lib.rs                         |  50 ++++
 35 files changed, 1297 insertions(+), 92 deletions(-)
 create mode 100644 nexus/authz-macros/outputs/rack.txt
 create mode 100644 nexus/db-macros/outputs/asset_with_uuid_kind.txt
 create mode 100644 nexus/db-macros/outputs/resource_with_uuid_kind.txt
 create mode 100644 nexus/db-macros/outputs/sled.txt
 create mode 100644 nexus/db-macros/src/test_helpers.rs
 create mode 100644 nexus/db-model/src/typed_uuid.rs
 create mode 100644 nexus/macros-common/Cargo.toml
 create mode 100644 nexus/macros-common/src/lib.rs
 create mode 100644 uuid-kinds/Cargo.toml
 create mode 100644 uuid-kinds/README.adoc
 create mode 100644 uuid-kinds/src/lib.rs

diff --git a/.config/hakari.toml b/.config/hakari.toml
index 0d883dc6f6..3c08a89e12 100644
--- a/.config/hakari.toml
+++ b/.config/hakari.toml
@@ -30,4 +30,13 @@ platforms = [
 exact-versions = true
 
 [traversal-excludes]
-workspace-members = ["xtask"]
+workspace-members = [
+    # Exclude xtask because it needs to be built quickly.
+    "xtask",
+    
+    # Exclude omicron-uuid-kinds because it is a no-std crate. Depending on the
+    # workspace-hack isn't too problematic because other projects pulling in
+    # omicron as a git dependency will only see an empty workspace-hack. But
+    # let's make this explicit.
+    "omicron-uuid-kinds",
+]
diff --git a/Cargo.lock b/Cargo.lock
index c5f0192dbd..fca461f24b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -354,6 +354,7 @@ version = "0.1.0"
 dependencies = [
  "expectorate",
  "heck 0.4.1",
+ "nexus-macros-common",
  "omicron-workspace-hack",
  "prettyplease",
  "proc-macro2",
@@ -1548,6 +1549,7 @@ version = "0.1.0"
 dependencies = [
  "expectorate",
  "heck 0.4.1",
+ "nexus-macros-common",
  "omicron-workspace-hack",
  "prettyplease",
  "proc-macro2",
@@ -4256,6 +4258,17 @@ dependencies = [
  "unicode-segmentation",
 ]
 
+[[package]]
+name = "newtype-uuid"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a5ff2b31594942586c1520da8f1e5c705729ec67b3c2ad0fe459f0b576e4d9a"
+dependencies = [
+ "schemars",
+ "serde",
+ "uuid",
+]
+
 [[package]]
 name = "newtype_derive"
 version = "0.1.6"
@@ -4318,6 +4331,7 @@ dependencies = [
  "anyhow",
  "chrono",
  "db-macros",
+ "derive-where",
  "diesel",
  "expectorate",
  "hex",
@@ -4330,6 +4344,7 @@ dependencies = [
  "omicron-common",
  "omicron-passwords",
  "omicron-rpaths",
+ "omicron-uuid-kinds",
  "omicron-workspace-hack",
  "parse-display",
  "pq-sys",
@@ -4388,6 +4403,7 @@ dependencies = [
  "omicron-rpaths",
  "omicron-sled-agent",
  "omicron-test-utils",
+ "omicron-uuid-kinds",
  "omicron-workspace-hack",
  "once_cell",
  "openapiv3",
@@ -4482,6 +4498,16 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "nexus-macros-common"
+version = "0.1.0"
+dependencies = [
+ "omicron-workspace-hack",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.48",
+]
+
 [[package]]
 name = "nexus-test-interface"
 version = "0.1.0"
@@ -4557,6 +4583,7 @@ dependencies = [
  "gateway-client",
  "omicron-common",
  "omicron-passwords",
+ "omicron-uuid-kinds",
  "omicron-workspace-hack",
  "openssl",
  "parse-display",
@@ -4865,6 +4892,7 @@ dependencies = [
  "ipnetwork",
  "libc",
  "macaddr",
+ "omicron-uuid-kinds",
  "omicron-workspace-hack",
  "once_cell",
  "parse-display",
@@ -5319,6 +5347,14 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "omicron-uuid-kinds"
+version = "0.1.0"
+dependencies = [
+ "newtype-uuid",
+ "schemars",
+]
+
 [[package]]
 name = "omicron-workspace-hack"
 version = "0.1.0"
diff --git a/Cargo.toml b/Cargo.toml
index bf9ebd9354..f71bd77730 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -44,6 +44,7 @@ members = [
     "nexus/defaults",
     "nexus/deployment",
     "nexus/inventory",
+    "nexus/macros-common",
     "nexus/test-interface",
     "nexus/test-utils-macros",
     "nexus/test-utils",
@@ -66,6 +67,7 @@ members = [
     "tufaceous",
     "update-common",
     "update-engine",
+    "uuid-kinds",
     "wicket-common",
     "wicket-dbg",
     "wicket",
@@ -113,6 +115,7 @@ default-members = [
     "nexus",
     "nexus/authz-macros",
     "nexus/blueprint-execution",
+    "nexus/macros-common",
     "nexus/db-macros",
     "nexus/db-model",
     "nexus/db-queries",
@@ -138,6 +141,7 @@ default-members = [
     "tufaceous",
     "update-common",
     "update-engine",
+    "uuid-kinds",
     "wicket-common",
     "wicket-dbg",
     "wicket",
@@ -256,6 +260,7 @@ nexus-db-queries = { path = "nexus/db-queries" }
 nexus-defaults = { path = "nexus/defaults" }
 nexus-deployment = { path = "nexus/deployment" }
 nexus-inventory = { path = "nexus/inventory" }
+nexus-macros-common = { path = "nexus/macros-common" }
 omicron-certificates = { path = "certificates" }
 omicron-passwords = { path = "passwords" }
 omicron-workspace-hack = "0.1.0"
@@ -415,6 +420,14 @@ zeroize = { version = "1.7.0", features = ["zeroize_derive", "std"] }
 zip = { version = "0.6.6", default-features = false, features = ["deflate","bzip2"] }
 zone = { version = "0.3", default-features = false, features = ["async", "sync"] }
 
+# newtype-uuid is set to default-features = false because we don't want to
+# depend on std in omicron-uuid-kinds (in case a no-std library wants to access
+# the kinds). However, uses of omicron-uuid-kinds _within omicron_ will have
+# std and the other features enabled because they'll refer to it via
+# omicron-uuid-kinds.workspace = true.
+newtype-uuid = { version = "1.0.1", default-features = false }
+omicron-uuid-kinds = { path = "uuid-kinds", features = ["serde", "schemars08", "uuid-v4"] }
+
 # NOTE: The test profile inherits from the dev profile, so settings under
 # profile.dev get inherited. AVOID setting anything under profile.test: that
 # will cause dev and test builds to diverge, which will cause more Cargo build
@@ -570,6 +583,7 @@ opt-level = 3
 [profile.dev.package.keccak]
 opt-level = 3
 
+
 #
 # It's common during development to use a local copy of various complex
 # dependencies.  If you want to use those, uncomment one of these blocks.
@@ -617,3 +631,17 @@ path = "workspace-hack"
 [patch.crates-io.samael]
 git = "https://github.com/oxidecomputer/samael"
 branch = "oxide/omicron"
+
+# Several crates such as crucible and propolis have have a Git dependency on
+# this repo. Omicron itself depends on these crates, which can lead to two
+# copies of these crates in the dependency graph. (As a Git dependency, and as
+# a path dependency.) The goal of omicron-uuid-kinds is to provide a unified
+# registry of UUID kinds. Two copies of the same kinds floating around is
+# unnecessary and painful.
+#
+# This directive ensures that whenever we see omicron-uuid-kinds as a Git
+# dependency, we'll use the path dependency version of the crate instead.
+#
+# See also: uuid-kinds/README.adoc.
+[patch."https://github.com/oxidecomputer/omicron"]
+omicron-uuid-kinds = { path = "uuid-kinds" }
diff --git a/common/Cargo.toml b/common/Cargo.toml
index ebb8c8c9b4..5628a93397 100644
--- a/common/Cargo.toml
+++ b/common/Cargo.toml
@@ -17,6 +17,7 @@ hex.workspace = true
 http.workspace = true
 ipnetwork.workspace = true
 macaddr.workspace = true
+omicron-uuid-kinds.workspace = true
 proptest = { workspace = true, optional = true }
 rand.workspace = true
 reqwest = { workspace = true, features = ["rustls-tls", "stream"] }
diff --git a/common/src/api/external/error.rs b/common/src/api/external/error.rs
index f7eb257e8f..2cb3dc0d6e 100644
--- a/common/src/api/external/error.rs
+++ b/common/src/api/external/error.rs
@@ -9,6 +9,7 @@
 use crate::api::external::Name;
 use crate::api::external::ResourceType;
 use dropshot::HttpError;
+use omicron_uuid_kinds::GenericUuid;
 use serde::Deserialize;
 use serde::Serialize;
 use std::fmt::Display;
@@ -152,6 +153,11 @@ pub enum LookupType {
 }
 
 impl LookupType {
+    /// Constructs a `ById` lookup type from a typed or untyped UUID.
+    pub fn by_id<T: GenericUuid>(id: T) -> Self {
+        LookupType::ById(id.into_untyped_uuid())
+    }
+
     /// Returns an ObjectNotFound error appropriate for the case where this
     /// lookup failed
     pub fn into_not_found(self, type_name: ResourceType) -> Error {
diff --git a/nexus/authz-macros/Cargo.toml b/nexus/authz-macros/Cargo.toml
index e9bdaf4708..4d2640abee 100644
--- a/nexus/authz-macros/Cargo.toml
+++ b/nexus/authz-macros/Cargo.toml
@@ -9,6 +9,7 @@ proc-macro = true
 
 [dependencies]
 heck.workspace = true
+nexus-macros-common.workspace = true
 proc-macro2.workspace = true
 quote.workspace = true
 serde.workspace = true
diff --git a/nexus/authz-macros/outputs/rack.txt b/nexus/authz-macros/outputs/rack.txt
new file mode 100644
index 0000000000..40826951ee
--- /dev/null
+++ b/nexus/authz-macros/outputs/rack.txt
@@ -0,0 +1,75 @@
+///`authz` type for a resource of type RackUsed to uniquely identify a resource of type Rack across renames, moves, etc., and to do authorization checks (see  [`crate::context::OpContext::authorize()`]).  See [`crate::authz`] module-level documentation for more information.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Rack {
+    parent: Fleet,
+    key: ::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::RackKind>,
+    lookup_type: LookupType,
+}
+impl Rack {
+    /// Makes a new `authz` struct for this resource with the given
+    /// `parent`, unique key `key`, looked up as described by
+    /// `lookup_type`
+    pub fn new(
+        parent: Fleet,
+        key: ::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::RackKind>,
+        lookup_type: LookupType,
+    ) -> Rack {
+        Rack {
+            parent,
+            key: key.into(),
+            lookup_type,
+        }
+    }
+    /// A version of `new` that takes the primary key type directly.
+    /// This is only different from [`Self::new`] if this resource
+    /// uses a different input key type.
+    pub fn with_primary_key(
+        parent: Fleet,
+        key: ::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::RackKind>,
+        lookup_type: LookupType,
+    ) -> Rack {
+        Rack { parent, key, lookup_type }
+    }
+    pub fn id(&self) -> ::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::RackKind> {
+        self.key.clone().into()
+    }
+    /// Describes how to register this type with Oso
+    pub(super) fn init() -> Init {
+        use oso::PolarClass;
+        Init {
+            polar_snippet: "\n                resource Rack {\n                    permissions = [\n                        \"list_children\",\n                        \"modify\",\n                        \"read\",\n                        \"create_child\",\n                    ];\n                    \n                    relations = { parent_fleet: Fleet };\n                    \"list_children\" if \"viewer\" on \"parent_fleet\";\n                    \"read\" if \"viewer\" on \"parent_fleet\";\n                    \"modify\" if \"admin\" on \"parent_fleet\";\n                    \"create_child\" if \"admin\" on \"parent_fleet\";\n                }\n                has_relation(fleet: Fleet, \"parent_fleet\", child: Rack)\n                    if child.fleet = fleet;\n            ",
+            polar_class: Rack::get_polar_class(),
+        }
+    }
+}
+impl Eq for Rack {}
+impl PartialEq for Rack {
+    fn eq(&self, other: &Self) -> bool {
+        self.key == other.key
+    }
+}
+impl oso::PolarClass for Rack {
+    fn get_polar_class_builder() -> oso::ClassBuilder<Self> {
+        oso::Class::builder()
+            .with_equality_check()
+            .add_method(
+                "has_role",
+                |r: &Rack, actor: AuthenticatedActor, role: String| { false },
+            )
+            .add_attribute_getter("fleet", |r: &Rack| r.parent.clone())
+    }
+}
+impl ApiResource for Rack {
+    fn parent(&self) -> Option<&dyn AuthorizedResource> {
+        Some(&self.parent)
+    }
+    fn resource_type(&self) -> ResourceType {
+        ResourceType::Rack
+    }
+    fn lookup_type(&self) -> &LookupType {
+        &self.lookup_type
+    }
+    fn as_resource_with_roles(&self) -> Option<&dyn ApiResourceWithRoles> {
+        None
+    }
+}
diff --git a/nexus/authz-macros/src/lib.rs b/nexus/authz-macros/src/lib.rs
index 648ae6d952..0548113339 100644
--- a/nexus/authz-macros/src/lib.rs
+++ b/nexus/authz-macros/src/lib.rs
@@ -6,9 +6,11 @@
 
 extern crate proc_macro;
 
+use nexus_macros_common::PrimaryKeyType;
 use proc_macro2::TokenStream;
 use quote::{format_ident, quote};
 use serde_tokenstream::ParseWrapper;
+use syn::parse_quote;
 
 /// Defines a structure and helpers for describing an API resource for authz
 ///
@@ -81,6 +83,24 @@ use serde_tokenstream::ParseWrapper;
 /// }
 /// ```
 ///
+/// ## Resources with typed UUID primary keys
+///
+/// Some resources use a [newtype_uuid](https://crates.io/crates/newtype_uuid)
+/// `TypedUuid` as their primary key (and resources should generally move over
+/// to that model).
+///
+/// This can be specified with `primary_key = { uuid_kind = MyKind }`:
+///
+/// ```ignore
+/// authz_resource! {
+///     name = "LoopbackAddress",
+///     parent = "Fleet",
+///     primary_key = { uuid_kind = LoopbackAddressKind },
+///     roles_allowed = false,
+///     polar_snippet = FleetChild,
+/// }
+/// ```
+///
 /// ## Resources with non-id primary keys
 ///
 /// Most API resources use "id" (a Uuid) as an immutable, unique identifier.
@@ -137,7 +157,7 @@ pub fn authz_resource(
 }
 
 /// Arguments for [`authz_resource!`]
-#[derive(serde::Deserialize)]
+#[derive(serde::Deserialize, Debug)]
 struct Input {
     /// Name of the resource
     ///
@@ -146,8 +166,10 @@ struct Input {
     name: String,
     /// Name of the parent `authz` resource
     parent: String,
-    /// Rust type for the primary key for this resource
-    primary_key: ParseWrapper<syn::Type>,
+    /// Rust type for the primary key for this resource.
+    primary_key: InputPrimaryKeyType,
+    /// The `TypedUuidKind` for this resource. Must be exclusive
+    ///
     /// Rust type for the input key for this resource (the key users specify
     /// for this resource, convertible to `primary_key`).
     ///
@@ -160,8 +182,77 @@ struct Input {
     polar_snippet: PolarSnippet,
 }
 
+#[derive(Debug)]
+struct InputPrimaryKeyType(PrimaryKeyType);
+
+impl<'de> serde::Deserialize<'de> for InputPrimaryKeyType {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        // Attempt to parse as either a string or a map.
+        struct PrimaryKeyVisitor;
+
+        impl<'de2> serde::de::Visitor<'de2> for PrimaryKeyVisitor {
+            type Value = PrimaryKeyType;
+
+            fn expecting(
+                &self,
+                formatter: &mut std::fmt::Formatter,
+            ) -> std::fmt::Result {
+                formatter.write_str(
+                    "a Rust type, or a map with a single key `uuid_kind`",
+                )
+            }
+
+            fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+            where
+                E: serde::de::Error,
+            {
+                syn::parse_str(value)
+                    .map(PrimaryKeyType::Standard)
+                    .map_err(|e| E::custom(e.to_string()))
+            }
+
+            // seq represents a tuple type
+            fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+            where
+                A: serde::de::SeqAccess<'de2>,
+            {
+                let mut elements = vec![];
+                while let Some(element) =
+                    seq.next_element::<ParseWrapper<syn::Type>>()?
+                {
+                    elements.push(element.into_inner());
+                }
+                Ok(PrimaryKeyType::Standard(parse_quote!((#(#elements,)*))))
+            }
+
+            fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
+            where
+                A: serde::de::MapAccess<'de2>,
+            {
+                let key: String = map.next_key()?.ok_or_else(|| {
+                    serde::de::Error::custom("expected a single key")
+                })?;
+                if key == "uuid_kind" {
+                    // uuid kinds must be plain identifiers
+                    let value: ParseWrapper<syn::Ident> = map.next_value()?;
+                    Ok(PrimaryKeyType::new_typed_uuid(&value))
+                } else {
+                    Err(serde::de::Error::custom(
+                        "expected a single key `uuid_kind`",
+                    ))
+                }
+            }
+        }
+
+        deserializer.deserialize_any(PrimaryKeyVisitor).map(InputPrimaryKeyType)
+    }
+}
+
 /// How to generate the Polar snippet for this resource
-#[derive(serde::Deserialize)]
+#[derive(serde::Deserialize, Debug)]
 enum PolarSnippet {
     /// Don't generate it at all -- it's generated elsewhere
     Custom,
@@ -185,9 +276,8 @@ fn do_authz_resource(
     let resource_name = format_ident!("{}", input.name);
     let parent_resource_name = format_ident!("{}", input.parent);
     let parent_as_snake = heck::AsSnakeCase(&input.parent).to_string();
-    let primary_key_type = &*input.primary_key;
-    let input_key_type =
-        &**input.input_key.as_ref().unwrap_or(&input.primary_key);
+    let primary_key_type = input.primary_key.0.external();
+    let input_key_type = input.input_key.as_deref().unwrap_or(primary_key_type);
 
     let (has_role_body, as_roles_body, api_resource_roles_trait) =
         if input.roles_allowed {
@@ -493,6 +583,16 @@ mod tests {
         })
         .unwrap();
         assert_contents("outputs/instance.txt", &pretty_format(output));
+
+        let output = do_authz_resource(quote! {
+            name = "Rack",
+            parent = "Fleet",
+            primary_key = { uuid_kind = RackKind },
+            roles_allowed = false,
+            polar_snippet = FleetChild,
+        })
+        .unwrap();
+        assert_contents("outputs/rack.txt", &pretty_format(output));
     }
 
     fn pretty_format(input: TokenStream) -> String {
diff --git a/nexus/db-macros/Cargo.toml b/nexus/db-macros/Cargo.toml
index 46e5d9a5d6..8032ba814d 100644
--- a/nexus/db-macros/Cargo.toml
+++ b/nexus/db-macros/Cargo.toml
@@ -10,6 +10,7 @@ proc-macro = true
 
 [dependencies]
 heck.workspace = true
+nexus-macros-common.workspace = true
 proc-macro2.workspace = true
 quote.workspace = true
 serde.workspace = true
diff --git a/nexus/db-macros/outputs/asset_with_uuid_kind.txt b/nexus/db-macros/outputs/asset_with_uuid_kind.txt
new file mode 100644
index 0000000000..a4a756248d
--- /dev/null
+++ b/nexus/db-macros/outputs/asset_with_uuid_kind.txt
@@ -0,0 +1,49 @@
+///Auto-generated identity for [`AssetWithUuidKind`] from deriving [`macro@Asset`].
+#[derive(
+    Clone,
+    Debug,
+    PartialEq,
+    Selectable,
+    Queryable,
+    Insertable,
+    serde::Serialize,
+    serde::Deserialize
+)]
+#[diesel(table_name = my_target)]
+pub struct AssetWithUuidKindIdentity {
+    pub id: crate::typed_uuid::DbTypedUuid<::omicron_uuid_kinds::CustomKind>,
+    pub time_created: ::chrono::DateTime<::chrono::Utc>,
+    pub time_modified: ::chrono::DateTime<::chrono::Utc>,
+}
+impl AssetWithUuidKindIdentity {
+    pub fn new(
+        id: ::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::CustomKind>,
+    ) -> Self {
+        let now = ::chrono::Utc::now();
+        Self {
+            id: crate::to_db_typed_uuid(id),
+            time_created: now,
+            time_modified: now,
+        }
+    }
+}
+trait __AssetWithUuidKindIdentityMarker {}
+impl __AssetWithUuidKindIdentityMarker for AssetWithUuidKindIdentity {}
+const _: () = {
+    fn assert_identity<T: __AssetWithUuidKindIdentityMarker>() {}
+    fn assert_all() {
+        assert_identity::<AssetWithUuidKindIdentity>();
+    }
+};
+impl ::nexus_types::identity::Asset for AssetWithUuidKind {
+    type IdType = ::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::CustomKind>;
+    fn id(&self) -> ::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::CustomKind> {
+        ::omicron_uuid_kinds::TypedUuid::from(self.identity.id)
+    }
+    fn time_created(&self) -> ::chrono::DateTime<::chrono::Utc> {
+        self.identity.time_created
+    }
+    fn time_modified(&self) -> ::chrono::DateTime<::chrono::Utc> {
+        self.identity.time_modified
+    }
+}
diff --git a/nexus/db-macros/outputs/project.txt b/nexus/db-macros/outputs/project.txt
index 9f4b5cfaa2..333cdb7acf 100644
--- a/nexus/db-macros/outputs/project.txt
+++ b/nexus/db-macros/outputs/project.txt
@@ -373,7 +373,9 @@ impl<'a> Project<'a> {
                     e,
                     ErrorHandler::NotFoundByLookup(
                         ResourceType::Project,
-                        LookupType::ById(v0.clone()),
+                        LookupType::ById(
+                            ::omicron_uuid_kinds::GenericUuid::into_untyped_uuid(*v0),
+                        ),
                     ),
                 )
             })?;
@@ -386,7 +388,7 @@ impl<'a> Project<'a> {
         let authz_project = Self::make_authz(
             &authz_silo,
             &db_row,
-            LookupType::ById(v0.clone()),
+            LookupType::ById(::omicron_uuid_kinds::GenericUuid::into_untyped_uuid(*v0)),
         );
         Ok((authz_silo, authz_project, db_row))
     }
diff --git a/nexus/db-macros/outputs/resource_with_uuid_kind.txt b/nexus/db-macros/outputs/resource_with_uuid_kind.txt
new file mode 100644
index 0000000000..8d1628e7c9
--- /dev/null
+++ b/nexus/db-macros/outputs/resource_with_uuid_kind.txt
@@ -0,0 +1,66 @@
+///Auto-generated identity for [`ResourceWithUuidKind`] from deriving [`macro@Resource`].
+#[derive(
+    Clone,
+    Debug,
+    PartialEq,
+    Eq,
+    Selectable,
+    Queryable,
+    Insertable,
+    serde::Serialize,
+    serde::Deserialize
+)]
+#[diesel(table_name = my_target)]
+pub struct ResourceWithUuidKindIdentity {
+    pub id: crate::typed_uuid::DbTypedUuid<::omicron_uuid_kinds::CustomKind>,
+    pub name: crate::db::model::Name,
+    pub description: ::std::string::String,
+    pub time_created: ::chrono::DateTime<::chrono::Utc>,
+    pub time_modified: ::chrono::DateTime<::chrono::Utc>,
+    pub time_deleted: ::std::option::Option<chrono::DateTime<chrono::Utc>>,
+}
+impl ResourceWithUuidKindIdentity {
+    pub fn new(
+        id: ::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::CustomKind>,
+        params: ::omicron_common::api::external::IdentityMetadataCreateParams,
+    ) -> Self {
+        let now = ::chrono::Utc::now();
+        Self {
+            id: crate::to_db_typed_uuid(id),
+            name: params.name.into(),
+            description: params.description,
+            time_created: now,
+            time_modified: now,
+            time_deleted: None,
+        }
+    }
+}
+trait __ResourceWithUuidKindIdentityMarker {}
+impl __ResourceWithUuidKindIdentityMarker for ResourceWithUuidKindIdentity {}
+const _: () = {
+    fn assert_identity<T: __ResourceWithUuidKindIdentityMarker>() {}
+    fn assert_all() {
+        assert_identity::<ResourceWithUuidKindIdentity>();
+    }
+};
+impl ::nexus_types::identity::Resource for ResourceWithUuidKind {
+    type IdType = ::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::CustomKind>;
+    fn id(&self) -> ::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::CustomKind> {
+        ::omicron_uuid_kinds::TypedUuid::from(self.identity.id)
+    }
+    fn name(&self) -> &::omicron_common::api::external::Name {
+        &self.identity.name.0
+    }
+    fn description(&self) -> &str {
+        &self.identity.description
+    }
+    fn time_created(&self) -> ::chrono::DateTime<::chrono::Utc> {
+        self.identity.time_created
+    }
+    fn time_modified(&self) -> ::chrono::DateTime<::chrono::Utc> {
+        self.identity.time_modified
+    }
+    fn time_deleted(&self) -> ::std::option::Option<::chrono::DateTime<::chrono::Utc>> {
+        self.identity.time_deleted
+    }
+}
diff --git a/nexus/db-macros/outputs/silo_user.txt b/nexus/db-macros/outputs/silo_user.txt
index 2c9568ff54..1f42db54e8 100644
--- a/nexus/db-macros/outputs/silo_user.txt
+++ b/nexus/db-macros/outputs/silo_user.txt
@@ -169,14 +169,16 @@ impl<'a> SiloUser<'a> {
                     e,
                     ErrorHandler::NotFoundByLookup(
                         ResourceType::SiloUser,
-                        LookupType::ById(v0.clone()),
+                        LookupType::ById(
+                            ::omicron_uuid_kinds::GenericUuid::into_untyped_uuid(*v0),
+                        ),
                     ),
                 )
             })?;
         let authz_silo_user = Self::make_authz(
             &&authz::FLEET,
             &db_row,
-            LookupType::ById(v0.clone()),
+            LookupType::ById(::omicron_uuid_kinds::GenericUuid::into_untyped_uuid(*v0)),
         );
         Ok((authz_silo_user, db_row))
     }
diff --git a/nexus/db-macros/outputs/sled.txt b/nexus/db-macros/outputs/sled.txt
new file mode 100644
index 0000000000..30654b0e90
--- /dev/null
+++ b/nexus/db-macros/outputs/sled.txt
@@ -0,0 +1,182 @@
+///Selects a resource of type Sled (or any of its children, using the functions on this struct) for lookup or fetch
+pub enum Sled<'a> {
+    /// An error occurred while selecting the resource
+    ///
+    /// This error will be returned by any lookup/fetch attempts.
+    Error(Root<'a>, Error),
+    /// We're looking for a resource with the given primary key
+    ///
+    /// This has no parent container -- a by-id lookup is always global
+    PrimaryKey(
+        Root<'a>,
+        ::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::SledKind>,
+    ),
+}
+impl<'a> Sled<'a> {
+    /// Fetch the record corresponding to the selected resource
+    ///
+    /// This is equivalent to `fetch_for(authz::Action::Read)`.
+    pub async fn fetch(&self) -> LookupResult<(authz::Sled, nexus_db_model::Sled)> {
+        self.fetch_for(authz::Action::Read).await
+    }
+    /// Turn the Result<T, E> of [`fetch`] into a Result<Option<T>, E>.
+    pub async fn optional_fetch(
+        &self,
+    ) -> LookupResult<Option<(authz::Sled, nexus_db_model::Sled)>> {
+        self.optional_fetch_for(authz::Action::Read).await
+    }
+    /// Fetch the record corresponding to the selected resource and
+    /// check whether the caller is allowed to do the specified `action`
+    ///
+    /// The return value is a tuple that also includes the `authz`
+    /// objects for all resources along the path to this one (i.e., all
+    /// parent resources) and the authz object for this resource itself.
+    /// These objects are useful for identifying those resources by
+    /// id, for doing other authz checks, or for looking up related
+    /// objects.
+    pub async fn fetch_for(
+        &self,
+        action: authz::Action,
+    ) -> LookupResult<(authz::Sled, nexus_db_model::Sled)> {
+        let lookup = self.lookup_root();
+        let opctx = &lookup.opctx;
+        let datastore = &lookup.datastore;
+        match &self {
+            Sled::Error(_, error) => Err(error.clone()),
+            Sled::PrimaryKey(_, v0) => {
+                Self::fetch_by_id_for(opctx, datastore, v0, action).await
+            }
+        }
+    }
+    /// Turn the Result<T, E> of [`fetch_for`] into a Result<Option<T>, E>.
+    pub async fn optional_fetch_for(
+        &self,
+        action: authz::Action,
+    ) -> LookupResult<Option<(authz::Sled, nexus_db_model::Sled)>> {
+        let result = self.fetch_for(action).await;
+        match result {
+            Err(Error::ObjectNotFound { type_name: _, lookup_type: _ }) => Ok(None),
+            _ => Ok(Some(result?)),
+        }
+    }
+    /// Fetch an `authz` object for the selected resource and check
+    /// whether the caller is allowed to do the specified `action`
+    ///
+    /// The return value is a tuple that also includes the `authz`
+    /// objects for all resources along the path to this one (i.e., all
+    /// parent resources) and the authz object for this resource itself.
+    /// These objects are useful for identifying those resources by
+    /// id, for doing other authz checks, or for looking up related
+    /// objects.
+    pub async fn lookup_for(
+        &self,
+        action: authz::Action,
+    ) -> LookupResult<(authz::Sled,)> {
+        let lookup = self.lookup_root();
+        let opctx = &lookup.opctx;
+        let (authz_sled,) = self.lookup().await?;
+        opctx.authorize(action, &authz_sled).await?;
+        Ok((authz_sled,))
+    }
+    /// Turn the Result<T, E> of [`lookup_for`] into a Result<Option<T>, E>.
+    pub async fn optional_lookup_for(
+        &self,
+        action: authz::Action,
+    ) -> LookupResult<Option<(authz::Sled,)>> {
+        let result = self.lookup_for(action).await;
+        match result {
+            Err(Error::ObjectNotFound { type_name: _, lookup_type: _ }) => Ok(None),
+            _ => Ok(Some(result?)),
+        }
+    }
+    /// Fetch the "authz" objects for the selected resource and all its
+    /// parents
+    ///
+    /// This function does not check whether the caller has permission
+    /// to read this information.  That's why it's not `pub`.  Outside
+    /// this module, you want `lookup_for(authz::Action)`.
+    async fn lookup(&self) -> LookupResult<(authz::Sled,)> {
+        let lookup = self.lookup_root();
+        let opctx = &lookup.opctx;
+        let datastore = &lookup.datastore;
+        match &self {
+            Sled::Error(_, error) => Err(error.clone()),
+            Sled::PrimaryKey(_, v0) => {
+                let (authz_sled, _) = Self::lookup_by_id_no_authz(opctx, datastore, v0)
+                    .await?;
+                Ok((authz_sled,))
+            }
+        }
+    }
+    /// Build the `authz` object for this resource
+    fn make_authz(
+        authz_parent: &authz::Fleet,
+        db_row: &nexus_db_model::Sled,
+        lookup_type: LookupType,
+    ) -> authz::Sled {
+        authz::Sled::with_primary_key(authz_parent.clone(), db_row.id(), lookup_type)
+    }
+    /// Getting the [`LookupPath`] for this lookup
+    ///
+    /// This is used when we actually query the database.  At that
+    /// point, we need the `OpContext` and `DataStore` that are being
+    /// used for this lookup.
+    fn lookup_root(&self) -> &LookupPath<'a> {
+        match &self {
+            Sled::Error(root, ..) => root.lookup_root(),
+            Sled::PrimaryKey(root, ..) => root.lookup_root(),
+        }
+    }
+    /// Fetch the database row for a resource by doing a lookup by id
+    ///
+    /// This function checks whether the caller has permissions to read
+    /// the requested data.  However, it's not intended to be used
+    /// outside this module.  See `fetch_for(authz::Action)`.
+    async fn fetch_by_id_for(
+        opctx: &OpContext,
+        datastore: &DataStore,
+        v0: &::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::SledKind>,
+        action: authz::Action,
+    ) -> LookupResult<(authz::Sled, nexus_db_model::Sled)> {
+        let (authz_sled, db_row) = Self::lookup_by_id_no_authz(opctx, datastore, v0)
+            .await?;
+        opctx.authorize(action, &authz_sled).await?;
+        Ok((authz_sled, db_row))
+    }
+    /// Lowest-level function for looking up a resource in the database
+    /// by id
+    ///
+    /// This function does not check whether the caller has permission
+    /// to read this information.  That's why it's not `pub`.  Outside
+    /// this module, you want `fetch()` or `lookup_for(authz::Action)`.
+    async fn lookup_by_id_no_authz(
+        opctx: &OpContext,
+        datastore: &DataStore,
+        v0: &::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::SledKind>,
+    ) -> LookupResult<(authz::Sled, nexus_db_model::Sled)> {
+        use db::schema::sled::dsl;
+        let db_row = dsl::sled
+            .filter(dsl::time_deleted.is_null())
+            .filter(dsl::id.eq(::nexus_db_model::to_db_typed_uuid(v0.clone())))
+            .select(nexus_db_model::Sled::as_select())
+            .get_result_async(&*datastore.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| {
+                public_error_from_diesel(
+                    e,
+                    ErrorHandler::NotFoundByLookup(
+                        ResourceType::Sled,
+                        LookupType::ById(
+                            ::omicron_uuid_kinds::GenericUuid::into_untyped_uuid(*v0),
+                        ),
+                    ),
+                )
+            })?;
+        let authz_sled = Self::make_authz(
+            &&authz::FLEET,
+            &db_row,
+            LookupType::ById(::omicron_uuid_kinds::GenericUuid::into_untyped_uuid(*v0)),
+        );
+        Ok((authz_sled, db_row))
+    }
+}
diff --git a/nexus/db-macros/src/lib.rs b/nexus/db-macros/src/lib.rs
index fd15b59128..fd9aae4b0a 100644
--- a/nexus/db-macros/src/lib.rs
+++ b/nexus/db-macros/src/lib.rs
@@ -13,13 +13,17 @@
 
 extern crate proc_macro;
 
+use nexus_macros_common::PrimaryKeyType;
 use proc_macro2::TokenStream;
 use quote::{format_ident, quote};
+use serde_tokenstream::ParseWrapper;
 use syn::spanned::Spanned;
-use syn::{Data, DataStruct, DeriveInput, Error, Fields, Ident};
+use syn::{parse_quote, Data, DataStruct, DeriveInput, Error, Fields, Ident};
 
 mod lookup;
 mod subquery;
+#[cfg(test)]
+mod test_helpers;
 
 /// Defines a structure and helper functions for looking up resources
 ///
@@ -65,8 +69,24 @@ mod subquery;
 /// }
 /// ```
 ///
-/// These define `Project<'a>` and `Instance<'a>`.  For more on these structs
-/// and how they're used, see nexus/src/db/lookup.rs.
+/// These define `Project<'a>` and `Instance<'a>`.
+///
+/// It is also possible to use a `TypedUuid` as a key column, by specifying
+/// `uuid_kind` rather than `rust_type`. For example:
+///
+/// ```ignore
+/// lookup_resource! {
+///     name = "Sled",
+///     ancestors = [ "Organization", "Project" ],
+///     children = [],
+///     lookup_by_name = true,
+///     soft_deletes = true,
+///     primary_key_columns = [ { column_name = "id", uuid_kind = SledType } ]
+/// }
+/// ```
+///
+/// For more on these structs and how they're used, see
+/// nexus/db-queries/src/lookup.rs.
 // Allow private intra-doc links.  This is useful because the `Input` struct
 // cannot be exported (since we're a proc macro crate, and we can't expose
 // a struct), but its documentation is very useful.
@@ -81,11 +101,14 @@ pub fn lookup_resource(
     }
 }
 
-/// Looks for a Meta-style attribute with a particular identifier.
+/// Looks for a Diesel Meta-style attribute with a particular identifier.
 ///
 /// As an example, for an attribute like `#[diesel(foo = bar)]`, we can find this
 /// attribute by calling `get_nv_attr(&item.attrs, "foo")`.
-fn get_nv_attr(attrs: &[syn::Attribute], name: &str) -> Option<NameValue> {
+fn get_diesel_nv_attr(
+    attrs: &[syn::Attribute],
+    name: &str,
+) -> Option<NameValue> {
     attrs
         .iter()
         .filter(|attr| attr.path().is_ident("diesel"))
@@ -141,11 +164,21 @@ pub fn subquery_target(
 }
 
 // Describes which derive macro is being used; allows sharing common code.
+#[derive(Clone, Copy, Debug)]
 enum IdentityVariant {
     Asset,
     Resource,
 }
 
+impl IdentityVariant {
+    fn attr_name(&self) -> &'static str {
+        match self {
+            IdentityVariant::Asset => "asset",
+            IdentityVariant::Resource => "resource",
+        }
+    }
+}
+
 /// Implements the "Resource" trait, and generates a bespoke Identity struct.
 ///
 /// Many tables within our database make use of common fields,
@@ -160,7 +193,7 @@ enum IdentityVariant {
 /// Although these fields can be refactored into a common structure (to be used
 /// within the context of Diesel) they must be uniquely identified for a single
 /// table.
-#[proc_macro_derive(Resource)]
+#[proc_macro_derive(Resource, attributes(resource))]
 pub fn resource_target(
     input: proc_macro::TokenStream,
 ) -> proc_macro::TokenStream {
@@ -175,7 +208,7 @@ pub fn resource_target(
 /// - ID
 /// - Time Created
 /// - Time Modified
-#[proc_macro_derive(Asset)]
+#[proc_macro_derive(Asset, attributes(asset))]
 pub fn asset_target(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
     derive_impl(input.into(), IdentityVariant::Asset)
         .unwrap_or_else(|e| e.to_compile_error())
@@ -208,18 +241,22 @@ fn derive_impl(
     let name = &item.ident;
 
     // Ensure that the "table_name" attribute exists, and get it.
-    let table_nv = get_nv_attr(&item.attrs, "table_name").ok_or_else(|| {
-        Error::new(
-            item.span(),
-            format!(
-                "Resource needs 'table_name' attribute.\n\
+    let table_nv =
+        get_diesel_nv_attr(&item.attrs, "table_name").ok_or_else(|| {
+            Error::new(
+                item.span(),
+                format!(
+                    "Resource needs 'table_name' attribute.\n\
                      Try adding #[diesel(table_name = your_table_name)] to {}.",
-                name
-            ),
-        )
-    })?;
+                    name
+                ),
+            )
+        })?;
     let table_name = table_nv.value;
 
+    let input = MacroAttributes::parse_from_attrs(&item.attrs, flavor)?;
+    let uuid_ty = input.uuid_ty();
+
     // Ensure that a field named "identity" exists within this struct.
     if let Data::Struct(ref data) = item.data {
         // We extract type of "identity" and enforce it is the expected type
@@ -237,28 +274,64 @@ fn derive_impl(
                 )
             })?;
 
-        return Ok(build(name, &table_name, &field.ty, flavor));
+        return Ok(build(name, &table_name, &field.ty, &uuid_ty, flavor));
     }
 
     Err(Error::new(item.span(), "Resource can only be derived for structs"))
 }
 
+/// Attributes specific to the `Resource` and `Asset` derive macros.
+#[derive(serde::Deserialize)]
+#[serde(deny_unknown_fields)]
+struct MacroAttributes {
+    /// The `TypedUuid` type parameter.
+    #[serde(default)]
+    uuid_kind: Option<ParseWrapper<syn::Ident>>,
+}
+
+impl MacroAttributes {
+    fn parse_from_attrs(
+        attrs: &[syn::Attribute],
+        flavor: IdentityVariant,
+    ) -> syn::Result<Self> {
+        let inner_attrs = attrs
+            .iter()
+            .filter_map(|attr| {
+                attr.path().is_ident(flavor.attr_name()).then(|| {
+                    let meta_list = attr.meta.require_list()?;
+                    Ok::<_, syn::Error>(&meta_list.tokens)
+                })
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+        let tokens = quote! { #(#inner_attrs,)* };
+        serde_tokenstream::from_tokenstream(&tokens)
+    }
+
+    fn uuid_ty(&self) -> PrimaryKeyType {
+        self.uuid_kind.as_ref().map_or_else(
+            || PrimaryKeyType::Standard(parse_quote!(::uuid::Uuid)),
+            |v| PrimaryKeyType::new_typed_uuid(v),
+        )
+    }
+}
+
 // Emits generated structures, depending on the requested flavor of identity.
 fn build(
     struct_name: &Ident,
     table_name: &syn::Path,
     observed_identity_ty: &syn::Type,
+    uuid_ty: &PrimaryKeyType,
     flavor: IdentityVariant,
 ) -> TokenStream {
     let (identity_struct, resource_impl) = {
         match flavor {
             IdentityVariant::Resource => (
-                build_resource_identity(struct_name, table_name),
-                build_resource_impl(struct_name, observed_identity_ty),
+                build_resource_identity(struct_name, table_name, uuid_ty),
+                build_resource_impl(struct_name, observed_identity_ty, uuid_ty),
             ),
             IdentityVariant::Asset => (
-                build_asset_identity(struct_name, table_name),
-                build_asset_impl(struct_name, observed_identity_ty),
+                build_asset_identity(struct_name, table_name, uuid_ty),
+                build_asset_impl(struct_name, observed_identity_ty, uuid_ty),
             ),
         }
     };
@@ -272,18 +345,25 @@ fn build(
 fn build_resource_identity(
     struct_name: &Ident,
     table_name: &syn::Path,
+    uuid_ty: &PrimaryKeyType,
 ) -> TokenStream {
     let identity_doc = format!(
         "Auto-generated identity for [`{}`] from deriving [`macro@Resource`].",
         struct_name,
     );
     let identity_name = format_ident!("{}Identity", struct_name);
+
+    let external_uuid_ty = uuid_ty.external();
+    let db_uuid_ty = uuid_ty.db();
+    let convert_external_to_db =
+        uuid_ty.external_to_db_nexus_db_model(quote! { id });
+
     quote! {
         #[doc = #identity_doc]
         #[derive(Clone, Debug, PartialEq, Eq, Selectable, Queryable, Insertable, serde::Serialize, serde::Deserialize)]
         #[diesel(table_name = #table_name) ]
         pub struct #identity_name {
-            pub id: ::uuid::Uuid,
+            pub id: #db_uuid_ty,
             pub name: crate::db::model::Name,
             pub description: ::std::string::String,
             pub time_created: ::chrono::DateTime<::chrono::Utc>,
@@ -293,12 +373,12 @@ fn build_resource_identity(
 
         impl #identity_name {
             pub fn new(
-                id: ::uuid::Uuid,
+                id: #external_uuid_ty,
                 params: ::omicron_common::api::external::IdentityMetadataCreateParams
             ) -> Self {
                 let now = ::chrono::Utc::now();
                 Self {
-                    id,
+                    id: #convert_external_to_db,
                     name: params.name.into(),
                     description: params.description,
                     time_created: now,
@@ -314,29 +394,36 @@ fn build_resource_identity(
 fn build_asset_identity(
     struct_name: &Ident,
     table_name: &syn::Path,
+    uuid_ty: &PrimaryKeyType,
 ) -> TokenStream {
     let identity_doc = format!(
         "Auto-generated identity for [`{}`] from deriving [`macro@Asset`].",
         struct_name,
     );
     let identity_name = format_ident!("{}Identity", struct_name);
+
+    let external_uuid_ty = uuid_ty.external();
+    let db_uuid_ty = uuid_ty.db();
+    let convert_external_to_db =
+        uuid_ty.external_to_db_nexus_db_model(quote! { id });
+
     quote! {
         #[doc = #identity_doc]
         #[derive(Clone, Debug, PartialEq, Selectable, Queryable, Insertable, serde::Serialize, serde::Deserialize)]
         #[diesel(table_name = #table_name) ]
         pub struct #identity_name {
-            pub id: ::uuid::Uuid,
+            pub id: #db_uuid_ty,
             pub time_created: ::chrono::DateTime<::chrono::Utc>,
             pub time_modified: ::chrono::DateTime<::chrono::Utc>,
         }
 
         impl #identity_name {
             pub fn new(
-                id: ::uuid::Uuid,
+                id: #external_uuid_ty,
             ) -> Self {
                 let now = ::chrono::Utc::now();
                 Self {
-                    id,
+                    id: #convert_external_to_db,
                     time_created: now,
                     time_modified: now,
                 }
@@ -349,9 +436,15 @@ fn build_asset_identity(
 fn build_resource_impl(
     struct_name: &Ident,
     observed_identity_type: &syn::Type,
+    uuid_ty: &PrimaryKeyType,
 ) -> TokenStream {
     let identity_trait = format_ident!("__{}IdentityMarker", struct_name);
     let identity_name = format_ident!("{}Identity", struct_name);
+
+    let external_uuid_ty = uuid_ty.external();
+    let convert_db_to_external =
+        uuid_ty.db_to_external(quote! { self.identity.id });
+
     quote! {
         // Verify that the field named "identity" is actually the generated
         // type within the struct deriving Resource.
@@ -365,8 +458,10 @@ fn build_resource_impl(
         };
 
         impl ::nexus_types::identity::Resource for #struct_name {
-            fn id(&self) -> ::uuid::Uuid {
-                self.identity.id
+            type IdType = #external_uuid_ty;
+
+            fn id(&self) -> #external_uuid_ty {
+                #convert_db_to_external
             }
 
             fn name(&self) -> &::omicron_common::api::external::Name {
@@ -396,9 +491,15 @@ fn build_resource_impl(
 fn build_asset_impl(
     struct_name: &Ident,
     observed_identity_type: &syn::Type,
+    uuid_ty: &PrimaryKeyType,
 ) -> TokenStream {
     let identity_trait = format_ident!("__{}IdentityMarker", struct_name);
     let identity_name = format_ident!("{}Identity", struct_name);
+
+    let external_uuid_ty = uuid_ty.external();
+    let convert_db_to_external =
+        uuid_ty.db_to_external(quote! { self.identity.id });
+
     quote! {
         // Verify that the field named "identity" is actually the generated
         // type within the struct deriving Asset.
@@ -412,8 +513,10 @@ fn build_asset_impl(
         };
 
         impl ::nexus_types::identity::Asset for #struct_name {
-            fn id(&self) -> ::uuid::Uuid {
-                self.identity.id
+            type IdType = #external_uuid_ty;
+
+            fn id(&self) -> #external_uuid_ty {
+                #convert_db_to_external
             }
 
             fn time_created(&self) -> ::chrono::DateTime<::chrono::Utc> {
@@ -431,6 +534,10 @@ fn build_asset_impl(
 mod tests {
     use super::*;
 
+    use crate::test_helpers::pretty_format;
+
+    use expectorate::assert_contents;
+
     #[test]
     fn test_derive_metadata_identity_fails_without_table_name() {
         let out = derive_impl(
@@ -532,4 +639,64 @@ mod tests {
         );
         assert!(out.is_ok());
     }
+
+    #[test]
+    fn test_derive_with_unknown_field() {
+        let out = derive_impl(
+            quote! {
+                #[derive(Resource)]
+                #[diesel(table_name = my_target)]
+                #[resource(foo = bar)]
+                struct MyTarget {
+                    identity: MyTargetIdentity,
+                    name: String,
+                    is_cool: bool,
+                }
+            },
+            IdentityVariant::Resource,
+        );
+        let error = out.expect_err("input has unknown parameter for resource");
+        assert!(error.to_string().contains("unknown field `foo`"));
+    }
+
+    #[test]
+    fn test_derive_snapshots() {
+        let out = derive_impl(
+            quote! {
+                #[derive(Asset)]
+                #[diesel(table_name = my_target)]
+                #[asset(uuid_kind = CustomKind)]
+                struct AssetWithUuidKind {
+                    identity: AssetWithUuidKindIdentity,
+                    name: String,
+                    is_cool: bool,
+                }
+            },
+            IdentityVariant::Asset,
+        )
+        .unwrap();
+        assert_contents(
+            "outputs/asset_with_uuid_kind.txt",
+            &pretty_format(out),
+        );
+
+        let out = derive_impl(
+            quote! {
+                #[derive(Resource)]
+                #[diesel(table_name = my_target)]
+                #[resource(uuid_kind = CustomKind)]
+                struct ResourceWithUuidKind {
+                    identity: ResourceWithUuidKindIdentity,
+                    name: String,
+                    is_cool: bool,
+                }
+            },
+            IdentityVariant::Resource,
+        )
+        .unwrap();
+        assert_contents(
+            "outputs/resource_with_uuid_kind.txt",
+            &pretty_format(out),
+        );
+    }
 }
diff --git a/nexus/db-macros/src/lookup.rs b/nexus/db-macros/src/lookup.rs
index c04c373ccb..3d3e93a863 100644
--- a/nexus/db-macros/src/lookup.rs
+++ b/nexus/db-macros/src/lookup.rs
@@ -6,10 +6,11 @@
 //!
 //! See nexus/src/db/lookup.rs.
 
+use nexus_macros_common::PrimaryKeyType;
 use proc_macro2::TokenStream;
 use quote::{format_ident, quote};
 use serde_tokenstream::ParseWrapper;
-use std::ops::Deref;
+use syn::spanned::Spanned;
 
 //
 // INPUT (arguments to the macro)
@@ -39,7 +40,7 @@ pub struct Input {
     /// whether lookup by name is supported (usually within the parent collection)
     lookup_by_name: bool,
     /// Description of the primary key columns
-    primary_key_columns: Vec<PrimaryKeyColumn>,
+    primary_key_columns: Vec<InputPrimaryKeyColumn>,
     /// This resources supports soft-deletes
     soft_deletes: bool,
     /// This resource appears under the `Silo` hierarchy, but nevertheless
@@ -52,10 +53,47 @@ pub struct Input {
     visible_outside_silo: bool,
 }
 
-#[derive(serde::Deserialize)]
 struct PrimaryKeyColumn {
     column_name: String,
-    rust_type: ParseWrapper<syn::Type>,
+    ty: PrimaryKeyType,
+}
+
+#[derive(serde::Deserialize)]
+struct InputPrimaryKeyColumn {
+    column_name: String,
+    // Exactly one of rust_type and uuid_kind must be specified.
+    #[serde(default)]
+    rust_type: Option<ParseWrapper<syn::Type>>,
+    #[serde(default)]
+    uuid_kind: Option<ParseWrapper<syn::Ident>>,
+}
+
+impl InputPrimaryKeyColumn {
+    fn validate(self) -> syn::Result<PrimaryKeyColumn> {
+        let ty = match (self.rust_type, self.uuid_kind) {
+            (Some(rust_type), Some(_)) => {
+                return Err(syn::Error::new(
+                    rust_type.span(),
+                    "only one of rust_type and uuid_kind may be specified",
+                ));
+            }
+            (Some(rust_type), None) => {
+                PrimaryKeyType::Standard(rust_type.into_inner())
+            }
+            (None, Some(uuid_kind)) => {
+                PrimaryKeyType::new_typed_uuid(&uuid_kind)
+            }
+            (None, None) => {
+                return Err(syn::Error::new(
+                    proc_macro2::Span::call_site(),
+                    "for primary_key_columns, \
+                    one of rust_type and uuid_kind must be specified",
+                ));
+            }
+        };
+
+        Ok(PrimaryKeyColumn { column_name: self.column_name, ty })
+    }
 }
 
 //
@@ -106,7 +144,7 @@ pub struct Config {
 }
 
 impl Config {
-    fn for_input(input: Input) -> Config {
+    fn for_input(input: Input) -> syn::Result<Config> {
         let resource = Resource::for_name(&input.name);
 
         let mut path_types: Vec<_> =
@@ -127,7 +165,13 @@ impl Config {
         let silo_restricted = !input.visible_outside_silo
             && input.ancestors.iter().any(|s| s == "Silo");
 
-        Config {
+        let primary_key_columns: Vec<_> = input
+            .primary_key_columns
+            .into_iter()
+            .map(|c| c.validate())
+            .collect::<syn::Result<_>>()?;
+
+        Ok(Config {
             resource,
             silo_restricted,
             path_types,
@@ -135,9 +179,9 @@ impl Config {
             parent,
             child_resources,
             lookup_by_name: input.lookup_by_name,
-            primary_key_columns: input.primary_key_columns,
+            primary_key_columns,
             soft_deletes: input.soft_deletes,
-        }
+        })
     }
 }
 
@@ -172,7 +216,7 @@ pub fn lookup_resource(
     raw_input: TokenStream,
 ) -> Result<TokenStream, syn::Error> {
     let input = serde_tokenstream::from_tokenstream::<Input>(&raw_input)?;
-    let config = Config::for_input(input);
+    let config = Config::for_input(input)?;
 
     let resource_name = &config.resource.name;
     let the_basics = generate_struct(&config);
@@ -204,8 +248,7 @@ fn generate_struct(config: &Config) -> TokenStream {
         functions on this struct) for lookup or fetch",
         resource_name
     );
-    let pkey_types =
-        config.primary_key_columns.iter().map(|c| c.rust_type.deref());
+    let pkey_types = config.primary_key_columns.iter().map(|c| c.ty.external());
 
     /* configure the lookup enum */
     let name_variant = if config.lookup_by_name {
@@ -706,11 +749,8 @@ fn generate_database_functions(config: &Config) -> TokenStream {
     let resource_as_snake = format_ident!("{}", &config.resource.name_as_snake);
     let path_types = &config.path_types;
     let path_authz_names = &config.path_authz_names;
-    let pkey_types: Vec<_> = config
-        .primary_key_columns
-        .iter()
-        .map(|c| c.rust_type.deref())
-        .collect();
+    let pkey_types: Vec<_> =
+        config.primary_key_columns.iter().map(|c| c.ty.external()).collect();
     let pkey_column_names = config
         .primary_key_columns
         .iter()
@@ -721,6 +761,18 @@ fn generate_database_functions(config: &Config) -> TokenStream {
         .enumerate()
         .map(|(i, _)| format_ident!("v{}", i))
         .collect();
+
+    // Generate tokens that also perform conversion from external to db types,
+    // if necessary.
+    let pkey_names_convert: Vec<_> = config
+        .primary_key_columns
+        .iter()
+        .zip(pkey_names.iter())
+        .map(|(col, name)| {
+            col.ty.external_to_db_other(quote! { #name.clone() })
+        })
+        .collect();
+
     let (
         parent_lookup_arg_formal,
         parent_lookup_arg_actual,
@@ -837,7 +889,11 @@ fn generate_database_functions(config: &Config) -> TokenStream {
     let lookup_type = if config.primary_key_columns.len() == 1
         && config.primary_key_columns[0].column_name == "id"
     {
-        quote! { LookupType::ById(#(#pkey_names.clone())*) }
+        let pkey_name = &pkey_names[0];
+        let by_id = quote! {
+            ::omicron_uuid_kinds::GenericUuid::into_untyped_uuid(*#pkey_name)
+        };
+        quote! { LookupType::ById(#by_id) }
     } else {
         let fmtstr = config
             .primary_key_columns
@@ -889,7 +945,7 @@ fn generate_database_functions(config: &Config) -> TokenStream {
 
             let db_row = dsl::#resource_as_snake
                 #soft_delete_filter
-                #(.filter(dsl::#pkey_column_names.eq(#pkey_names.clone())))*
+                #(.filter(dsl::#pkey_column_names.eq(#pkey_names_convert)))*
                 .select(nexus_db_model::#resource_name::as_select())
                 .get_result_async(&*datastore.pool_connection_authorized(opctx).await?)
                 .await
@@ -915,9 +971,10 @@ fn generate_database_functions(config: &Config) -> TokenStream {
 
 #[cfg(test)]
 mod test {
+    use crate::test_helpers::pretty_format;
+
     use super::lookup_resource;
     use expectorate::assert_contents;
-    use proc_macro2::TokenStream;
     use quote::quote;
 
     /// Ensure that generated code is as expected.
@@ -953,6 +1010,17 @@ mod test {
         .unwrap();
         assert_contents("outputs/silo_user.txt", &pretty_format(output));
 
+        let output = lookup_resource(quote! {
+            name = "Sled",
+            ancestors = [],
+            children = [],
+            lookup_by_name = false,
+            soft_deletes = true,
+            primary_key_columns = [ { column_name = "id", uuid_kind = SledKind } ]
+        })
+        .unwrap();
+        assert_contents("outputs/sled.txt", &pretty_format(output));
+
         let output = lookup_resource(quote! {
             name = "UpdateArtifact",
             ancestors = [],
@@ -968,9 +1036,4 @@ mod test {
         .unwrap();
         assert_contents("outputs/update_artifact.txt", &pretty_format(output));
     }
-
-    fn pretty_format(input: TokenStream) -> String {
-        let parsed = syn::parse2(input).unwrap();
-        prettyplease::unparse(&parsed)
-    }
 }
diff --git a/nexus/db-macros/src/test_helpers.rs b/nexus/db-macros/src/test_helpers.rs
new file mode 100644
index 0000000000..c069cb5c02
--- /dev/null
+++ b/nexus/db-macros/src/test_helpers.rs
@@ -0,0 +1,10 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use proc_macro2::TokenStream;
+
+pub(crate) fn pretty_format(input: TokenStream) -> String {
+    let parsed = syn::parse2(input).unwrap();
+    prettyplease::unparse(&parsed)
+}
diff --git a/nexus/db-model/Cargo.toml b/nexus/db-model/Cargo.toml
index 477ce7d11f..43b6f6a4b0 100644
--- a/nexus/db-model/Cargo.toml
+++ b/nexus/db-model/Cargo.toml
@@ -10,11 +10,13 @@ omicron-rpaths.workspace = true
 [dependencies]
 anyhow.workspace = true
 chrono.workspace = true
+derive-where.workspace = true
 diesel = { workspace = true, features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] }
 hex.workspace = true
 ipnetwork.workspace = true
 macaddr.workspace = true
 newtype_derive.workspace = true
+omicron-uuid-kinds.workspace = true
 parse-display.workspace = true
 # See omicron-rpaths for more about the "pq-sys" dependency.
 pq-sys = "*"
diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs
index b77d56059e..ecbb8365fe 100644
--- a/nexus/db-model/src/lib.rs
+++ b/nexus/db-model/src/lib.rs
@@ -81,6 +81,7 @@ mod snapshot;
 mod ssh_key;
 mod switch;
 mod tuf_repo;
+mod typed_uuid;
 mod unsigned;
 mod user_builtin;
 mod utilization;
@@ -170,6 +171,7 @@ pub use switch::*;
 pub use switch_interface::*;
 pub use switch_port::*;
 pub use tuf_repo::*;
+pub use typed_uuid::to_db_typed_uuid;
 pub use user_builtin::*;
 pub use utilization::*;
 pub use virtual_provisioning_collection::*;
diff --git a/nexus/db-model/src/switch_interface.rs b/nexus/db-model/src/switch_interface.rs
index 71673354ea..78f502ed4c 100644
--- a/nexus/db-model/src/switch_interface.rs
+++ b/nexus/db-model/src/switch_interface.rs
@@ -9,6 +9,8 @@ use ipnetwork::IpNetwork;
 use nexus_types::external_api::params;
 use nexus_types::identity::Asset;
 use omicron_common::api::external;
+use omicron_uuid_kinds::LoopbackAddressKind;
+use omicron_uuid_kinds::TypedUuid;
 use serde::{Deserialize, Serialize};
 use uuid::Uuid;
 
@@ -105,6 +107,7 @@ impl Into<external::SwitchVlanInterfaceConfig> for SwitchVlanInterfaceConfig {
     Deserialize,
 )]
 #[diesel(table_name = loopback_address)]
+#[asset(uuid_kind = LoopbackAddressKind)]
 pub struct LoopbackAddress {
     #[diesel(embed)]
     pub identity: LoopbackAddressIdentity,
@@ -118,7 +121,7 @@ pub struct LoopbackAddress {
 
 impl LoopbackAddress {
     pub fn new(
-        id: Option<Uuid>,
+        id: Option<TypedUuid<LoopbackAddressKind>>,
         address_lot_block_id: Uuid,
         rsvd_address_lot_block_id: Uuid,
         rack_id: Uuid,
@@ -128,7 +131,7 @@ impl LoopbackAddress {
     ) -> Self {
         Self {
             identity: LoopbackAddressIdentity::new(
-                id.unwrap_or(Uuid::new_v4()),
+                id.unwrap_or(TypedUuid::new_v4()),
             ),
             address_lot_block_id,
             rsvd_address_lot_block_id,
diff --git a/nexus/db-model/src/tuf_repo.rs b/nexus/db-model/src/tuf_repo.rs
index 5fa2a0aac7..4a64566a62 100644
--- a/nexus/db-model/src/tuf_repo.rs
+++ b/nexus/db-model/src/tuf_repo.rs
@@ -6,6 +6,7 @@ use std::str::FromStr;
 
 use crate::{
     schema::{tuf_artifact, tuf_repo, tuf_repo_artifact},
+    typed_uuid::DbTypedUuid,
     SemverVersion,
 };
 use chrono::{DateTime, Utc};
@@ -17,6 +18,8 @@ use omicron_common::{
         ArtifactKind,
     },
 };
+use omicron_uuid_kinds::TufRepoKind;
+use omicron_uuid_kinds::TypedUuid;
 use serde::{Deserialize, Serialize};
 use std::fmt;
 use uuid::Uuid;
@@ -73,7 +76,7 @@ impl TufRepoDescription {
 )]
 #[diesel(table_name = tuf_repo)]
 pub struct TufRepo {
-    pub id: Uuid,
+    pub id: DbTypedUuid<TufRepoKind>,
     pub time_created: DateTime<Utc>,
     // XXX: We're overloading ArtifactHash here to also mean the hash of the
     // repository zip itself.
@@ -94,7 +97,7 @@ impl TufRepo {
         file_name: String,
     ) -> Self {
         Self {
-            id: Uuid::new_v4(),
+            id: TypedUuid::new_v4().into(),
             time_created: Utc::now(),
             sha256,
             targets_role_version: targets_role_version as i64,
@@ -132,8 +135,8 @@ impl TufRepo {
     }
 
     /// Returns the repository's ID.
-    pub fn id(&self) -> Uuid {
-        self.id
+    pub fn id(&self) -> TypedUuid<TufRepoKind> {
+        self.id.into()
     }
 
     /// Returns the targets role version.
diff --git a/nexus/db-model/src/typed_uuid.rs b/nexus/db-model/src/typed_uuid.rs
new file mode 100644
index 0000000000..7785b8c7dc
--- /dev/null
+++ b/nexus/db-model/src/typed_uuid.rs
@@ -0,0 +1,116 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Typed UUID instances.
+
+use derive_where::derive_where;
+use diesel::backend::Backend;
+use diesel::deserialize::{self, FromSql};
+use diesel::serialize::{self, ToSql};
+use diesel::sql_types;
+use omicron_uuid_kinds::{GenericUuid, TypedUuid, TypedUuidKind};
+use serde::{Deserialize, Serialize};
+use std::fmt;
+use std::str::FromStr;
+use uuid::Uuid;
+
+/// Returns the corresponding `DbTypedUuid` for this `TypedUuid`.
+///
+/// Code external to the `db-model` crate sometimes needs a way to convert a
+/// `TypedUuid` to a `DbTypedUuid`. We don't want `DbTypedUuid` to be used
+/// anywhere, so we don't make it public. Instead, we expose this function.
+#[inline]
+pub fn to_db_typed_uuid<T: TypedUuidKind>(id: TypedUuid<T>) -> DbTypedUuid<T> {
+    DbTypedUuid(id)
+}
+
+/// A UUID with information about the kind of type it is.
+///
+/// Despite the fact that this is marked `pub`, this is *private* to the
+/// `db-model` crate (this type is not exported at the top level). External
+/// users must use omicron-common's `TypedUuid`.
+#[derive_where(Clone, Copy, Eq, Ord, PartialEq, PartialOrd, Hash)]
+#[derive(AsExpression, FromSqlRow, Serialize, Deserialize)]
+#[diesel(sql_type = sql_types::Uuid)]
+#[serde(transparent, bound = "")]
+pub struct DbTypedUuid<T: TypedUuidKind>(pub(crate) TypedUuid<T>);
+
+impl<T: TypedUuidKind, DB> ToSql<sql_types::Uuid, DB> for DbTypedUuid<T>
+where
+    DB: Backend,
+    Uuid: ToSql<sql_types::Uuid, DB>,
+{
+    fn to_sql<'a>(
+        &'a self,
+        out: &mut serialize::Output<'a, '_, DB>,
+    ) -> serialize::Result {
+        self.0.as_untyped_uuid().to_sql(out)
+    }
+}
+
+impl<T: TypedUuidKind, DB> FromSql<sql_types::Uuid, DB> for DbTypedUuid<T>
+where
+    DB: Backend,
+    Uuid: FromSql<sql_types::Uuid, DB>,
+{
+    #[inline]
+    fn from_sql(bytes: DB::RawValue<'_>) -> deserialize::Result<Self> {
+        let id = Uuid::from_sql(bytes)?;
+        Ok(TypedUuid::from_untyped_uuid(id).into())
+    }
+}
+
+impl<T: TypedUuidKind> fmt::Debug for DbTypedUuid<T> {
+    #[inline]
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl<T: TypedUuidKind> fmt::Display for DbTypedUuid<T> {
+    #[inline]
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl<T: TypedUuidKind> FromStr for DbTypedUuid<T> {
+    type Err = omicron_uuid_kinds::ParseError;
+
+    #[inline]
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(TypedUuid::from_str(s)?.into())
+    }
+}
+
+impl<T: TypedUuidKind> From<TypedUuid<T>> for DbTypedUuid<T> {
+    #[inline]
+    fn from(id: TypedUuid<T>) -> Self {
+        Self(id)
+    }
+}
+
+impl<T: TypedUuidKind> From<DbTypedUuid<T>> for TypedUuid<T> {
+    #[inline]
+    fn from(id: DbTypedUuid<T>) -> Self {
+        id.0
+    }
+}
+
+impl<T: TypedUuidKind> GenericUuid for DbTypedUuid<T> {
+    #[inline]
+    fn from_untyped_uuid(uuid: Uuid) -> Self {
+        TypedUuid::from_untyped_uuid(uuid).into()
+    }
+
+    #[inline]
+    fn into_untyped_uuid(self) -> Uuid {
+        self.0.into_untyped_uuid()
+    }
+
+    #[inline]
+    fn as_untyped_uuid(&self) -> &Uuid {
+        self.0.as_untyped_uuid()
+    }
+}
diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml
index 9cdcc88e6a..9c9a30799e 100644
--- a/nexus/db-queries/Cargo.toml
+++ b/nexus/db-queries/Cargo.toml
@@ -45,7 +45,7 @@ static_assertions.workspace = true
 steno.workspace = true
 swrite.workspace = true
 thiserror.workspace = true
-tokio = { workspace = true, features = [ "full" ] }
+tokio = { workspace = true, features = ["full"] }
 uuid.workspace = true
 usdt.workspace = true
 
@@ -55,6 +55,7 @@ nexus-db-model.workspace = true
 nexus-types.workspace = true
 omicron-common.workspace = true
 omicron-passwords.workspace = true
+omicron-uuid-kinds.workspace = true
 oximeter.workspace = true
 omicron-workspace-hack.workspace = true
 
diff --git a/nexus/db-queries/src/authz/api_resources.rs b/nexus/db-queries/src/authz/api_resources.rs
index b4fd4e1890..70bc9ab2eb 100644
--- a/nexus/db-queries/src/authz/api_resources.rs
+++ b/nexus/db-queries/src/authz/api_resources.rs
@@ -869,7 +869,7 @@ authz_resource! {
 authz_resource! {
     name = "LoopbackAddress",
     parent = "Fleet",
-    primary_key = Uuid,
+    primary_key = { uuid_kind = LoopbackAddressKind },
     roles_allowed = false,
     polar_snippet = FleetChild,
 }
@@ -1068,7 +1068,7 @@ authz_resource! {
 authz_resource! {
     name = "TufRepo",
     parent = "Fleet",
-    primary_key = Uuid,
+    primary_key = { uuid_kind = TufRepoKind },
     roles_allowed = false,
     polar_snippet = FleetChild,
 }
diff --git a/nexus/db-queries/src/authz/policy_test/resources.rs b/nexus/db-queries/src/authz/policy_test/resources.rs
index 3e87f6db51..96cefb3db4 100644
--- a/nexus/db-queries/src/authz/policy_test/resources.rs
+++ b/nexus/db-queries/src/authz/policy_test/resources.rs
@@ -10,6 +10,7 @@ use crate::authz;
 use crate::db::model::ArtifactId;
 use nexus_db_model::SemverVersion;
 use omicron_common::api::external::LookupType;
+use omicron_uuid_kinds::GenericUuid;
 use oso::PolarClass;
 use std::collections::BTreeSet;
 use uuid::Uuid;
@@ -132,7 +133,7 @@ pub async fn make_resources(
     builder.new_resource(authz::TufRepo::new(
         authz::FLEET,
         tuf_repo_id,
-        LookupType::ById(tuf_repo_id),
+        LookupType::ById(tuf_repo_id.into_untyped_uuid()),
     ));
 
     let artifact_id = ArtifactId {
@@ -160,7 +161,7 @@ pub async fn make_resources(
     builder.new_resource(authz::LoopbackAddress::new(
         authz::FLEET,
         loopback_address_id,
-        LookupType::ById(loopback_address_id),
+        LookupType::ById(loopback_address_id.into_untyped_uuid()),
     ));
 
     builder.build()
diff --git a/nexus/db-queries/src/db/datastore/switch_interface.rs b/nexus/db-queries/src/db/datastore/switch_interface.rs
index 67f16fa08f..aa4f6747fd 100644
--- a/nexus/db-queries/src/db/datastore/switch_interface.rs
+++ b/nexus/db-queries/src/db/datastore/switch_interface.rs
@@ -17,11 +17,14 @@ use crate::transaction_retry::OptionalError;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use diesel::{ExpressionMethods, QueryDsl, SelectableHelper};
 use ipnetwork::IpNetwork;
+use nexus_db_model::to_db_typed_uuid;
 use nexus_types::external_api::params::LoopbackAddressCreate;
 use omicron_common::api::external::{
     CreateResult, DataPageParams, DeleteResult, Error, ListResultVec,
     LookupResult, ResourceType,
 };
+use omicron_uuid_kinds::LoopbackAddressKind;
+use omicron_uuid_kinds::TypedUuid;
 use uuid::Uuid;
 
 impl DataStore {
@@ -29,7 +32,7 @@ impl DataStore {
         &self,
         opctx: &OpContext,
         params: &LoopbackAddressCreate,
-        id: Option<Uuid>,
+        id: Option<TypedUuid<LoopbackAddressKind>>,
         authz_address_lot: &authz::AddressLot,
     ) -> CreateResult<LoopbackAddress> {
         use db::schema::loopback_address::dsl;
@@ -130,7 +133,7 @@ impl DataStore {
         self.transaction_retry_wrapper("loopback_address_delete")
             .transaction(&conn, |conn| async move {
                 let la = diesel::delete(dsl::loopback_address)
-                    .filter(dsl::id.eq(id))
+                    .filter(dsl::id.eq(to_db_typed_uuid(id)))
                     .returning(LoopbackAddress::as_returning())
                     .get_result_async(&conn)
                     .await?;
@@ -159,7 +162,7 @@ impl DataStore {
         let conn = self.pool_connection_authorized(opctx).await?;
 
         loopback_dsl::loopback_address
-            .filter(loopback_address::id.eq(id))
+            .filter(loopback_address::id.eq(to_db_typed_uuid(id)))
             .select(LoopbackAddress::as_select())
             .limit(1)
             .first_async::<LoopbackAddress>(&*conn)
diff --git a/nexus/db-queries/src/db/datastore/update.rs b/nexus/db-queries/src/db/datastore/update.rs
index 3725797f83..d73a3d903f 100644
--- a/nexus/db-queries/src/db/datastore/update.rs
+++ b/nexus/db-queries/src/db/datastore/update.rs
@@ -21,8 +21,9 @@ use omicron_common::api::external::{
     self, CreateResult, LookupResult, LookupType, ResourceType,
     TufRepoInsertStatus,
 };
+use omicron_uuid_kinds::TufRepoKind;
+use omicron_uuid_kinds::TypedUuid;
 use swrite::{swrite, SWrite};
-use uuid::Uuid;
 
 /// The return value of [`DataStore::update_tuf_repo_description_insert`].
 ///
@@ -43,7 +44,7 @@ impl TufRepoInsertResponse {
 }
 
 async fn artifacts_for_repo(
-    repo_id: Uuid,
+    repo_id: TypedUuid<TufRepoKind>,
     conn: &async_bb8_diesel::Connection<crate::db::DbConnection>,
 ) -> Result<Vec<TufArtifact>, DieselError> {
     use db::schema::tuf_artifact::dsl as tuf_artifact_dsl;
@@ -61,7 +62,10 @@ async fn artifacts_for_repo(
     // Don't bother paginating because each repo should only have a few (under
     // 20) artifacts.
     tuf_repo_artifact_dsl::tuf_repo_artifact
-        .filter(tuf_repo_artifact_dsl::tuf_repo_id.eq(repo_id))
+        .filter(
+            tuf_repo_artifact_dsl::tuf_repo_id
+                .eq(nexus_db_model::to_db_typed_uuid(repo_id)),
+        )
         .inner_join(tuf_artifact_dsl::tuf_artifact.on(join_on_dsl))
         .select(TufArtifact::as_select())
         .load_async(conn)
@@ -138,7 +142,7 @@ impl DataStore {
                 )
             })?;
 
-        let artifacts = artifacts_for_repo(repo.id, &conn)
+        let artifacts = artifacts_for_repo(repo.id.into(), &conn)
             .await
             .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
         Ok(TufRepoDescription { repo, artifacts })
@@ -177,7 +181,8 @@ async fn insert_impl(
             }
 
             // Just return the existing repo along with all of its artifacts.
-            let artifacts = artifacts_for_repo(existing_repo.id, &conn).await?;
+            let artifacts =
+                artifacts_for_repo(existing_repo.id.into(), &conn).await?;
 
             let recorded =
                 TufRepoDescription { repo: existing_repo, artifacts };
diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs
index 1cf14c5a8f..18ea369685 100644
--- a/nexus/db-queries/src/db/lookup.rs
+++ b/nexus/db-queries/src/db/lookup.rs
@@ -21,6 +21,8 @@ use nexus_db_model::Name;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::InternalContext;
 use omicron_common::api::external::{LookupResult, LookupType, ResourceType};
+use omicron_uuid_kinds::TufRepoKind;
+use omicron_uuid_kinds::TypedUuid;
 use uuid::Uuid;
 
 /// Look up an API resource in the database
@@ -431,7 +433,7 @@ impl<'a> LookupPath<'a> {
     }
 
     /// Select a resource of type TufRepo, identified by its UUID.
-    pub fn tuf_repo_id(self, id: Uuid) -> TufRepo<'a> {
+    pub fn tuf_repo_id(self, id: TypedUuid<TufRepoKind>) -> TufRepo<'a> {
         TufRepo::PrimaryKey(Root { lookup_root: self }, id)
     }
 
@@ -863,7 +865,7 @@ lookup_resource! {
     children = [],
     lookup_by_name = false,
     soft_deletes = false,
-    primary_key_columns = [ { column_name = "id", rust_type = Uuid } ]
+    primary_key_columns = [ { column_name = "id", uuid_kind = TufRepoKind } ]
 }
 
 lookup_resource! {
diff --git a/nexus/macros-common/Cargo.toml b/nexus/macros-common/Cargo.toml
new file mode 100644
index 0000000000..9d4390a6d2
--- /dev/null
+++ b/nexus/macros-common/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "nexus-macros-common"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+
+[dependencies]
+proc-macro2.workspace = true
+syn = { workspace = true, features = ["extra-traits"] }
+quote.workspace = true
+omicron-workspace-hack.workspace = true
diff --git a/nexus/macros-common/src/lib.rs b/nexus/macros-common/src/lib.rs
new file mode 100644
index 0000000000..a2f51e1b5c
--- /dev/null
+++ b/nexus/macros-common/src/lib.rs
@@ -0,0 +1,109 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Code shared by Nexus macros.
+
+use proc_macro2::TokenStream;
+use quote::quote;
+use syn::parse_quote;
+
+/// The type of a primary key in a database.
+#[derive(Debug)]
+pub enum PrimaryKeyType {
+    /// A regular type.
+    Standard(syn::Type),
+
+    /// A typed UUID, which requires special handling.
+    TypedUuid {
+        /// The external type. This is used almost everywhere.
+        external: syn::Type,
+
+        /// The internal type used in nexus-db-model.
+        db: syn::Type,
+    },
+}
+
+impl PrimaryKeyType {
+    /// Constructs a new `TypedUuid` variant.
+    pub fn new_typed_uuid(kind: &syn::Ident) -> Self {
+        let external = parse_quote!(::omicron_uuid_kinds::TypedUuid<::omicron_uuid_kinds::#kind>);
+        let db = parse_quote!(crate::typed_uuid::DbTypedUuid<::omicron_uuid_kinds::#kind>);
+        PrimaryKeyType::TypedUuid { external, db }
+    }
+
+    /// Returns the external type for this primary key.
+    pub fn external(&self) -> &syn::Type {
+        match self {
+            PrimaryKeyType::Standard(path) => path,
+            PrimaryKeyType::TypedUuid { external, .. } => external,
+        }
+    }
+
+    /// Converts self into the external type.
+    pub fn into_external(self) -> syn::Type {
+        match self {
+            PrimaryKeyType::Standard(path) => path,
+            PrimaryKeyType::TypedUuid { external, .. } => external,
+        }
+    }
+
+    /// Returns the database type for this primary key.
+    ///
+    /// For the `TypedUuid` variant, the db type is only accessible within the
+    /// `nexus-db-model` crate.
+    pub fn db(&self) -> &syn::Type {
+        match self {
+            PrimaryKeyType::Standard(path) => path,
+            PrimaryKeyType::TypedUuid { db, .. } => db,
+        }
+    }
+
+    /// Converts self into the database type.
+    pub fn into_db(self) -> syn::Type {
+        match self {
+            PrimaryKeyType::Standard(path) => path,
+            PrimaryKeyType::TypedUuid { db, .. } => db,
+        }
+    }
+
+    /// Returns tokens for a conversion from external to db types, given an
+    /// expression as input.
+    ///
+    /// This is specialized for the nexus-db-model crate.
+    pub fn external_to_db_nexus_db_model(
+        &self,
+        tokens: TokenStream,
+    ) -> TokenStream {
+        match self {
+            PrimaryKeyType::Standard(_) => tokens,
+            PrimaryKeyType::TypedUuid { .. } => {
+                quote! { crate::to_db_typed_uuid(#tokens) }
+            }
+        }
+    }
+
+    /// Returns tokens for a conversion from external to db types, given an
+    /// expression as input.
+    ///
+    /// This is used for all crates *except* nexus-db-model.
+    pub fn external_to_db_other(&self, tokens: TokenStream) -> TokenStream {
+        match self {
+            PrimaryKeyType::Standard(_) => tokens,
+            PrimaryKeyType::TypedUuid { .. } => {
+                quote! { ::nexus_db_model::to_db_typed_uuid(#tokens) }
+            }
+        }
+    }
+
+    /// Returns tokens for a conversion from db to external types, given an
+    /// expression as input.
+    pub fn db_to_external(&self, tokens: TokenStream) -> TokenStream {
+        match self {
+            PrimaryKeyType::Standard(_) => tokens,
+            PrimaryKeyType::TypedUuid { .. } => {
+                quote! { ::omicron_uuid_kinds::TypedUuid::from(#tokens) }
+            }
+        }
+    }
+}
diff --git a/nexus/types/Cargo.toml b/nexus/types/Cargo.toml
index dff0f73be7..93716d4804 100644
--- a/nexus/types/Cargo.toml
+++ b/nexus/types/Cargo.toml
@@ -9,6 +9,7 @@ anyhow.workspace = true
 chrono.workspace = true
 base64.workspace = true
 futures.workspace = true
+omicron-uuid-kinds.workspace = true
 openssl.workspace = true
 parse-display.workspace = true
 schemars = { workspace = true, features = ["chrono", "uuid1"] }
diff --git a/nexus/types/src/identity.rs b/nexus/types/src/identity.rs
index ededb926df..6f214d6f9a 100644
--- a/nexus/types/src/identity.rs
+++ b/nexus/types/src/identity.rs
@@ -9,6 +9,7 @@
 use chrono::{DateTime, Utc};
 use omicron_common::api::external::IdentityMetadata;
 use omicron_common::api::external::Name;
+use omicron_uuid_kinds::GenericUuid;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use uuid::Uuid;
@@ -23,7 +24,9 @@ use uuid::Uuid;
 ///
 /// May be derived from [`macro@db-macros::Resource`].
 pub trait Resource {
-    fn id(&self) -> Uuid;
+    type IdType: GenericUuid;
+
+    fn id(&self) -> Self::IdType;
     fn name(&self) -> &Name;
     fn description(&self) -> &str;
     fn time_created(&self) -> DateTime<Utc>;
@@ -32,7 +35,7 @@ pub trait Resource {
 
     fn identity(&self) -> IdentityMetadata {
         IdentityMetadata {
-            id: self.id(),
+            id: self.id().into_untyped_uuid(),
             name: self.name().clone(),
             description: self.description().to_string(),
             time_created: self.time_created(),
@@ -60,13 +63,15 @@ pub struct AssetIdentityMetadata {
 ///
 /// May be derived from [`macro@db-macros::Asset`].
 pub trait Asset {
-    fn id(&self) -> Uuid;
+    type IdType: GenericUuid;
+
+    fn id(&self) -> Self::IdType;
     fn time_created(&self) -> DateTime<Utc>;
     fn time_modified(&self) -> DateTime<Utc>;
 
     fn identity(&self) -> AssetIdentityMetadata {
         AssetIdentityMetadata {
-            id: self.id(),
+            id: self.id().into_untyped_uuid(),
             time_created: self.time_created(),
             time_modified: self.time_modified(),
         }
diff --git a/uuid-kinds/Cargo.toml b/uuid-kinds/Cargo.toml
new file mode 100644
index 0000000000..126ba8bcf8
--- /dev/null
+++ b/uuid-kinds/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "omicron-uuid-kinds"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+
+# The dependencies and features are written so as to make it easy for no-std
+# code to import omicron-uuid-kinds. All the features below are turned on
+# within omicron.
+
+[dependencies]
+newtype-uuid.workspace = true
+schemars = { workspace = true, optional = true }
+
+[features]
+default = ["std"]
+serde = ["newtype-uuid/serde"]
+schemars08 = ["newtype-uuid/schemars08", "schemars"]
+std = ["newtype-uuid/std"]
+uuid-v4 = ["newtype-uuid/v4"]
diff --git a/uuid-kinds/README.adoc b/uuid-kinds/README.adoc
new file mode 100644
index 0000000000..1a22477aad
--- /dev/null
+++ b/uuid-kinds/README.adoc
@@ -0,0 +1,73 @@
+# omicron-uuid-kinds
+
+In omicron and other Oxide projects, we use UUIDs to identify many different
+kinds of entities. Using a single `Uuid` type for all of them risks mixing up
+all the different kinds of IDs.
+
+To address that, we're actively moving towards typed UUIDs with the
+https://github.com/oxidecomputer/newtype-uuid[newtype-uuid] crate. The goal is
+that each kind of entity will have a marker type representing a _UUID kind_
+associated with it. Then, the type system will make it significantly harder to
+mix different kinds of entities.
+
+*`omicron-uuid-kinds` is a centralized registry for UUID kinds*, one that can
+be shared across Oxide repos. `omicron-uuid-kinds` supports no-std so the kinds
+can be shared with embedded code as well.
+
+## Determinations
+
+As part of this effort, we've made several decisions that could have gone a
+different way. This section documents those choices.
+
+### One big registry or several smaller ones?
+
+Rather than having a single registry, an option is to have several smaller ones
+(perhaps one to two in each repo). The best answer to that is currently
+unclear, but putting them in the same crate for now has several advantages:
+
+* The macros updated in this PR know where uuid kinds will be found, so they
+let you specify just a bare name as the kind.
+
+* It's simpler.
+
+The disadvantage of this is that any change to this registry will cause all of
+omicron to be rebuilt. Hopefully, once most UUIDs are converted over, this
+crate won't be touched too much.
+
+This decision involves some level of commitment, because splitting up a
+registry will probably require several days of work. For example, macros may
+need to be updated to handle multiple sources, import paths would have to be
+changed across several repos, and so on.
+
+Crates which solve more abstract problems and are completely independent of
+omicron, like https://github.com/oxidecomputer/dropshot/[Dropshot] and
+https://github.com/oxidecomputer/steno[Steno], will have their own registries
+(probably as a module within the crates themselves).
+
+### Where should the registry live?
+
+Once we've decided on a single registry, the next question is where this
+registry should be stored. There was some debate in `#oxide-control-plane`, and
+two options stood out:
+
+1. In omicron itself.
+
+    * The upside of this is that it is easy to add new UUID kinds within
+      omicron, especially as we transition over existing untyped UUIDs.
+
+    * The downside is that we also want to use these kinds in crucible and
+      propolis -- and pulling in `omicron-uuid-kinds` as a Git dependency would
+      typically cause two copies of `omicron-uuid-kinds` to be floating around.
+      That can result in a lot of pain, including confusing errors.
+
+2. In a separate repo. While it eliminates the issue with duplicated
+dependencies, it does add a fair bit more friction. That's because users
+updating omicron would now have to make two separate, coordinated PRs.
+
+We've chosen 1 for now, because agility within omicron outweighs other
+concerns. *The downside is mitigated* by using a `[patch]` directive; see the
+workspace root's `Cargo.toml` for more about this.
+
+This is straightforward to change in the future. If we find that 1 is too much
+of a bother, we'll easily be able to switch to 2 while keeping the name of the
+crate the same (minimizing churn).
diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs
new file mode 100644
index 0000000000..12bc756d68
--- /dev/null
+++ b/uuid-kinds/src/lib.rs
@@ -0,0 +1,50 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+//! A registry for UUID kinds used in Omicron and related projects.
+//!
+//! See this crate's `README.md` for more information.
+
+// Export these types so that other users don't have to pull in newtype-uuid.
+#[doc(no_inline)]
+pub use newtype_uuid::{
+    GenericUuid, ParseError, TagError, TypedUuid, TypedUuidKind, TypedUuidTag,
+};
+
+#[cfg(feature = "schemars08")]
+use schemars::JsonSchema;
+
+macro_rules! impl_typed_uuid_kind {
+    ($($kind:ident => $tag:literal),* $(,)?) => {
+        $(
+            #[cfg_attr(feature = "schemars08", derive(JsonSchema))]
+            pub enum $kind {}
+
+            impl TypedUuidKind for $kind {
+                #[inline]
+                fn tag() -> TypedUuidTag {
+                    // `const` ensures that tags are validated at compile-time.
+                    const TAG: TypedUuidTag = TypedUuidTag::new($tag);
+                    TAG
+                }
+            }
+        )*
+    };
+}
+
+// NOTE:
+//
+// This should generally be an append-only list. Removing items from this list
+// will not break things for now (because newtype-uuid does not currently alter
+// any serialization formats), but it may involve some degree of churn across
+// repos.
+//
+// Please keep this list in alphabetical order.
+
+impl_typed_uuid_kind! {
+    LoopbackAddressKind => "loopback_address",
+    TufRepoKind => "tuf_repo",
+}

From 475fb621d43b47ab3f4e9bd69fb21a695d147d73 Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Fri, 16 Feb 2024 09:06:59 -0500
Subject: [PATCH 008/157] Fix bad service NIC slots (#5080)

This is the second half of the fix for #5056. #5065 (already merged)
fixed _how_ we were getting service NICs with nonzero slot values, and
this PR adds a schema migration to apply a one-time fix to any existing
service NICs with nonzero slot values. This matters to the
Reconfigurator, because currently the NICs sled-agent thinks it has
don't match the NICs recorded in CRDB (differing only by slot number).

Closes #5056.
---
 nexus/db-model/src/schema.rs               | 2 +-
 nexus/db-queries/src/db/mod.rs             | 5 +++++
 nexus/db-queries/src/db/pool_connection.rs | 2 +-
 nexus/tests/integration_tests/schema.rs    | 6 ++++++
 schema/crdb/35.0.0/up.sql                  | 6 ++++++
 schema/crdb/dbinit.sql                     | 2 +-
 6 files changed, 20 insertions(+), 3 deletions(-)
 create mode 100644 schema/crdb/35.0.0/up.sql

diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index 9ebf7516cf..3d9050b1af 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(34, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(35, 0, 0);
 
 table! {
     disk (id) {
diff --git a/nexus/db-queries/src/db/mod.rs b/nexus/db-queries/src/db/mod.rs
index 184ebe5f8a..d5262166ee 100644
--- a/nexus/db-queries/src/db/mod.rs
+++ b/nexus/db-queries/src/db/mod.rs
@@ -35,6 +35,11 @@ pub mod subquery;
 pub(crate) mod true_or_cast_error;
 mod update_and_check;
 
+/// Batch statement to disable full table scans.
+// This is `pub` so tests that don't go through our connection pool can disable
+// full table scans the same way pooled connections do.
+pub use pool_connection::DISALLOW_FULL_TABLE_SCAN_SQL;
+
 #[cfg(test)]
 mod test_utils;
 
diff --git a/nexus/db-queries/src/db/pool_connection.rs b/nexus/db-queries/src/db/pool_connection.rs
index d9c50ff26c..66fb125a7c 100644
--- a/nexus/db-queries/src/db/pool_connection.rs
+++ b/nexus/db-queries/src/db/pool_connection.rs
@@ -78,7 +78,7 @@ static CUSTOM_TYPE_KEYS: &'static [&'static str] = &[
 ];
 const CUSTOM_TYPE_SCHEMA: &'static str = "public";
 
-const DISALLOW_FULL_TABLE_SCAN_SQL: &str =
+pub const DISALLOW_FULL_TABLE_SCAN_SQL: &str =
     "set disallow_full_table_scans = on; set large_full_scan_rows = 0;";
 
 #[derive(Debug)]
diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs
index 2d496fcd8e..1c23f1b842 100644
--- a/nexus/tests/integration_tests/schema.rs
+++ b/nexus/tests/integration_tests/schema.rs
@@ -10,6 +10,7 @@ use nexus_db_model::schema::SCHEMA_VERSION as LATEST_SCHEMA_VERSION;
 use nexus_db_queries::db::datastore::{
     all_sql_for_version_migration, EARLIEST_SUPPORTED_VERSION,
 };
+use nexus_db_queries::db::DISALLOW_FULL_TABLE_SCAN_SQL;
 use nexus_test_utils::{db, load_test_config, ControlPlaneTestContextBuilder};
 use omicron_common::api::external::SemverVersion;
 use omicron_common::api::internal::shared::SwitchLocation;
@@ -118,6 +119,11 @@ async fn apply_update(
 
     let client = crdb.connect().await.expect("failed to connect");
 
+    client
+        .batch_execute(DISALLOW_FULL_TABLE_SCAN_SQL)
+        .await
+        .expect("failed to disallow full table scans");
+
     // We skip this for the earliest supported version because these tables
     // might not exist yet.
     if version != EARLIEST_SUPPORTED_VERSION {
diff --git a/schema/crdb/35.0.0/up.sql b/schema/crdb/35.0.0/up.sql
new file mode 100644
index 0000000000..f50e8dc881
--- /dev/null
+++ b/schema/crdb/35.0.0/up.sql
@@ -0,0 +1,6 @@
+-- This isn't realy a schema migration, but is instead a one-off fix for
+-- incorrect data (https://github.com/oxidecomputer/omicron/issues/5056) after
+-- the root cause for the incorrect data has been addressed.
+UPDATE omicron.public.network_interface
+    SET slot = 0
+    WHERE kind = 'service' AND time_deleted IS NULL;
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 254080e92d..6c82b63e6e 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -3515,7 +3515,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '34.0.0', NULL)
+    ( TRUE, NOW(), NOW(), '35.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;

From e2f5eea78d0c3b59a117a3bc78bb4635ce740d94 Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Fri, 16 Feb 2024 12:50:04 -0500
Subject: [PATCH 009/157] Fix typo in printing blueprint diff (#5087)

This seems weird! :)

```
         zone faa4010d-9979-47d3-b104-7c269def8cbb type crucible underlay IP fd00:1122:3344:101::c (unchanged)
         zone fb81a9cc-267e-48d8-bdb3-32dc6225a04f type crucible underlay IP fd00:1122:3344:101::8 (unchanged)
+        zone d86f17d2-e1a5-4744-85c7-8bb3d989ac76 type fd00:1122:3344:101::21 underlay IP nexus (added)
```
---
 nexus/types/src/deployment.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs
index 06427507d5..490097d46d 100644
--- a/nexus/types/src/deployment.rs
+++ b/nexus/types/src/deployment.rs
@@ -615,8 +615,8 @@ impl<'a> std::fmt::Display for OmicronZonesDiff<'a> {
                     f,
                     "+        zone {} type {} underlay IP {} (added)",
                     zone.id,
-                    zone.underlay_address,
                     zone.zone_type.label(),
+                    zone.underlay_address,
                 )?;
             }
         }

From cd901535e9a1075bdcf1f99b257cce2923ce0fb2 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Fri, 16 Feb 2024 10:06:52 -0800
Subject: [PATCH 010/157] chore(deps): update rust crate clap to 4.5 (#5083)

---
 Cargo.lock                | 144 ++++++++++++++++----------------------
 Cargo.toml                |   2 +-
 workspace-hack/Cargo.toml |  14 ++--
 3 files changed, 64 insertions(+), 96 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index fca461f24b..b339054e14 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -112,9 +112,9 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.5.0"
+version = "0.6.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c"
+checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5"
 dependencies = [
  "anstyle",
  "anstyle-parse",
@@ -150,12 +150,12 @@ dependencies = [
 
 [[package]]
 name = "anstyle-wincon"
-version = "2.1.0"
+version = "3.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd"
+checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
 dependencies = [
  "anstyle",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -332,7 +332,7 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc7b2dbe9169059af0f821e811180fddc971fc210c776c133c7819ccd6e478db"
 dependencies = [
- "rustix 0.38.31",
+ "rustix",
  "tempfile",
  "windows-sys 0.52.0",
 ]
@@ -988,9 +988,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.4.3"
+version = "4.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6"
+checksum = "80c21025abd42669a92efc996ef13cfb2c5c627858421ea58d5c3b331a6c134f"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -998,22 +998,22 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.4.2"
+version = "4.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08"
+checksum = "458bf1f341769dfcf849846f65dffdf9146daa56bcd2a47cb4e1de9915567c99"
 dependencies = [
  "anstream",
  "anstyle",
  "clap_lex",
- "strsim 0.10.0",
+ "strsim 0.11.0",
  "terminal_size",
 ]
 
 [[package]]
 name = "clap_derive"
-version = "4.4.2"
+version = "4.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873"
+checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47"
 dependencies = [
  "heck 0.4.1",
  "proc-macro2",
@@ -1023,9 +1023,9 @@ dependencies = [
 
 [[package]]
 name = "clap_lex"
-version = "0.5.1"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961"
+checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
 
 [[package]]
 name = "clipboard-win"
@@ -1201,7 +1201,7 @@ dependencies = [
  "anes",
  "cast",
  "ciborium",
- "clap 4.4.3",
+ "clap 4.5.0",
  "criterion-plot",
  "futures",
  "is-terminal",
@@ -1852,7 +1852,7 @@ dependencies = [
  "anyhow",
  "camino",
  "chrono",
- "clap 4.4.3",
+ "clap 4.5.0",
  "dns-service-client",
  "dropshot",
  "expectorate",
@@ -2268,7 +2268,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5"
 dependencies = [
  "cfg-if",
- "rustix 0.38.31",
+ "rustix",
  "windows-sys 0.48.0",
 ]
 
@@ -2279,7 +2279,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7e5768da2206272c81ef0b5e951a41862938a6070da63bcea197899942d3b947"
 dependencies = [
  "cfg-if",
- "rustix 0.38.31",
+ "rustix",
  "windows-sys 0.52.0",
 ]
 
@@ -2557,7 +2557,7 @@ name = "gateway-cli"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "clap 4.4.3",
+ "clap 4.5.0",
  "futures",
  "gateway-client",
  "gateway-messages",
@@ -3462,7 +3462,7 @@ dependencies = [
  "bytes",
  "camino",
  "cancel-safe-futures",
- "clap 4.4.3",
+ "clap 4.5.0",
  "ddm-admin-client",
  "display-error-chain",
  "futures",
@@ -3522,7 +3522,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
- "clap 4.4.3",
+ "clap 4.5.0",
  "dropshot",
  "expectorate",
  "hyper 0.14.27",
@@ -3603,7 +3603,7 @@ name = "internal-dns-cli"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "clap 4.4.3",
+ "clap 4.5.0",
  "dropshot",
  "internal-dns",
  "omicron-common",
@@ -3613,17 +3613,6 @@ dependencies = [
  "trust-dns-resolver",
 ]
 
-[[package]]
-name = "io-lifetimes"
-version = "1.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2"
-dependencies = [
- "hermit-abi 0.3.2",
- "libc",
- "windows-sys 0.48.0",
-]
-
 [[package]]
 name = "ipcc"
 version = "0.1.0"
@@ -3675,7 +3664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
 dependencies = [
  "hermit-abi 0.3.2",
- "rustix 0.38.31",
+ "rustix",
  "windows-sys 0.48.0",
 ]
 
@@ -3883,7 +3872,7 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6d8de370f98a6cb8a4606618e53e802f93b094ddec0f96988eaec2c27e6e9ce7"
 dependencies = [
- "clap 4.4.3",
+ "clap 4.5.0",
  "termcolor",
  "threadpool",
 ]
@@ -3911,12 +3900,6 @@ version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
 
-[[package]]
-name = "linux-raw-sys"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
-
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.13"
@@ -3945,7 +3928,7 @@ version = "0.2.4"
 source = "git+https://github.com/oxidecomputer/lpc55_support#96f064eaae5e95930efaab6c29fd1b2e22225dac"
 dependencies = [
  "bitfield",
- "clap 4.4.3",
+ "clap 4.5.0",
  "packed_struct",
  "serde",
 ]
@@ -4925,7 +4908,7 @@ dependencies = [
  "anyhow",
  "camino",
  "camino-tempfile",
- "clap 4.4.3",
+ "clap 4.5.0",
  "dropshot",
  "expectorate",
  "futures",
@@ -4958,7 +4941,7 @@ dependencies = [
  "anyhow",
  "base64",
  "camino",
- "clap 4.4.3",
+ "clap 4.5.0",
  "dropshot",
  "expectorate",
  "futures",
@@ -5009,7 +4992,7 @@ dependencies = [
  "camino-tempfile",
  "cancel-safe-futures",
  "chrono",
- "clap 4.4.3",
+ "clap 4.5.0",
  "criterion",
  "crucible-agent-client",
  "crucible-pantry-client",
@@ -5119,7 +5102,7 @@ dependencies = [
  "anyhow",
  "async-bb8-diesel",
  "chrono",
- "clap 4.4.3",
+ "clap 4.5.0",
  "crossterm",
  "crucible-agent-client",
  "csv",
@@ -5170,7 +5153,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "camino",
- "clap 4.4.3",
+ "clap 4.5.0",
  "expectorate",
  "futures",
  "hex",
@@ -5237,7 +5220,7 @@ dependencies = [
  "cancel-safe-futures",
  "cfg-if",
  "chrono",
- "clap 4.4.3",
+ "clap 4.5.0",
  "crucible-agent-client",
  "ddm-admin-client",
  "derive_more",
@@ -5373,7 +5356,7 @@ dependencies = [
  "bytes",
  "chrono",
  "cipher",
- "clap 4.4.3",
+ "clap 4.5.0",
  "clap_builder",
  "console",
  "const-oid",
@@ -5387,7 +5370,6 @@ dependencies = [
  "dof 0.3.0",
  "either",
  "elliptic-curve",
- "errno",
  "ff",
  "flate2",
  "futures",
@@ -5436,7 +5418,7 @@ dependencies = [
  "regex-syntax 0.8.2",
  "reqwest",
  "ring 0.17.7",
- "rustix 0.38.31",
+ "rustix",
  "schemars",
  "semver 1.0.21",
  "serde",
@@ -5750,7 +5732,7 @@ dependencies = [
  "anyhow",
  "camino",
  "chrono",
- "clap 4.4.3",
+ "clap 4.5.0",
  "dropshot",
  "expectorate",
  "futures",
@@ -5793,7 +5775,7 @@ dependencies = [
  "bytes",
  "camino",
  "chrono",
- "clap 4.4.3",
+ "clap 4.5.0",
  "dropshot",
  "expectorate",
  "futures",
@@ -5863,7 +5845,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "chrono",
- "clap 4.4.3",
+ "clap 4.5.0",
  "dropshot",
  "nexus-client",
  "omicron-common",
@@ -5885,7 +5867,7 @@ dependencies = [
  "anyhow",
  "camino",
  "chrono",
- "clap 4.4.3",
+ "clap 4.5.0",
  "omicron-workspace-hack",
  "uuid",
 ]
@@ -6689,7 +6671,7 @@ dependencies = [
  "anyhow",
  "atty",
  "base64",
- "clap 4.4.3",
+ "clap 4.5.0",
  "dropshot",
  "futures",
  "hyper 0.14.27",
@@ -7429,20 +7411,6 @@ dependencies = [
  "toolchain_find",
 ]
 
-[[package]]
-name = "rustix"
-version = "0.37.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06"
-dependencies = [
- "bitflags 1.3.2",
- "errno",
- "io-lifetimes",
- "libc",
- "linux-raw-sys 0.3.8",
- "windows-sys 0.48.0",
-]
-
 [[package]]
 name = "rustix"
 version = "0.38.31"
@@ -7452,7 +7420,7 @@ dependencies = [
  "bitflags 2.4.0",
  "errno",
  "libc",
- "linux-raw-sys 0.4.13",
+ "linux-raw-sys",
  "windows-sys 0.52.0",
 ]
 
@@ -8465,7 +8433,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
- "clap 4.4.3",
+ "clap 4.5.0",
  "dropshot",
  "futures",
  "gateway-messages",
@@ -8655,6 +8623,12 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
+[[package]]
+name = "strsim"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01"
+
 [[package]]
 name = "structmeta"
 version = "0.2.0"
@@ -8952,7 +8926,7 @@ checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67"
 dependencies = [
  "cfg-if",
  "fastrand",
- "rustix 0.38.31",
+ "rustix",
  "windows-sys 0.52.0",
 ]
 
@@ -8978,11 +8952,11 @@ dependencies = [
 
 [[package]]
 name = "terminal_size"
-version = "0.2.6"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e6bf6f19e9f8ed8d4048dc22981458ebcf406d67e94cd422e5ecd73d63b3237"
+checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7"
 dependencies = [
- "rustix 0.37.23",
+ "rustix",
  "windows-sys 0.48.0",
 ]
 
@@ -9643,7 +9617,7 @@ dependencies = [
  "assert_cmd",
  "camino",
  "chrono",
- "clap 4.4.3",
+ "clap 4.5.0",
  "console",
  "datatest-stable",
  "fs-err",
@@ -9889,7 +9863,7 @@ dependencies = [
  "camino",
  "camino-tempfile",
  "chrono",
- "clap 4.4.3",
+ "clap 4.5.0",
  "debug-ignore",
  "display-error-chain",
  "dropshot",
@@ -9920,7 +9894,7 @@ dependencies = [
  "camino",
  "camino-tempfile",
  "cancel-safe-futures",
- "clap 4.4.3",
+ "clap 4.5.0",
  "debug-ignore",
  "derive-where",
  "either",
@@ -10334,7 +10308,7 @@ dependencies = [
  "buf-list",
  "camino",
  "ciborium",
- "clap 4.4.3",
+ "clap 4.5.0",
  "crossterm",
  "futures",
  "humantime",
@@ -10395,7 +10369,7 @@ dependencies = [
  "bytes",
  "camino",
  "ciborium",
- "clap 4.4.3",
+ "clap 4.5.0",
  "crossterm",
  "omicron-workspace-hack",
  "reedline",
@@ -10420,7 +10394,7 @@ dependencies = [
  "bytes",
  "camino",
  "camino-tempfile",
- "clap 4.4.3",
+ "clap 4.5.0",
  "ddm-admin-client",
  "debug-ignore",
  "display-error-chain",
@@ -10737,7 +10711,7 @@ dependencies = [
  "camino",
  "cargo_metadata",
  "cargo_toml",
- "clap 4.4.3",
+ "clap 4.5.0",
 ]
 
 [[package]]
@@ -10870,7 +10844,7 @@ name = "zone-network-setup"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "clap 4.4.3",
+ "clap 4.5.0",
  "dropshot",
  "illumos-utils",
  "omicron-common",
diff --git a/Cargo.toml b/Cargo.toml
index f71bd77730..b140b3946a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -177,7 +177,7 @@ chacha20poly1305 = "0.10.1"
 ciborium = "0.2.2"
 cfg-if = "1.0"
 chrono = { version = "0.4", features = [ "serde" ] }
-clap = { version = "4.4", features = ["cargo", "derive", "env", "wrap_help"] }
+clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] }
 cookie = "0.18"
 criterion = { version = "0.5.1", features = [ "async_tokio" ] }
 crossbeam = "0.8"
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index 098be26460..28338ffce8 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -28,8 +28,8 @@ byteorder = { version = "1.5.0" }
 bytes = { version = "1.5.0", features = ["serde"] }
 chrono = { version = "0.4.31", features = ["alloc", "serde"] }
 cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] }
-clap = { version = "4.4.3", features = ["cargo", "derive", "env", "wrap_help"] }
-clap_builder = { version = "4.4.2", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
+clap = { version = "4.5.0", features = ["cargo", "derive", "env", "wrap_help"] }
+clap_builder = { version = "4.5.0", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
 console = { version = "0.15.8" }
 const-oid = { version = "0.9.5", default-features = false, features = ["db", "std"] }
 crossbeam-epoch = { version = "0.9.15" }
@@ -134,8 +134,8 @@ byteorder = { version = "1.5.0" }
 bytes = { version = "1.5.0", features = ["serde"] }
 chrono = { version = "0.4.31", features = ["alloc", "serde"] }
 cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] }
-clap = { version = "4.4.3", features = ["cargo", "derive", "env", "wrap_help"] }
-clap_builder = { version = "4.4.2", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
+clap = { version = "4.5.0", features = ["cargo", "derive", "env", "wrap_help"] }
+clap_builder = { version = "4.5.0", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
 console = { version = "0.15.8" }
 const-oid = { version = "0.9.5", default-features = false, features = ["db", "std"] }
 crossbeam-epoch = { version = "0.9.15" }
@@ -242,28 +242,24 @@ rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.x86_64-apple-darwin.dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
-errno = { version = "0.3.8", default-features = false, features = ["std"] }
 mio = { version = "0.8.9", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.x86_64-apple-darwin.build-dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
-errno = { version = "0.3.8", default-features = false, features = ["std"] }
 mio = { version = "0.8.9", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.aarch64-apple-darwin.dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
-errno = { version = "0.3.8", default-features = false, features = ["std"] }
 mio = { version = "0.8.9", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.aarch64-apple-darwin.build-dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
-errno = { version = "0.3.8", default-features = false, features = ["std"] }
 mio = { version = "0.8.9", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
@@ -271,7 +267,6 @@ rustix = { version = "0.38.31", features = ["fs", "termios"] }
 [target.x86_64-unknown-illumos.dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
 dof = { version = "0.3.0", default-features = false, features = ["des"] }
-errno = { version = "0.3.8", default-features = false, features = ["std"] }
 mio = { version = "0.8.9", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
@@ -281,7 +276,6 @@ toml_edit = { version = "0.19.15", features = ["serde"] }
 [target.x86_64-unknown-illumos.build-dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
 dof = { version = "0.3.0", default-features = false, features = ["des"] }
-errno = { version = "0.3.8", default-features = false, features = ["std"] }
 mio = { version = "0.8.9", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }

From 25eb244f6a05691e456838e2da2fb985c72a5a52 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Fri, 16 Feb 2024 19:09:23 +0000
Subject: [PATCH 011/157] chore(deps): update rust crate either to 1.10.0
 (#5084)

---
 Cargo.lock                | 4 ++--
 Cargo.toml                | 2 +-
 workspace-hack/Cargo.toml | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index b339054e14..eeca299eff 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2094,9 +2094,9 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.9.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
+checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a"
 
 [[package]]
 name = "elliptic-curve"
diff --git a/Cargo.toml b/Cargo.toml
index b140b3946a..abcd0db4f1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -201,7 +201,7 @@ dns-service-client = { path = "clients/dns-service-client" }
 dpd-client = { path = "clients/dpd-client" }
 dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] }
 dyn-clone = "1.0.16"
-either = "1.9.0"
+either = "1.10.0"
 expectorate = "1.1.0"
 fatfs = "0.3.6"
 filetime = "0.2.23"
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index 28338ffce8..6b7096b8fe 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -39,7 +39,7 @@ crypto-common = { version = "0.1.6", default-features = false, features = ["getr
 der = { version = "0.7.8", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] }
 diesel = { version = "2.1.4", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] }
 digest = { version = "0.10.7", features = ["mac", "oid", "std"] }
-either = { version = "1.9.0" }
+either = { version = "1.10.0" }
 elliptic-curve = { version = "0.13.8", features = ["ecdh", "hazmat", "pem", "std"] }
 ff = { version = "0.13.0", default-features = false, features = ["alloc"] }
 flate2 = { version = "1.0.28" }
@@ -145,7 +145,7 @@ crypto-common = { version = "0.1.6", default-features = false, features = ["getr
 der = { version = "0.7.8", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] }
 diesel = { version = "2.1.4", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] }
 digest = { version = "0.10.7", features = ["mac", "oid", "std"] }
-either = { version = "1.9.0" }
+either = { version = "1.10.0" }
 elliptic-curve = { version = "0.13.8", features = ["ecdh", "hazmat", "pem", "std"] }
 ff = { version = "0.13.0", default-features = false, features = ["alloc"] }
 flate2 = { version = "1.0.28" }

From 9e9e60c7f6456c721348e9edb1b432d310e4b7aa Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Fri, 16 Feb 2024 12:43:26 -0800
Subject: [PATCH 012/157] chore(deps): update rust crate toml_edit to 0.22.6
 (#4996)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock                | 33 ++++++++++++++++-----------------
 Cargo.toml                |  2 +-
 workspace-hack/Cargo.toml |  6 ++++--
 3 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index eeca299eff..ecdd8c235e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5442,6 +5442,7 @@ dependencies = [
  "toml 0.7.8",
  "toml_datetime",
  "toml_edit 0.19.15",
+ "toml_edit 0.22.6",
  "tracing",
  "trust-dns-proto",
  "unicode-bidi",
@@ -9338,7 +9339,7 @@ dependencies = [
  "serde",
  "serde_spanned",
  "toml_datetime",
- "toml_edit 0.22.4",
+ "toml_edit 0.22.6",
 ]
 
 [[package]]
@@ -9360,31 +9361,20 @@ dependencies = [
  "serde",
  "serde_spanned",
  "toml_datetime",
- "winnow",
+ "winnow 0.5.15",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.21.1"
+version = "0.22.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
-dependencies = [
- "indexmap 2.2.3",
- "toml_datetime",
- "winnow",
-]
-
-[[package]]
-name = "toml_edit"
-version = "0.22.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c9ffdf896f8daaabf9b66ba8e77ea1ed5ed0f72821b398aba62352e95062951"
+checksum = "2c1b5fd4128cc8d3e0cb74d4ed9a9cc7c7284becd4df68f5f940e1ad123606f6"
 dependencies = [
  "indexmap 2.2.3",
  "serde",
  "serde_spanned",
  "toml_datetime",
- "winnow",
+ "winnow 0.6.1",
 ]
 
 [[package]]
@@ -10337,7 +10327,7 @@ dependencies = [
  "tokio",
  "tokio-util",
  "toml 0.8.10",
- "toml_edit 0.21.1",
+ "toml_edit 0.22.6",
  "tui-tree-widget",
  "unicode-width",
  "update-engine",
@@ -10664,6 +10654,15 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "winnow"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d90f4e0f530c4c69f62b80d839e9ef3855edc9cba471a160c4d692deed62b401"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "winreg"
 version = "0.50.0"
diff --git a/Cargo.toml b/Cargo.toml
index abcd0db4f1..f15170e2e1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -397,7 +397,7 @@ tokio-stream = "0.1.14"
 tokio-tungstenite = "0.20"
 tokio-util = { version = "0.7.10", features = ["io", "io-util"] }
 toml = "0.8.10"
-toml_edit = "0.21.1"
+toml_edit = "0.22.6"
 tough = { version = "0.16.0", features = [ "http" ] }
 trust-dns-client = "0.22"
 trust-dns-proto = "0.22"
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index 6b7096b8fe..c204b56b4d 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -107,6 +107,7 @@ tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serd
 tokio-stream = { version = "0.1.14", features = ["net"] }
 tokio-util = { version = "0.7.10", features = ["codec", "io-util"] }
 toml = { version = "0.7.8" }
+toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.6", features = ["serde"] }
 tracing = { version = "0.1.37", features = ["log"] }
 trust-dns-proto = { version = "0.22.0" }
 unicode-bidi = { version = "0.3.13" }
@@ -214,6 +215,7 @@ tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serd
 tokio-stream = { version = "0.1.14", features = ["net"] }
 tokio-util = { version = "0.7.10", features = ["codec", "io-util"] }
 toml = { version = "0.7.8" }
+toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.6", features = ["serde"] }
 tracing = { version = "0.1.37", features = ["log"] }
 trust-dns-proto = { version = "0.22.0" }
 unicode-bidi = { version = "0.3.13" }
@@ -271,7 +273,7 @@ mio = { version = "0.8.9", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] }
-toml_edit = { version = "0.19.15", features = ["serde"] }
+toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] }
 
 [target.x86_64-unknown-illumos.build-dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
@@ -280,6 +282,6 @@ mio = { version = "0.8.9", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] }
-toml_edit = { version = "0.19.15", features = ["serde"] }
+toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] }
 
 ### END HAKARI SECTION

From 47a416fe1ee5fcf9ade8788fabf4195369d1b9b3 Mon Sep 17 00:00:00 2001
From: bnaecker <ben@oxide.computer>
Date: Fri, 16 Feb 2024 13:31:23 -0800
Subject: [PATCH 013/157] Order `NextItem` subquery results predictably (#5067)

- Fixes #5055
- Add a new column, `index` to the `NextItem` subquery, which indexes
the shifts from 0..n.
- Add an `ORDER BY index` clause to guarantee order.
- Add test
---
 nexus/db-queries/src/db/queries/disk.rs       |  11 +-
 .../src/db/queries/network_interface.rs       | 104 +++++--
 nexus/db-queries/src/db/queries/next_item.rs  | 262 ++++++++++++++++--
 nexus/db-queries/src/db/queries/vpc.rs        |   8 +-
 4 files changed, 331 insertions(+), 54 deletions(-)

diff --git a/nexus/db-queries/src/db/queries/disk.rs b/nexus/db-queries/src/db/queries/disk.rs
index 9fd56c3ce8..dc1a31dd01 100644
--- a/nexus/db-queries/src/db/queries/disk.rs
+++ b/nexus/db-queries/src/db/queries/disk.rs
@@ -46,11 +46,12 @@ struct NextDiskSlot {
 
 impl NextDiskSlot {
     fn new(instance_id: Uuid) -> Self {
-        let generator = DefaultShiftGenerator {
-            base: 0,
-            max_shift: i64::try_from(MAX_DISKS_PER_INSTANCE).unwrap(),
-            min_shift: 0,
-        };
+        let generator = DefaultShiftGenerator::new(
+            0,
+            i64::try_from(MAX_DISKS_PER_INSTANCE).unwrap(),
+            0,
+        )
+        .expect("invalid min/max shift");
         Self { inner: NextItem::new_scoped(generator, instance_id) }
     }
 }
diff --git a/nexus/db-queries/src/db/queries/network_interface.rs b/nexus/db-queries/src/db/queries/network_interface.rs
index 0643089316..a22c80b232 100644
--- a/nexus/db-queries/src/db/queries/network_interface.rs
+++ b/nexus/db-queries/src/db/queries/network_interface.rs
@@ -516,8 +516,8 @@ impl NextIpv4Address {
         let subnet = IpNetwork::from(subnet);
         let net = IpNetwork::from(first_available_address(&subnet));
         let max_shift = i64::from(last_address_offset(&subnet));
-        let generator =
-            DefaultShiftGenerator { base: net, max_shift, min_shift: 0 };
+        let generator = DefaultShiftGenerator::new(net, max_shift, 0)
+            .expect("invalid min/max shift");
         Self { inner: NextItem::new_scoped(generator, subnet_id) }
     }
 }
@@ -575,12 +575,13 @@ pub struct NextNicSlot {
 
 impl NextNicSlot {
     pub fn new(parent_id: Uuid) -> Self {
-        let generator = DefaultShiftGenerator {
-            base: 0,
-            max_shift: i64::try_from(MAX_NICS_PER_INSTANCE)
+        let generator = DefaultShiftGenerator::new(
+            0,
+            i64::try_from(MAX_NICS_PER_INSTANCE)
                 .expect("Too many network interfaces"),
-            min_shift: 0,
-        };
+            0,
+        )
+        .expect("invalid min/max shift");
         Self { inner: NextItem::new_scoped(generator, parent_id) }
     }
 }
@@ -607,25 +608,62 @@ pub struct NextMacAddress {
     >,
 }
 
+// Helper to ensure we correctly compute the min/max shifts for a next MAC
+// query.
+#[derive(Copy, Clone, Debug)]
+struct NextMacShifts {
+    base: MacAddr,
+    min_shift: i64,
+    max_shift: i64,
+}
+
+impl NextMacShifts {
+    fn for_guest() -> Self {
+        let base = MacAddr::random_guest();
+        Self::shifts_for(base, MacAddr::MIN_GUEST_ADDR, MacAddr::MAX_GUEST_ADDR)
+    }
+
+    fn for_system() -> NextMacShifts {
+        let base = MacAddr::random_system();
+        Self::shifts_for(
+            base,
+            MacAddr::MIN_SYSTEM_ADDR,
+            MacAddr::MAX_SYSTEM_ADDR,
+        )
+    }
+
+    fn shifts_for(base: MacAddr, min: i64, max: i64) -> NextMacShifts {
+        let x = base.to_i64();
+
+        // The max shift is the distance to the last value. This min shift is
+        // always expressed as a negative number, giving the largest leftward
+        // shift, i.e., the distance to the first value.
+        let max_shift = max - x;
+        let min_shift = min - x;
+        Self { base, min_shift, max_shift }
+    }
+}
+
 impl NextMacAddress {
     pub fn new(vpc_id: Uuid, kind: NetworkInterfaceKind) -> Self {
         let (base, max_shift, min_shift) = match kind {
             NetworkInterfaceKind::Instance => {
-                let base = MacAddr::random_guest();
-                let x = base.to_i64();
-                let max_shift = MacAddr::MAX_GUEST_ADDR - x;
-                let min_shift = x - MacAddr::MIN_GUEST_ADDR;
+                let NextMacShifts { base, min_shift, max_shift } =
+                    NextMacShifts::for_guest();
                 (base.into(), max_shift, min_shift)
             }
             NetworkInterfaceKind::Service => {
-                let base = MacAddr::random_system();
-                let x = base.to_i64();
-                let max_shift = MacAddr::MAX_SYSTEM_ADDR - x;
-                let min_shift = x - MacAddr::MAX_SYSTEM_ADDR;
+                let NextMacShifts { base, min_shift, max_shift } =
+                    NextMacShifts::for_system();
                 (base.into(), max_shift, min_shift)
             }
         };
-        let generator = DefaultShiftGenerator { base, max_shift, min_shift };
+        let generator = DefaultShiftGenerator::new(base, max_shift, min_shift)
+            .unwrap_or_else(|| {
+                panic!(
+                "invalid min shift ({min_shift}) or max_shift ({max_shift})"
+            )
+            });
         Self { inner: NextItem::new_scoped(generator, vpc_id) }
     }
 }
@@ -1713,6 +1751,7 @@ mod tests {
     use crate::db::model::NetworkInterface;
     use crate::db::model::Project;
     use crate::db::model::VpcSubnet;
+    use crate::db::queries::network_interface::NextMacShifts;
     use async_bb8_diesel::AsyncRunQueryDsl;
     use dropshot::test_util::LogContext;
     use ipnetwork::Ipv4Network;
@@ -2801,4 +2840,37 @@ mod tests {
             "fd00::5".parse::<IpAddr>().unwrap(),
         );
     }
+
+    #[test]
+    fn test_next_mac_shifts_for_system() {
+        let NextMacShifts { base, min_shift, max_shift } =
+            NextMacShifts::for_system();
+        assert!(base.is_system());
+        assert!(
+            min_shift <= 0,
+            "expected min shift to be negative, found {min_shift}"
+        );
+        assert!(max_shift >= 0, "found {max_shift}");
+        let x = base.to_i64();
+        assert_eq!(x + min_shift, MacAddr::MIN_SYSTEM_ADDR);
+        assert_eq!(x + max_shift, MacAddr::MAX_SYSTEM_ADDR);
+    }
+
+    #[test]
+    fn test_next_mac_shifts_for_guest() {
+        let NextMacShifts { base, min_shift, max_shift } =
+            NextMacShifts::for_guest();
+        assert!(base.is_guest());
+        assert!(
+            min_shift <= 0,
+            "expected min shift to be negative, found {min_shift}"
+        );
+        assert!(
+            max_shift >= 0,
+            "expected max shift to be positive, found {max_shift}"
+        );
+        let x = base.to_i64();
+        assert_eq!(x + min_shift, MacAddr::MIN_GUEST_ADDR);
+        assert_eq!(x + max_shift, MacAddr::MAX_GUEST_ADDR);
+    }
 }
diff --git a/nexus/db-queries/src/db/queries/next_item.rs b/nexus/db-queries/src/db/queries/next_item.rs
index 007aec943d..769c891349 100644
--- a/nexus/db-queries/src/db/queries/next_item.rs
+++ b/nexus/db-queries/src/db/queries/next_item.rs
@@ -35,7 +35,7 @@ use uuid::Uuid;
 /// SELECT
 ///     <first_address> + shift AS ip
 /// FROM
-///     generate_series(0, <max_shift>) AS shift
+///     generate_series(0, <max_shift>) AS shift,
 /// LEFT OUTER JOIN
 ///     network_interface
 /// ON
@@ -43,21 +43,22 @@ use uuid::Uuid;
 ///     (<subnet_id>, <first_address> + shift, TRUE)
 /// WHERE
 ///     ip IS NULL
+/// ORDER BY ip
 /// LIMIT 1
 /// ```
 ///
-/// This query selects the lowest address in the IP subnet that's not already
-/// allocated to a guest interface. Note that the query is linear in the number
-/// of _allocated_ guest addresses. <first_address> and <max_shift> are chosen
-/// based on the subnet and its size, and take into account reserved IP
-/// addresses (such as the broadcast address).
+/// This query selects the next address after <first_address> in the IP subnet
+/// that's not already allocated to a guest interface. Note that the query is
+/// linear in the number of _allocated_ guest addresses. <first_address> and
+/// <max_shift> are chosen based on the subnet and its size, and take into
+/// account reserved IP addresses (such as the broadcast address).
 ///
 /// General query structure
 /// -----------------------
 ///
 /// Much of the value of this type comes from the ability to specify the
 /// starting point for a scan for the next item. In the case above, of an IP
-/// address for a guest NIC, we always try to allocate the lowest available
+/// address for a guest NIC, we always try to allocate the next available
 /// address. This implies the search is linear in the number of allocated IP
 /// addresses, but that runtime cost is acceptable for a few reasons. First, the
 /// predictability of the addresses is nice. Second, the subnets can generally
@@ -81,16 +82,16 @@ use uuid::Uuid;
 /// FROM
 ///     (
 ///         SELECT
-///             shift
+///             "index", shift
 ///         FROM
-///             generate_series(0, <max_shift>)
-///         AS shift
+///             generate_series(0, <n>) AS "index"
+///             generate_series(0, <max_shift>) AS shift,
 ///         UNION ALL
 ///         SELECT
 ///             shift
 ///         FROM
-///             generate_series(<min_shift>, -1)
-///         AS shift
+///             generate_series(<n + 1>, <range_size>) AS "index"
+///             generate_series(<min_shift>, -1) AS shift,
 /// LEFT OUTER JOIN
 ///     <table>
 /// ON
@@ -98,6 +99,7 @@ use uuid::Uuid;
 ///     (<scope_key>, <base> + shift, TRUE)
 /// WHERE
 ///     <item_column> IS NULL
+/// ORDER BY "index"
 /// LIMIT 1
 /// ```
 ///
@@ -120,6 +122,18 @@ use uuid::Uuid;
 /// +------------------------------------------------+
 /// ```
 ///
+/// Ordering
+/// --------
+///
+/// The subquery is designed to select the next address _after_ the provided
+/// base. To preserve this behavior even in situations where we wrap around the
+/// end of the range, we include the `index` column when generating the shifts,
+/// and order the result by that column.
+///
+/// The CockroachDB docs on [ordering] specify that queries without an explicit
+/// `ORDER BY` clause are returned "as the coordinating nodes receives them."
+/// Without this clause, the order is non-deterministic.
+///
 /// Shift generators
 /// ----------------
 ///
@@ -146,6 +160,8 @@ use uuid::Uuid;
 /// there is no scope, which means the items must be globally unique in the
 /// entire table (among non-deleted items). The query is structured slightly
 /// differently in these two cases.
+///
+/// [ordering]: https://www.cockroachlabs.com/docs/stable/select-clause#sorting-and-limiting-query-results
 #[derive(Debug, Clone, Copy)]
 pub(super) struct NextItem<
     Table,
@@ -220,6 +236,9 @@ where
     }
 }
 
+const SHIFT_COLUMN_IDENT: &str = "shift";
+const INDEX_COLUMN_IDENT: &str = "index";
+
 impl<Table, Item, ItemColumn, ScopeKey, ScopeColumn, Generator>
     QueryFragment<Pg>
     for NextItem<Table, Item, ItemColumn, ScopeKey, ScopeColumn, Generator>
@@ -262,7 +281,7 @@ where
             self.shift_generator.base(),
         )?;
         out.push_sql(" + ");
-        out.push_identifier("shift")?;
+        out.push_identifier(SHIFT_COLUMN_IDENT)?;
         out.push_sql(", TRUE) ");
 
         push_next_item_where_clause::<Table, ItemColumn>(out.reborrow())
@@ -303,7 +322,7 @@ where
             self.shift_generator.base(),
         )?;
         out.push_sql(" + ");
-        out.push_identifier("shift")?;
+        out.push_identifier(SHIFT_COLUMN_IDENT)?;
         out.push_sql(", TRUE) ");
 
         push_next_item_where_clause::<Table, ItemColumn>(out.reborrow())
@@ -338,7 +357,7 @@ where
         shift_generator.base(),
     )?;
     out.push_sql(" + ");
-    out.push_identifier("shift")?;
+    out.push_identifier(SHIFT_COLUMN_IDENT)?;
     out.push_sql(" AS ");
     out.push_identifier(ItemColumn::NAME)?;
     out.push_sql(" FROM (");
@@ -350,7 +369,7 @@ where
 // Push the final where clause shared by scoped and unscoped next item queries.
 //
 // ```sql
-// WHERE <item_column> IS NULL LIMIT 1
+// WHERE <item_column> IS NULL ORDER BY "index" LIMIT 1
 // ```
 fn push_next_item_where_clause<Table, ItemColumn>(
     mut out: AstPass<Pg>,
@@ -363,7 +382,9 @@ where
 {
     out.push_sql(" WHERE ");
     out.push_identifier(ItemColumn::NAME)?;
-    out.push_sql(" IS NULL LIMIT 1");
+    out.push_sql(" IS NULL ORDER BY ");
+    out.push_identifier(INDEX_COLUMN_IDENT)?;
+    out.push_sql(" LIMIT 1");
     Ok(())
 }
 
@@ -414,17 +435,24 @@ pub trait ShiftGenerator<Item> {
     /// Return the minimum shift from the base item for the scan.
     fn min_shift(&self) -> &i64;
 
+    /// Return the indices of the generated shifts for the scan.
+    fn shift_indices(&self) -> &ShiftIndices;
+
     /// Insert the part of the query represented by the shift of items into the
     /// provided AstPass.
     ///
     /// The default implementation pushes:
     ///
     /// ```sql
-    /// SELECT generate_series(0, <max_shift>) AS shift
+    /// SELECT
+    ///     generate_series(0, <n>) as "index"
+    ///     generate_series(0, <max_shift>) AS shift,
     /// UNION ALL
-    /// SELECT generate_series(<min_shift>, -1) AS shift
+    /// SELECT
+    ///     generate_series(<n+1>, <range_size>) as "index"
+    ///     generate_series(<min_shift>, -1) AS shift,
     /// ```
-    fn walk_ast<'a, Table, ItemColumn>(
+    fn walk_ast<'a, 'b, Table, ItemColumn>(
         &'a self,
         mut out: AstPass<'_, 'a, Pg>,
     ) -> diesel::QueryResult<()>
@@ -434,15 +462,83 @@ pub trait ShiftGenerator<Item> {
         Item: ToSql<<ItemColumn as Expression>::SqlType, Pg> + Copy,
         ItemColumn: Column<Table = Table> + Copy,
         Pg: HasSqlType<<ItemColumn as Expression>::SqlType>,
+        'a: 'b,
     {
         out.push_sql("SELECT generate_series(0, ");
+        out.push_bind_param::<sql_types::BigInt, i64>(
+            self.shift_indices().first_end(),
+        )?;
+        out.push_sql(") AS ");
+        out.push_identifier(INDEX_COLUMN_IDENT)?;
+        out.push_sql(", generate_series(0, ");
         out.push_bind_param::<sql_types::BigInt, i64>(self.max_shift())?;
         out.push_sql(") AS ");
-        out.push_identifier("shift")?;
+        out.push_identifier(SHIFT_COLUMN_IDENT)?;
         out.push_sql(" UNION ALL SELECT generate_series(");
+        out.push_bind_param::<sql_types::BigInt, i64>(
+            self.shift_indices().second_start(),
+        )?;
+        out.push_sql(", ");
+        out.push_bind_param::<sql_types::BigInt, i64>(
+            self.shift_indices().second_end(),
+        )?;
+        out.push_sql(") AS ");
+        out.push_identifier(INDEX_COLUMN_IDENT)?;
+        out.push_sql(", generate_series(");
         out.push_bind_param::<sql_types::BigInt, i64>(self.min_shift())?;
         out.push_sql(", -1) AS ");
-        out.push_identifier("shift")
+        out.push_identifier(SHIFT_COLUMN_IDENT)
+    }
+}
+
+/// Helper to compute the range of the _index_ column, used to predictably sort
+/// the generated items.
+///
+/// This type cannot be created directly, it's generated internally by creating
+/// a `DefaultShiftGenerator`.
+// NOTE: This type mostly exists to satisfy annoying lifetime constraints
+// imposed by Diesel's `AstPass`. One can only push bind parameters that outlive
+// the AST pass itself, so you cannot push owned values or even references to
+// values generated anywhere within the `walk_ast()` implementation;
+#[derive(Copy, Clone, Debug)]
+pub struct ShiftIndices {
+    // The end of the first range.
+    first_end: i64,
+    // The start of the second range.
+    second_start: i64,
+    // The end of the second range.
+    //
+    // This is equal to the number of items generated for the range.
+    second_end: i64,
+}
+
+impl ShiftIndices {
+    fn new(max_shift: i64, min_shift: i64) -> Self {
+        assert!(max_shift >= 0);
+        assert!(min_shift <= 0);
+
+        // We're just generating the list of indices (0, n_items), but we need
+        // to split it in the middle. Specifically, we'll split it at
+        // `max_shift`, and then generate the remainder.
+        let first_end = max_shift;
+        let second_start = first_end + 1;
+        let second_end = max_shift - min_shift;
+        Self { first_end, second_start, second_end }
+    }
+
+    /// Return the end of the first set of indices.
+    pub fn first_end(&self) -> &i64 {
+        &self.first_end
+    }
+
+    /// Return the start of the second set of indices.
+    pub fn second_start(&self) -> &i64 {
+        &self.second_start
+    }
+
+    /// Return the end of the second set of indices.
+    pub fn second_end(&self) -> &i64 {
+        &self.second_end
     }
 }
 
@@ -450,9 +546,24 @@ pub trait ShiftGenerator<Item> {
 /// implementation.
 #[derive(Debug, Clone, Copy)]
 pub struct DefaultShiftGenerator<Item> {
-    pub base: Item,
-    pub max_shift: i64,
-    pub min_shift: i64,
+    base: Item,
+    max_shift: i64,
+    min_shift: i64,
+    shift_indices: ShiftIndices,
+}
+
+impl<Item> DefaultShiftGenerator<Item> {
+    /// Create a default generator, checking the provided ranges.
+    ///
+    /// Returns `None` if either the max_shift is less than 0, or the min_shift
+    /// is greater than 0.
+    pub fn new(base: Item, max_shift: i64, min_shift: i64) -> Option<Self> {
+        if max_shift < 0 || min_shift > 0 {
+            return None;
+        }
+        let shift_indices = ShiftIndices::new(max_shift, min_shift);
+        Some(Self { base, max_shift, min_shift, shift_indices })
+    }
 }
 
 impl<Item> ShiftGenerator<Item> for DefaultShiftGenerator<Item> {
@@ -467,6 +578,10 @@ impl<Item> ShiftGenerator<Item> for DefaultShiftGenerator<Item> {
     fn min_shift(&self) -> &i64 {
         &self.min_shift
     }
+
+    fn shift_indices(&self) -> &ShiftIndices {
+        &self.shift_indices
+    }
 }
 
 #[cfg(test)]
@@ -474,6 +589,7 @@ mod tests {
 
     use super::DefaultShiftGenerator;
     use super::NextItem;
+    use super::ShiftIndices;
     use crate::db;
     use async_bb8_diesel::AsyncRunQueryDsl;
     use async_bb8_diesel::AsyncSimpleConnection;
@@ -602,8 +718,7 @@ mod tests {
         //
         // This generator should start at 0, and then select over the range [0,
         // 10], wrapping back to 0.
-        let generator =
-            DefaultShiftGenerator { base: 0, max_shift: 10, min_shift: 0 };
+        let generator = DefaultShiftGenerator::new(0, 10, 0).unwrap();
         let query = NextItemQuery::new(generator);
         let it = diesel::insert_into(item::dsl::item)
             .values(query)
@@ -623,8 +738,7 @@ mod tests {
         assert_eq!(it.value, 1);
 
         // Insert 10, and guarantee that we get it back.
-        let generator =
-            DefaultShiftGenerator { base: 10, max_shift: 0, min_shift: -10 };
+        let generator = DefaultShiftGenerator::new(10, 0, -10).unwrap();
         let query = NextItemQuery::new(generator);
         let it = diesel::insert_into(item::dsl::item)
             .values(query)
@@ -647,4 +761,94 @@ mod tests {
         db.cleanup().await.unwrap();
         logctx.cleanup_successful();
     }
+
+    #[tokio::test]
+    async fn test_next_item_query_is_ordered_by_indices() {
+        // Setup the test database
+        let logctx =
+            dev::test_setup_log("test_next_item_query_is_ordered_by_indices");
+        let log = logctx.log.new(o!());
+        let mut db = test_setup_database(&log).await;
+        let cfg = crate::db::Config { url: db.pg_config().clone() };
+        let pool = Arc::new(crate::db::Pool::new(&logctx.log, &cfg));
+        let conn = pool.pool().get().await.unwrap();
+
+        // We're going to operate on a separate table, for simplicity.
+        setup_test_schema(&pool).await;
+
+        // To test ordering behavior, we'll generate a range where the natural
+        // order of the _items_ differs from their indices. I.e., we have some
+        // non-zero base. We'll make sure we order everything by those indices,
+        // not the items themselves.
+        const MIN_SHIFT: i64 = -5;
+        const MAX_SHIFT: i64 = 5;
+        const BASE: i32 = 5;
+        let generator =
+            DefaultShiftGenerator::new(BASE, MAX_SHIFT, MIN_SHIFT).unwrap();
+        let query = NextItemQuery::new(generator);
+
+        // Insert all items until there are none left.
+        let first_range = i64::from(BASE)..=i64::from(BASE) + MAX_SHIFT;
+        let second_range = i64::from(BASE) + MIN_SHIFT..i64::from(BASE);
+        let mut expected = first_range.chain(second_range);
+        while let Some(expected_value) = expected.next() {
+            let it = diesel::insert_into(item::dsl::item)
+                .values(query)
+                .returning(Item::as_returning())
+                .get_result_async(&*conn)
+                .await
+                .unwrap();
+            assert_eq!(i64::from(it.value), expected_value);
+        }
+        assert!(
+            expected.next().is_none(),
+            "Should have exhausted the expected values"
+        );
+        diesel::insert_into(item::dsl::item)
+            .values(query)
+            .returning(Item::as_returning())
+            .get_result_async(&*conn)
+            .await
+            .expect_err(
+                "The next item query should not have further items to generate",
+            );
+
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
+
+    #[test]
+    fn test_shift_indices() {
+        // In this case, we're generating a list of 11 items, all sequential. So
+        // we want the first set of arguments to `generate_series()` to be 0,
+        // 10, and the second set 11, 10. That means the first indices will be
+        // 0..=10 and the second 11..=10, i.e., empty.
+        let min_shift = 0;
+        let max_shift = 10;
+        let indices = ShiftIndices::new(max_shift, min_shift);
+        assert_eq!(indices.first_end, 10);
+        assert_eq!(indices.second_start, 11);
+        assert_eq!(indices.second_end, 10);
+
+        // Here, the list is split in half. We want to still result in a
+        // sequence 0..=10, split at 5. So the arguments to generate_series()
+        // should be (0, 5), and (6, 10).
+        let min_shift = -5;
+        let max_shift = 5;
+        let indices = ShiftIndices::new(max_shift, min_shift);
+        assert_eq!(indices.first_end, 5);
+        assert_eq!(indices.second_start, 6);
+        assert_eq!(indices.second_end, 10);
+
+        // This case tests where most the range is _before_ the base, i.e., the
+        // max shift is zero. Note that this technically still means we have one
+        // item in the list of available, which is the base itself (at a shift
+        // of 0).
+        let min_shift = -10;
+        let max_shift = 0;
+        let indices = ShiftIndices::new(max_shift, min_shift);
+        assert_eq!(indices.first_end, 0);
+        assert_eq!(indices.second_start, 1);
+        assert_eq!(indices.second_end, 10);
+    }
 }
diff --git a/nexus/db-queries/src/db/queries/vpc.rs b/nexus/db-queries/src/db/queries/vpc.rs
index c29a51adb0..2875ae6a05 100644
--- a/nexus/db-queries/src/db/queries/vpc.rs
+++ b/nexus/db-queries/src/db/queries/vpc.rs
@@ -246,8 +246,8 @@ struct NextVni {
 impl NextVni {
     fn new(vni: Vni) -> Self {
         let VniShifts { min_shift, max_shift } = VniShifts::new(vni);
-        let generator =
-            DefaultShiftGenerator { base: vni, max_shift, min_shift };
+        let generator = DefaultShiftGenerator::new(vni, max_shift, min_shift)
+            .expect("invalid min/max shift");
         let inner = NextItem::new_unscoped(generator);
         Self { inner }
     }
@@ -262,8 +262,8 @@ impl NextVni {
             -i32::try_from(base_u32)
                 .expect("Expected a valid VNI at this point"),
         );
-        let generator =
-            DefaultShiftGenerator { base: vni, max_shift, min_shift };
+        let generator = DefaultShiftGenerator::new(vni, max_shift, min_shift)
+            .expect("invalid min/max shift");
         let inner = NextItem::new_unscoped(generator);
         Self { inner }
     }

From 4e6af7a5b67bb10327189c137a2e9e8584e18f1b Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Sat, 17 Feb 2024 04:24:42 +0000
Subject: [PATCH 014/157] chore(deps): update rust crate supports-color to v3
 (#4984)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock | 9 ++++-----
 Cargo.toml | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ecdd8c235e..38a193bedb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3670,9 +3670,9 @@ dependencies = [
 
 [[package]]
 name = "is_ci"
-version = "1.1.1"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "616cde7c720bb2bb5824a224687d8f77bfd38922027f01d825cd7453be5099fb"
+checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45"
 
 [[package]]
 name = "itertools"
@@ -8775,11 +8775,10 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
 
 [[package]]
 name = "supports-color"
-version = "2.1.0"
+version = "3.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6398cde53adc3c4557306a96ce67b302968513830a77a95b2b17305d9719a89"
+checksum = "9829b314621dfc575df4e409e79f9d6a66a3bd707ab73f23cb4aa3a854ac854f"
 dependencies = [
- "is-terminal",
  "is_ci",
 ]
 
diff --git a/Cargo.toml b/Cargo.toml
index f15170e2e1..70d8e3ea48 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -377,7 +377,7 @@ static_assertions = "1.1.0"
 steno = "0.4.0"
 strum = { version = "0.26", features = [ "derive" ] }
 subprocess = "0.2.9"
-supports-color = "2.1.0"
+supports-color = "3.0.0"
 swrite = "0.1.0"
 libsw = { version = "3.3.1", features = ["tokio"] }
 syn = { version = "2.0" }

From aa61741059df67a3d6bf9bc0472536191420c210 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Sat, 17 Feb 2024 05:15:52 +0000
Subject: [PATCH 015/157] chore(deps): update taiki-e/install-action digest to
 bd71f12 (#5082)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index c263eecca3..5927cb43e6 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@14422f84f07a2d547893af56258f5e59333262df # v2
+        uses: taiki-e/install-action@6943331e01261cdff7420bbc2508cb463574e404 # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From 754dd48946ad3bd85f11497d83a35ae5c2ee1bd5 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Sat, 17 Feb 2024 05:16:51 +0000
Subject: [PATCH 016/157] chore(deps): update rust crate multimap to 0.10.0
 (#5093)

---
 Cargo.lock | 4 ++--
 Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 38a193bedb..3b19e5cd95 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4186,9 +4186,9 @@ dependencies = [
 
 [[package]]
 name = "multimap"
-version = "0.8.3"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
+checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03"
 dependencies = [
  "serde",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index 70d8e3ea48..2f39e41e64 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -252,7 +252,7 @@ mime_guess = "2.0.4"
 mockall = "0.12"
 newtype_derive = "0.1.6"
 mg-admin-client = { path = "clients/mg-admin-client" }
-multimap = "0.8.1"
+multimap = "0.10.0"
 nexus-blueprint-execution = { path = "nexus/blueprint-execution" }
 nexus-client = { path = "clients/nexus-client" }
 nexus-db-model = { path = "nexus/db-model" }

From 41e1c79a773834d86c5d95192deafb984e4fadfc Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Sat, 17 Feb 2024 05:56:29 +0000
Subject: [PATCH 017/157] chore(deps): update rust crate reedline to 0.29.0
 (#5094)

---
 Cargo.lock | 8 ++++----
 Cargo.toml | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 3b19e5cd95..6760688424 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4637,9 +4637,9 @@ checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
 
 [[package]]
 name = "nu-ansi-term"
-version = "0.49.0"
+version = "0.50.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c073d3c1930d0751774acf49e66653acecb416c3a54c6ec095a9b11caddb5a68"
+checksum = "dd2800e1520bdc966782168a627aa5d1ad92e33b984bf7c7615d31280c83ff14"
 dependencies = [
  "windows-sys 0.48.0",
 ]
@@ -6984,9 +6984,9 @@ dependencies = [
 
 [[package]]
 name = "reedline"
-version = "0.28.0"
+version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68f4e89a0f80909b3ca4bca9759ed37e4bfddb6f5d2ffb1b4ceb2b1638a3e1eb"
+checksum = "9e01ebfbdb1a88963121d3c928c97be7f10fec7795bec8b918c8cda1db7c29e6"
 dependencies = [
  "chrono",
  "crossterm",
diff --git a/Cargo.toml b/Cargo.toml
index 2f39e41e64..b0d82f8fd6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -323,7 +323,7 @@ rand = "0.8.5"
 ratatui = "0.26.1"
 rayon = "1.8"
 rcgen = "0.12.1"
-reedline = "0.28.0"
+reedline = "0.29.0"
 ref-cast = "1.0"
 regex = "1.10.3"
 regress = "0.8.0"

From d177ccb6e8ab571f45de87635a5699d78d2c6b04 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Sun, 18 Feb 2024 01:12:59 -0800
Subject: [PATCH 018/157] chore(deps): update rust crate ring to 0.17.8 (#5101)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock                | 29 +++++++++++++++--------------
 Cargo.toml                |  2 +-
 workspace-hack/Cargo.toml |  4 ++--
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 6760688424..8ed14304d0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5061,7 +5061,7 @@ dependencies = [
  "ref-cast",
  "regex",
  "reqwest",
- "ring 0.17.7",
+ "ring 0.17.8",
  "rustls 0.22.2",
  "rustls-pemfile 2.0.0",
  "samael",
@@ -5164,7 +5164,7 @@ dependencies = [
  "petgraph",
  "rayon",
  "reqwest",
- "ring 0.17.7",
+ "ring 0.17.8",
  "semver 1.0.21",
  "serde",
  "sled-hardware",
@@ -5317,7 +5317,7 @@ dependencies = [
  "rcgen",
  "regex",
  "reqwest",
- "ring 0.17.7",
+ "ring 0.17.8",
  "rustls 0.22.2",
  "slog",
  "subprocess",
@@ -5417,7 +5417,7 @@ dependencies = [
  "regex-automata 0.4.4",
  "regex-syntax 0.8.2",
  "reqwest",
- "ring 0.17.7",
+ "ring 0.17.8",
  "rustix",
  "schemars",
  "semver 1.0.21",
@@ -6930,7 +6930,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "48406db8ac1f3cbc7dcdb56ec355343817958a356ff430259bb07baf7607e1e1"
 dependencies = [
  "pem",
- "ring 0.17.7",
+ "ring 0.17.8",
  "time",
  "yasna",
 ]
@@ -7176,16 +7176,17 @@ dependencies = [
 
 [[package]]
 name = "ring"
-version = "0.17.7"
+version = "0.17.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74"
+checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
 dependencies = [
  "cc",
+ "cfg-if",
  "getrandom 0.2.10",
  "libc",
  "spin 0.9.8",
  "untrusted 0.9.0",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -7432,7 +7433,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9"
 dependencies = [
  "log",
- "ring 0.17.7",
+ "ring 0.17.8",
  "rustls-webpki 0.101.7",
  "sct",
 ]
@@ -7444,7 +7445,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41"
 dependencies = [
  "log",
- "ring 0.17.7",
+ "ring 0.17.8",
  "rustls-pki-types",
  "rustls-webpki 0.102.1",
  "subtle",
@@ -7495,7 +7496,7 @@ version = "0.101.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
 dependencies = [
- "ring 0.17.7",
+ "ring 0.17.8",
  "untrusted 0.9.0",
 ]
 
@@ -7505,7 +7506,7 @@ version = "0.102.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ef4ca26037c909dedb327b48c3327d0ba91d3dd3c4e05dad328f210ffb68e95b"
 dependencies = [
- "ring 0.17.7",
+ "ring 0.17.8",
  "rustls-pki-types",
  "untrusted 0.9.0",
 ]
@@ -9415,7 +9416,7 @@ dependencies = [
  "pem",
  "percent-encoding",
  "reqwest",
- "ring 0.17.7",
+ "ring 0.17.8",
  "serde",
  "serde_json",
  "serde_plain",
@@ -9647,7 +9648,7 @@ dependencies = [
  "omicron-workspace-hack",
  "parse-size",
  "rand 0.8.5",
- "ring 0.17.7",
+ "ring 0.17.8",
  "serde",
  "serde_json",
  "serde_path_to_error",
diff --git a/Cargo.toml b/Cargo.toml
index b0d82f8fd6..2ba68461d2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -328,7 +328,7 @@ ref-cast = "1.0"
 regex = "1.10.3"
 regress = "0.8.0"
 reqwest = { version = "0.11", default-features = false }
-ring = "0.17.7"
+ring = "0.17.8"
 rpassword = "7.3.1"
 rstest = "0.18.2"
 rustfmt-wrapper = "0.2"
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index c204b56b4d..bbb5191b4c 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -86,7 +86,7 @@ regex = { version = "1.10.3" }
 regex-automata = { version = "0.4.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
 regex-syntax = { version = "0.8.2" }
 reqwest = { version = "0.11.22", features = ["blocking", "json", "rustls-tls", "stream"] }
-ring = { version = "0.17.7", features = ["std"] }
+ring = { version = "0.17.8", features = ["std"] }
 schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid1"] }
 semver = { version = "1.0.21", features = ["serde"] }
 serde = { version = "1.0.196", features = ["alloc", "derive", "rc"] }
@@ -193,7 +193,7 @@ regex = { version = "1.10.3" }
 regex-automata = { version = "0.4.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
 regex-syntax = { version = "0.8.2" }
 reqwest = { version = "0.11.22", features = ["blocking", "json", "rustls-tls", "stream"] }
-ring = { version = "0.17.7", features = ["std"] }
+ring = { version = "0.17.8", features = ["std"] }
 schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid1"] }
 semver = { version = "1.0.21", features = ["serde"] }
 serde = { version = "1.0.196", features = ["alloc", "derive", "rc"] }

From 15e10b251ffdba73a69f972a4b278cbaa461b61d Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Sun, 18 Feb 2024 01:13:10 -0800
Subject: [PATCH 019/157] chore(deps): update rust crate rustls-pemfile to
 2.1.0 (#5095)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock | 14 +++++++-------
 Cargo.toml |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 8ed14304d0..2c6328dc9c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1984,7 +1984,7 @@ dependencies = [
  "percent-encoding",
  "proc-macro2",
  "rustls 0.22.2",
- "rustls-pemfile 2.0.0",
+ "rustls-pemfile 2.1.0",
  "schemars",
  "serde",
  "serde_json",
@@ -5063,7 +5063,7 @@ dependencies = [
  "reqwest",
  "ring 0.17.8",
  "rustls 0.22.2",
- "rustls-pemfile 2.0.0",
+ "rustls-pemfile 2.1.0",
  "samael",
  "schemars",
  "semver 1.0.21",
@@ -7459,7 +7459,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792"
 dependencies = [
  "openssl-probe",
- "rustls-pemfile 2.0.0",
+ "rustls-pemfile 2.1.0",
  "rustls-pki-types",
  "schannel",
  "security-framework",
@@ -7476,9 +7476,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pemfile"
-version = "2.0.0"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4"
+checksum = "3c333bb734fcdedcea57de1602543590f545f127dc8b533324318fd492c5c70b"
 dependencies = [
  "base64",
  "rustls-pki-types",
@@ -7486,9 +7486,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.1.0"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a"
+checksum = "048a63e5b3ac996d78d402940b5fa47973d2d080c6c6fffa1d0f19c4445310b7"
 
 [[package]]
 name = "rustls-webpki"
diff --git a/Cargo.toml b/Cargo.toml
index 2ba68461d2..61b9bb67fd 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -333,7 +333,7 @@ rpassword = "7.3.1"
 rstest = "0.18.2"
 rustfmt-wrapper = "0.2"
 rustls = "0.22.2"
-rustls-pemfile = "2.0.0"
+rustls-pemfile = "2.1.0"
 rustyline = "13.0.0"
 samael = { version = "0.0.14", features = ["xmlsec"] }
 schemars = "0.8.16"

From 9dd2325cde45e70084ce6b10a17394ed628bce8a Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Sun, 18 Feb 2024 10:11:45 +0000
Subject: [PATCH 020/157] fix(deps): update russh monorepo to 0.42.0 (#5098)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock                  | 23 +++++++++++++++++++----
 end-to-end-tests/Cargo.toml |  4 ++--
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2c6328dc9c..884f398a73 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5885,6 +5885,20 @@ dependencies = [
  "sha2",
 ]
 
+[[package]]
+name = "p521"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2"
+dependencies = [
+ "base16ct",
+ "ecdsa",
+ "elliptic-curve",
+ "primeorder",
+ "rand_core 0.6.4",
+ "sha2",
+]
+
 [[package]]
 name = "packed_struct"
 version = "0.10.1"
@@ -7277,9 +7291,9 @@ dependencies = [
 
 [[package]]
 name = "russh"
-version = "0.40.2"
+version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93dab9e1c313d0d04a42e39c0995943fc38c037e2e3fa9c33685777a1aecdfb2"
+checksum = "394cc2733c5b5ca9f342d9532b78599849633ccabdbf40f1af094cacf4d86b62"
 dependencies = [
  "aes",
  "aes-gcm",
@@ -7322,9 +7336,9 @@ dependencies = [
 
 [[package]]
 name = "russh-keys"
-version = "0.40.1"
+version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d0de3cb3cbfa773b7f170b6830565fac207a0d630cc666a29f80097cc374dd8"
+checksum = "3e98aa03d476f8d2bf6e4525291c1eb8e22f4ae9653d7a5458fd53cb0191c741"
 dependencies = [
  "aes",
  "async-trait",
@@ -7345,6 +7359,7 @@ dependencies = [
  "num-bigint",
  "num-integer",
  "p256",
+ "p521",
  "pbkdf2 0.11.0",
  "rand 0.7.3",
  "rand_core 0.6.4",
diff --git a/end-to-end-tests/Cargo.toml b/end-to-end-tests/Cargo.toml
index 5ff958dc9c..fee32a344e 100644
--- a/end-to-end-tests/Cargo.toml
+++ b/end-to-end-tests/Cargo.toml
@@ -16,8 +16,8 @@ omicron-test-utils.workspace = true
 oxide-client.workspace = true
 rand.workspace = true
 reqwest.workspace = true
-russh = "0.40.2"
-russh-keys = "0.40.1"
+russh = "0.42.0"
+russh-keys = "0.42.0"
 serde.workspace = true
 serde_json.workspace = true
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }

From 563234e35ca18d0df64b51431bff10906ba69bf8 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Sun, 18 Feb 2024 16:32:35 +0000
Subject: [PATCH 021/157] chore(deps): update rust crate textwrap to 0.16.1
 (#5102)

---
 Cargo.lock | 8 ++++----
 Cargo.toml | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 884f398a73..20726b84fe 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5141,7 +5141,7 @@ dependencies = [
  "strum 0.26.1",
  "subprocess",
  "tabled",
- "textwrap 0.16.0",
+ "textwrap 0.16.1",
  "tokio",
  "unicode-width",
  "uuid",
@@ -9014,9 +9014,9 @@ dependencies = [
 
 [[package]]
 name = "textwrap"
-version = "0.16.0"
+version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
+checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
 dependencies = [
  "smawk",
  "unicode-linebreak",
@@ -10338,7 +10338,7 @@ dependencies = [
  "slog-term",
  "supports-color",
  "tempfile",
- "textwrap 0.16.0",
+ "textwrap 0.16.1",
  "tokio",
  "tokio-util",
  "toml 0.8.10",
diff --git a/Cargo.toml b/Cargo.toml
index 61b9bb67fd..c7ef94eaa2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -387,7 +387,7 @@ tempdir = "0.3"
 tempfile = "3.10"
 term = "0.7"
 termios = "0.3"
-textwrap = "0.16.0"
+textwrap = "0.16.1"
 test-strategy = "0.3.1"
 thiserror = "1.0"
 tofino = { git = "http://github.com/oxidecomputer/tofino", branch = "main" }

From 93de3fed3e82c39bed0cba9a6b128218e994af8f Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Tue, 20 Feb 2024 10:04:32 -0800
Subject: [PATCH 022/157] chore(deps): update rust crate serde_json to 1.0.114
 (#5106)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock                | 4 ++--
 Cargo.toml                | 2 +-
 workspace-hack/Cargo.toml | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 20726b84fe..dce0d06c3d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7867,9 +7867,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.113"
+version = "1.0.114"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79"
+checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0"
 dependencies = [
  "itoa",
  "ryu",
diff --git a/Cargo.toml b/Cargo.toml
index c7ef94eaa2..ed2e94f8eb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -342,7 +342,7 @@ semver = { version = "1.0.21", features = ["std", "serde"] }
 serde = { version = "1.0", default-features = false, features = [ "derive" ] }
 serde_derive = "1.0"
 serde_human_bytes = { git = "http://github.com/oxidecomputer/serde_human_bytes", branch = "main" }
-serde_json = "1.0.113"
+serde_json = "1.0.114"
 serde_path_to_error = "0.1.15"
 serde_tokenstream = "0.2"
 serde_urlencoded = "0.7.1"
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index bbb5191b4c..14e238ba61 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -90,7 +90,7 @@ ring = { version = "0.17.8", features = ["std"] }
 schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid1"] }
 semver = { version = "1.0.21", features = ["serde"] }
 serde = { version = "1.0.196", features = ["alloc", "derive", "rc"] }
-serde_json = { version = "1.0.113", features = ["raw_value", "unbounded_depth"] }
+serde_json = { version = "1.0.114", features = ["raw_value", "unbounded_depth"] }
 sha2 = { version = "0.10.8", features = ["oid"] }
 similar = { version = "2.3.0", features = ["inline", "unicode"] }
 slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] }
@@ -197,7 +197,7 @@ ring = { version = "0.17.8", features = ["std"] }
 schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid1"] }
 semver = { version = "1.0.21", features = ["serde"] }
 serde = { version = "1.0.196", features = ["alloc", "derive", "rc"] }
-serde_json = { version = "1.0.113", features = ["raw_value", "unbounded_depth"] }
+serde_json = { version = "1.0.114", features = ["raw_value", "unbounded_depth"] }
 sha2 = { version = "0.10.8", features = ["oid"] }
 similar = { version = "2.3.0", features = ["inline", "unicode"] }
 slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] }

From 172cf38036ce7ffd4a603c8f57b6172d3d3706d6 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Tue, 20 Feb 2024 10:04:41 -0800
Subject: [PATCH 023/157] chore(deps): update rust crate semver to 1.0.22
 (#5103)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock                | 24 ++++++++++++------------
 Cargo.toml                |  2 +-
 workspace-hack/Cargo.toml |  4 ++--
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index dce0d06c3d..0f901bdade 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -803,7 +803,7 @@ checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037"
 dependencies = [
  "camino",
  "cargo-platform",
- "semver 1.0.21",
+ "semver 1.0.22",
  "serde",
  "serde_json",
  "thiserror",
@@ -2786,7 +2786,7 @@ dependencies = [
  "once_cell",
  "pathdiff",
  "petgraph",
- "semver 1.0.21",
+ "semver 1.0.22",
  "serde",
  "serde_json",
  "smallvec 1.13.1",
@@ -4334,7 +4334,7 @@ dependencies = [
  "rand 0.8.5",
  "ref-cast",
  "schemars",
- "semver 1.0.21",
+ "semver 1.0.22",
  "serde",
  "serde_json",
  "sled-agent-client",
@@ -4885,7 +4885,7 @@ dependencies = [
  "regress",
  "reqwest",
  "schemars",
- "semver 1.0.21",
+ "semver 1.0.22",
  "serde",
  "serde_human_bytes",
  "serde_json",
@@ -5066,7 +5066,7 @@ dependencies = [
  "rustls-pemfile 2.1.0",
  "samael",
  "schemars",
- "semver 1.0.21",
+ "semver 1.0.22",
  "serde",
  "serde_json",
  "serde_urlencoded",
@@ -5165,7 +5165,7 @@ dependencies = [
  "rayon",
  "reqwest",
  "ring 0.17.8",
- "semver 1.0.21",
+ "semver 1.0.22",
  "serde",
  "sled-hardware",
  "slog",
@@ -5266,7 +5266,7 @@ dependencies = [
  "rcgen",
  "reqwest",
  "schemars",
- "semver 1.0.21",
+ "semver 1.0.22",
  "serde",
  "serde_human_bytes",
  "serde_json",
@@ -5420,7 +5420,7 @@ dependencies = [
  "ring 0.17.8",
  "rustix",
  "schemars",
- "semver 1.0.21",
+ "semver 1.0.22",
  "serde",
  "serde_json",
  "sha2",
@@ -5475,7 +5475,7 @@ dependencies = [
  "hex",
  "reqwest",
  "ring 0.16.20",
- "semver 1.0.21",
+ "semver 1.0.22",
  "serde",
  "serde_derive",
  "serde_json",
@@ -7412,7 +7412,7 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
 dependencies = [
- "semver 1.0.21",
+ "semver 1.0.22",
 ]
 
 [[package]]
@@ -7780,9 +7780,9 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.21"
+version = "1.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0"
+checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca"
 dependencies = [
  "serde",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index ed2e94f8eb..d3f099589f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -338,7 +338,7 @@ rustyline = "13.0.0"
 samael = { version = "0.0.14", features = ["xmlsec"] }
 schemars = "0.8.16"
 secrecy = "0.8.0"
-semver = { version = "1.0.21", features = ["std", "serde"] }
+semver = { version = "1.0.22", features = ["std", "serde"] }
 serde = { version = "1.0", default-features = false, features = [ "derive" ] }
 serde_derive = "1.0"
 serde_human_bytes = { git = "http://github.com/oxidecomputer/serde_human_bytes", branch = "main" }
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index 14e238ba61..ebe683e51a 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -88,7 +88,7 @@ regex-syntax = { version = "0.8.2" }
 reqwest = { version = "0.11.22", features = ["blocking", "json", "rustls-tls", "stream"] }
 ring = { version = "0.17.8", features = ["std"] }
 schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid1"] }
-semver = { version = "1.0.21", features = ["serde"] }
+semver = { version = "1.0.22", features = ["serde"] }
 serde = { version = "1.0.196", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1.0.114", features = ["raw_value", "unbounded_depth"] }
 sha2 = { version = "0.10.8", features = ["oid"] }
@@ -195,7 +195,7 @@ regex-syntax = { version = "0.8.2" }
 reqwest = { version = "0.11.22", features = ["blocking", "json", "rustls-tls", "stream"] }
 ring = { version = "0.17.8", features = ["std"] }
 schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid1"] }
-semver = { version = "1.0.21", features = ["serde"] }
+semver = { version = "1.0.22", features = ["serde"] }
 serde = { version = "1.0.196", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1.0.114", features = ["raw_value", "unbounded_depth"] }
 sha2 = { version = "0.10.8", features = ["oid"] }

From 915276d47096a9ce076453eb75aaa9de29477e80 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Tue, 20 Feb 2024 19:06:11 +0000
Subject: [PATCH 024/157] chore(deps): update rust crate assert_cmd to 2.0.14
 (#5105)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock | 4 ++--
 Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 0f901bdade..bcbcb3e9c8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -236,9 +236,9 @@ dependencies = [
 
 [[package]]
 name = "assert_cmd"
-version = "2.0.13"
+version = "2.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00ad3f3a942eee60335ab4342358c161ee296829e0d16ff42fc1d6cb07815467"
+checksum = "ed72493ac66d5804837f480ab3766c72bdfab91a65e565fc54fa9e42db0073a8"
 dependencies = [
  "anstyle",
  "bstr 1.6.0",
diff --git a/Cargo.toml b/Cargo.toml
index d3f099589f..db37547ea0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -155,7 +155,7 @@ anyhow = "1.0"
 api_identity = { path = "api_identity" }
 approx = "0.5.1"
 assert_matches = "1.5.0"
-assert_cmd = "2.0.13"
+assert_cmd = "2.0.14"
 async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "ed7ab5ef0513ba303d33efd41d3e9e381169d59b" }
 async-trait = "0.1.77"
 atomicwrites = "0.4.3"

From 1e76acade0bcac1415e8709ec52d23515164489c Mon Sep 17 00:00:00 2001
From: David Pacheco <dap@oxidecomputer.com>
Date: Tue, 20 Feb 2024 18:44:12 -0800
Subject: [PATCH 025/157] update internal DNS during blueprint execution
 (#4989)

---
 Cargo.lock                                    |   8 +
 clients/dns-service-client/Cargo.toml         |   1 +
 clients/dns-service-client/src/diff.rs        | 230 ++++++
 clients/dns-service-client/src/lib.rs         |  31 +-
 common/src/api/external/mod.rs                |   6 +
 dev-tools/omdb/src/bin/omdb/nexus.rs          |   2 +-
 internal-dns/src/config.rs                    | 102 ++-
 internal-dns/src/names.rs                     |   6 -
 nexus/blueprint-execution/Cargo.toml          |  11 +-
 nexus/blueprint-execution/src/dns.rs          | 745 ++++++++++++++++++
 nexus/blueprint-execution/src/lib.rs          |  76 +-
 .../blueprint-execution/src/omicron_zones.rs  | 192 ++---
 nexus/db-model/src/deployment.rs              |   3 +
 nexus/db-model/src/schema.rs                  |   4 +-
 nexus/db-queries/src/context.rs               |  59 ++
 .../db-queries/src/db/datastore/deployment.rs |  67 +-
 nexus/db-queries/src/db/datastore/dns.rs      | 277 ++++++-
 nexus/db-queries/src/db/datastore/mod.rs      |   9 +
 nexus/db-queries/src/db/datastore/silo.rs     |   5 +-
 nexus/db-queries/src/db/datastore/sled.rs     |  76 ++
 nexus/deployment/src/blueprint_builder.rs     | 109 ++-
 nexus/deployment/src/planner.rs               |  47 +-
 .../src/app/background/blueprint_execution.rs |  71 +-
 nexus/src/app/background/blueprint_load.rs    |   2 +
 nexus/src/app/background/init.rs              |   3 +-
 nexus/src/app/deployment.rs                   |  35 +-
 nexus/test-utils/src/lib.rs                   |  23 -
 nexus/types/src/deployment.rs                 |  18 +-
 nexus/types/src/inventory.rs                  |   8 +
 openapi/nexus-internal.json                   |  18 +
 schema/crdb/36.0.0/up1.sql                    |   8 +
 schema/crdb/36.0.0/up2.sql                    |   2 +
 schema/crdb/dbinit.sql                        |   7 +-
 sled-agent/src/rack_setup/plan/service.rs     |  95 ++-
 34 files changed, 1975 insertions(+), 381 deletions(-)
 create mode 100644 clients/dns-service-client/src/diff.rs
 create mode 100644 nexus/blueprint-execution/src/dns.rs
 create mode 100644 schema/crdb/36.0.0/up1.sql
 create mode 100644 schema/crdb/36.0.0/up2.sql

diff --git a/Cargo.lock b/Cargo.lock
index bcbcb3e9c8..e15afdfbab 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1887,6 +1887,7 @@ dependencies = [
 name = "dns-service-client"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
  "chrono",
  "http 0.2.11",
  "omicron-workspace-hack",
@@ -4267,21 +4268,28 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "chrono",
+ "dns-service-client",
  "futures",
  "httptest",
+ "internal-dns",
+ "ipnet",
  "nexus-db-model",
  "nexus-db-queries",
+ "nexus-deployment",
+ "nexus-inventory",
  "nexus-test-utils",
  "nexus-test-utils-macros",
  "nexus-types",
  "omicron-common",
  "omicron-nexus",
  "omicron-rpaths",
+ "omicron-test-utils",
  "omicron-workspace-hack",
  "pq-sys",
  "reqwest",
  "sled-agent-client",
  "slog",
+ "slog-error-chain",
  "tokio",
  "uuid",
 ]
diff --git a/clients/dns-service-client/Cargo.toml b/clients/dns-service-client/Cargo.toml
index 6132222b8a..27ffb66d88 100644
--- a/clients/dns-service-client/Cargo.toml
+++ b/clients/dns-service-client/Cargo.toml
@@ -5,6 +5,7 @@ edition = "2021"
 license = "MPL-2.0"
 
 [dependencies]
+anyhow.workspace = true
 chrono.workspace = true
 http.workspace = true
 progenitor.workspace = true
diff --git a/clients/dns-service-client/src/diff.rs b/clients/dns-service-client/src/diff.rs
new file mode 100644
index 0000000000..39d51cc974
--- /dev/null
+++ b/clients/dns-service-client/src/diff.rs
@@ -0,0 +1,230 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use crate::types::DnsConfigParams;
+use crate::types::DnsRecord;
+use crate::DnsRecords;
+use anyhow::ensure;
+use anyhow::Context;
+
+/// Compare the DNS records contained in two sets of DNS configuration
+#[derive(Debug)]
+pub struct DnsDiff<'a> {
+    left: &'a DnsRecords,
+    right: &'a DnsRecords,
+}
+
+impl<'a> DnsDiff<'a> {
+    /// Compare the DNS records contained in two sets of DNS configuration
+    ///
+    /// Both configurations are expected to contain exactly one zone and they
+    /// should have the same name.
+    pub fn new(
+        left: &'a DnsConfigParams,
+        right: &'a DnsConfigParams,
+    ) -> Result<DnsDiff<'a>, anyhow::Error> {
+        let left_zone = left.sole_zone().context("left side of diff")?;
+        let right_zone = right.sole_zone().context("right side of diff")?;
+
+        ensure!(
+            left_zone.zone_name == right_zone.zone_name,
+            "cannot compare DNS configuration from zones with different names: \
+            {:?} vs. {:?}", left_zone.zone_name, right_zone.zone_name,
+        );
+
+        Ok(DnsDiff { left: &left_zone.records, right: &right_zone.records })
+    }
+
+    /// Iterate over the names that are present in the `right` config but
+    /// absent in the `left` one (i.e., added between `left` and `right`)
+    pub fn names_added(&self) -> impl Iterator<Item = (&str, &[DnsRecord])> {
+        self.right
+            .iter()
+            .filter(|(k, _)| !self.left.contains_key(*k))
+            .map(|(k, v)| (k.as_ref(), v.as_ref()))
+    }
+
+    /// Iterate over the names that are present in the `left` config but
+    /// absent in the `right` one (i.e., removed between `left` and `right`)
+    pub fn names_removed(&self) -> impl Iterator<Item = (&str, &[DnsRecord])> {
+        self.left
+            .iter()
+            .filter(|(k, _)| !self.right.contains_key(*k))
+            .map(|(k, v)| (k.as_ref(), v.as_ref()))
+    }
+
+    /// Iterate over the names whose records changed between `left` and `right`.
+    pub fn names_changed(
+        &self,
+    ) -> impl Iterator<Item = (&str, &[DnsRecord], &[DnsRecord])> {
+        self.left.iter().filter_map(|(k, v1)| match self.right.get(k) {
+            Some(v2) if v1 != v2 => {
+                Some((k.as_ref(), v1.as_ref(), v2.as_ref()))
+            }
+            _ => None,
+        })
+    }
+
+    /// Returns true iff there are no differences in the DNS names and records
+    /// described by the given configurations
+    pub fn is_empty(&self) -> bool {
+        self.names_added().next().is_none()
+            && self.names_removed().next().is_none()
+            && self.names_changed().next().is_none()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::DnsDiff;
+    use crate::types::DnsConfigParams;
+    use crate::types::DnsConfigZone;
+    use crate::types::DnsRecord;
+    use chrono::Utc;
+    use std::collections::HashMap;
+    use std::net::Ipv4Addr;
+
+    const ZONE_NAME: &str = "dummy";
+
+    fn example() -> DnsConfigParams {
+        DnsConfigParams {
+            generation: 4,
+            time_created: Utc::now(),
+            zones: vec![DnsConfigZone {
+                zone_name: ZONE_NAME.to_string(),
+                records: HashMap::from([
+                    (
+                        "ex1".to_string(),
+                        vec![DnsRecord::A(Ipv4Addr::LOCALHOST)],
+                    ),
+                    (
+                        "ex2".to_string(),
+                        vec![DnsRecord::A("192.168.1.3".parse().unwrap())],
+                    ),
+                ]),
+            }],
+        }
+    }
+
+    #[test]
+    fn diff_invalid() {
+        let example_empty = DnsConfigParams {
+            generation: 3,
+            time_created: Utc::now(),
+            zones: vec![],
+        };
+
+        // Configs must have at least one zone.
+        let error = DnsDiff::new(&example_empty, &example_empty)
+            .expect_err("unexpectedly succeeded comparing two empty configs");
+        assert!(
+            format!("{:#}", error).contains("expected exactly one DNS zone")
+        );
+
+        let example = example();
+        let error = DnsDiff::new(&example_empty, &example)
+            .expect_err("unexpectedly succeeded comparing an empty config");
+        assert!(
+            format!("{:#}", error).contains("expected exactly one DNS zone")
+        );
+
+        // Configs must not have more than one zone.
+        let example_multiple = DnsConfigParams {
+            generation: 3,
+            time_created: Utc::now(),
+            zones: vec![
+                DnsConfigZone {
+                    zone_name: ZONE_NAME.to_string(),
+                    records: HashMap::new(),
+                },
+                DnsConfigZone {
+                    zone_name: "two".to_string(),
+                    records: HashMap::new(),
+                },
+            ],
+        };
+        let error = DnsDiff::new(&example_multiple, &example).expect_err(
+            "unexpectedly succeeded comparing config with multiple zones",
+        );
+        assert!(
+            format!("{:#}", error).contains("expected exactly one DNS zone")
+        );
+
+        // Cannot compare different zone names
+        let example_different_zone = DnsConfigParams {
+            generation: 3,
+            time_created: Utc::now(),
+            zones: vec![DnsConfigZone {
+                zone_name: format!("{}-other", ZONE_NAME),
+                records: HashMap::new(),
+            }],
+        };
+        let error = DnsDiff::new(&example_different_zone, &example).expect_err(
+            "unexpectedly succeeded comparing configs with \
+            different zone names",
+        );
+        assert_eq!(
+            format!("{:#}", error),
+            "cannot compare DNS configuration from zones with different \
+            names: \"dummy-other\" vs. \"dummy\""
+        );
+    }
+
+    #[test]
+    fn diff_equivalent() {
+        let example = example();
+        let diff = DnsDiff::new(&example, &example).unwrap();
+        assert!(diff.is_empty());
+        assert_eq!(diff.names_removed().count(), 0);
+        assert_eq!(diff.names_added().count(), 0);
+        assert_eq!(diff.names_changed().count(), 0);
+    }
+
+    #[test]
+    fn diff_different() {
+        let example = example();
+        let example2 = DnsConfigParams {
+            generation: 4,
+            time_created: Utc::now(),
+            zones: vec![DnsConfigZone {
+                zone_name: ZONE_NAME.to_string(),
+                records: HashMap::from([
+                    (
+                        "ex2".to_string(),
+                        vec![DnsRecord::A("192.168.1.4".parse().unwrap())],
+                    ),
+                    (
+                        "ex3".to_string(),
+                        vec![DnsRecord::A(std::net::Ipv4Addr::LOCALHOST)],
+                    ),
+                ]),
+            }],
+        };
+
+        let diff = DnsDiff::new(&example, &example2).unwrap();
+        assert!(!diff.is_empty());
+
+        let removed = diff.names_removed().collect::<Vec<_>>();
+        assert_eq!(removed.len(), 1);
+        assert_eq!(removed[0].0, "ex1");
+        assert_eq!(removed[0].1, vec![DnsRecord::A(Ipv4Addr::LOCALHOST)]);
+
+        let added = diff.names_added().collect::<Vec<_>>();
+        assert_eq!(added.len(), 1);
+        assert_eq!(added[0].0, "ex3");
+        assert_eq!(added[0].1, vec![DnsRecord::A(Ipv4Addr::LOCALHOST)]);
+
+        let changed = diff.names_changed().collect::<Vec<_>>();
+        assert_eq!(changed.len(), 1);
+        assert_eq!(changed[0].0, "ex2");
+        assert_eq!(
+            changed[0].1,
+            vec![DnsRecord::A("192.168.1.3".parse().unwrap())]
+        );
+        assert_eq!(
+            changed[0].2,
+            vec![DnsRecord::A("192.168.1.4".parse().unwrap())]
+        );
+    }
+}
diff --git a/clients/dns-service-client/src/lib.rs b/clients/dns-service-client/src/lib.rs
index 52c2b8bcd2..cd17a1559c 100644
--- a/clients/dns-service-client/src/lib.rs
+++ b/clients/dns-service-client/src/lib.rs
@@ -2,10 +2,17 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
+mod diff;
+
+use crate::Error as DnsConfigError;
+use anyhow::ensure;
+pub use diff::DnsDiff;
+use std::collections::HashMap;
+
 progenitor::generate_api!(
     spec = "../../openapi/dns-server.json",
     inner_type = slog::Logger,
-    derives = [schemars::JsonSchema, Eq, PartialEq],
+    derives = [schemars::JsonSchema, Clone, Eq, PartialEq],
     pre_hook = (|log: &slog::Logger, request: &reqwest::Request| {
         slog::debug!(log, "client request";
             "method" => %request.method(),
@@ -22,8 +29,6 @@ pub const ERROR_CODE_UPDATE_IN_PROGRESS: &'static str = "UpdateInProgress";
 pub const ERROR_CODE_BAD_UPDATE_GENERATION: &'static str =
     "BadUpdateGeneration";
 
-use crate::Error as DnsConfigError;
-
 /// Returns whether an error from this client should be retried
 pub fn is_retryable(error: &DnsConfigError<crate::types::Error>) -> bool {
     let response_value = match error {
@@ -84,3 +89,23 @@ pub fn is_retryable(error: &DnsConfigError<crate::types::Error>) -> bool {
 
     false
 }
+
+type DnsRecords = HashMap<String, Vec<types::DnsRecord>>;
+
+impl types::DnsConfigParams {
+    /// Given a high-level DNS configuration, return a reference to its sole
+    /// DNS zone.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if there are 0 or more than one zones in this
+    /// configuration.
+    pub fn sole_zone(&self) -> Result<&types::DnsConfigZone, anyhow::Error> {
+        ensure!(
+            self.zones.len() == 1,
+            "expected exactly one DNS zone, but found {}",
+            self.zones.len()
+        );
+        Ok(&self.zones[0])
+    }
+}
diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs
index cdc929d89b..17b4826f8c 100644
--- a/common/src/api/external/mod.rs
+++ b/common/src/api/external/mod.rs
@@ -679,6 +679,12 @@ impl From<&Generation> for i64 {
     }
 }
 
+impl From<Generation> for u64 {
+    fn from(g: Generation) -> Self {
+        g.0
+    }
+}
+
 impl From<u32> for Generation {
     fn from(value: u32) -> Self {
         Generation(u64::from(value))
diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs
index 9904263067..aed7d86ba0 100644
--- a/dev-tools/omdb/src/bin/omdb/nexus.rs
+++ b/dev-tools/omdb/src/bin/omdb/nexus.rs
@@ -870,7 +870,7 @@ async fn cmd_nexus_blueprints_diff(
     let b2 = client.blueprint_view(&args.blueprint2_id).await.with_context(
         || format!("fetching blueprint {}", args.blueprint2_id),
     )?;
-    println!("{}", b1.diff(&b2));
+    println!("{}", b1.diff_sleds(&b2));
     Ok(())
 }
 
diff --git a/internal-dns/src/config.rs b/internal-dns/src/config.rs
index bf1d9b763b..5eee34bd51 100644
--- a/internal-dns/src/config.rs
+++ b/internal-dns/src/config.rs
@@ -63,9 +63,9 @@
 use crate::names::{ServiceName, DNS_ZONE};
 use anyhow::{anyhow, ensure};
 use dns_service_client::types::{DnsConfigParams, DnsConfigZone, DnsRecord};
-use omicron_common::api::internal::shared::SwitchLocation;
+use omicron_common::api::external::Generation;
 use std::collections::BTreeMap;
-use std::net::{Ipv6Addr, SocketAddrV6};
+use std::net::Ipv6Addr;
 use uuid::Uuid;
 
 /// Zones that can be referenced within the internal DNS system.
@@ -147,8 +147,6 @@ pub struct DnsConfigBuilder {
     /// network
     sleds: BTreeMap<Sled, Ipv6Addr>,
 
-    scrimlets: BTreeMap<SwitchLocation, SocketAddrV6>,
-
     /// set of hosts of type "zone" that have been configured so far, mapping
     /// each zone's unique uuid to its sole IPv6 address on the control plane
     /// network
@@ -163,6 +161,9 @@ pub struct DnsConfigBuilder {
 
     /// similar to service_instances_zones, but for services that run on sleds
     service_instances_sleds: BTreeMap<ServiceName, BTreeMap<Sled, u16>>,
+
+    /// generation number for this config
+    generation: Generation,
 }
 
 /// Describes a host of type "sled" in the control plane DNS zone
@@ -192,16 +193,17 @@ impl DnsConfigBuilder {
         DnsConfigBuilder {
             sleds: BTreeMap::new(),
             zones: BTreeMap::new(),
-            scrimlets: BTreeMap::new(),
             service_instances_zones: BTreeMap::new(),
             service_instances_sleds: BTreeMap::new(),
+            generation: Generation::new(),
         }
     }
 
     /// Add a new host of type "sled" to the configuration
     ///
-    /// Returns a [`Sled`] that can be used with [`Self::service_backend_sled()`] to
-    /// specify that this sled is a backend for some higher-level service.
+    /// Returns a [`Sled`] that can be used with
+    /// [`Self::service_backend_sled()`] to specify that this sled is a backend
+    /// for some higher-level service.
     ///
     /// # Errors
     ///
@@ -223,19 +225,11 @@ impl DnsConfigBuilder {
         }
     }
 
-    pub fn host_scrimlet(
-        &mut self,
-        switch_location: SwitchLocation,
-        addr: SocketAddrV6,
-    ) -> anyhow::Result<()> {
-        self.scrimlets.insert(switch_location, addr);
-        Ok(())
-    }
-
     /// Add a new dendrite host of type "zone" to the configuration
     ///
-    /// Returns a [`Zone`] that can be used with [`Self::service_backend_zone()`] to
-    /// specify that this zone is a backend for some higher-level service.
+    /// Returns a [`Zone`] that can be used with
+    /// [`Self::service_backend_zone()`] to specify that this zone is a backend
+    /// for some higher-level service.
     ///
     /// # Errors
     ///
@@ -251,8 +245,9 @@ impl DnsConfigBuilder {
 
     /// Add a new host of type "zone" to the configuration
     ///
-    /// Returns a [`Zone`] that can be used with [`Self::service_backend_zone()`] to
-    /// specify that this zone is a backend for some higher-level service.
+    /// Returns a [`Zone`] that can be used with
+    /// [`Self::service_backend_zone()`] to specify that this zone is a backend
+    /// for some higher-level service.
     ///
     /// # Errors
     ///
@@ -363,6 +358,52 @@ impl DnsConfigBuilder {
         }
     }
 
+    /// Higher-level shorthand for adding a zone with a single backend service
+    ///
+    /// # Errors
+    ///
+    /// This function fails only if the given zone has already been added to the
+    /// configuration.
+    pub fn host_zone_with_one_backend(
+        &mut self,
+        zone_id: Uuid,
+        addr: Ipv6Addr,
+        service: ServiceName,
+        port: u16,
+    ) -> anyhow::Result<()> {
+        let zone = self.host_zone(zone_id, addr)?;
+        self.service_backend_zone(service, &zone, port)
+    }
+
+    /// Higher-level shorthand for adding a "switch" zone with its usual set of
+    /// backend services
+    ///
+    /// # Errors
+    ///
+    /// This function fails only if the given zone has already been added to the
+    /// configuration.
+    pub fn host_zone_switch(
+        &mut self,
+        sled_id: Uuid,
+        switch_zone_ip: Ipv6Addr,
+        dendrite_port: u16,
+        mgs_port: u16,
+        mgd_port: u16,
+    ) -> anyhow::Result<()> {
+        let zone = self.host_dendrite(sled_id, switch_zone_ip)?;
+        self.service_backend_zone(ServiceName::Dendrite, &zone, dendrite_port)?;
+        self.service_backend_zone(
+            ServiceName::ManagementGatewayService,
+            &zone,
+            mgs_port,
+        )?;
+        self.service_backend_zone(ServiceName::Mgd, &zone, mgd_port)
+    }
+
+    pub fn generation(&mut self, generation: Generation) {
+        self.generation = generation;
+    }
+
     /// Construct a complete [`DnsConfigParams`] (suitable for propagating to
     /// our DNS servers) for the control plane DNS zone described up to this
     /// point
@@ -378,23 +419,6 @@ impl DnsConfigBuilder {
             (zone.dns_name(), vec![DnsRecord::Aaaa(zone_ip)])
         });
 
-        let scrimlet_srv_records =
-            self.scrimlets.clone().into_iter().map(|(location, addr)| {
-                let srv = DnsRecord::Srv(dns_service_client::types::Srv {
-                    prio: 0,
-                    weight: 0,
-                    port: addr.port(),
-                    target: format!("{location}.scrimlet.{}", DNS_ZONE),
-                });
-                (ServiceName::Scrimlet(location).dns_name(), vec![srv])
-            });
-
-        let scrimlet_aaaa_records =
-            self.scrimlets.into_iter().map(|(location, addr)| {
-                let aaaa = DnsRecord::Aaaa(*addr.ip());
-                (format!("{location}.scrimlet"), vec![aaaa])
-            });
-
         // Assemble the set of SRV records, which implicitly point back at
         // zones' AAAA records.
         let srv_records_zones = self.service_instances_zones.into_iter().map(
@@ -439,12 +463,10 @@ impl DnsConfigBuilder {
             .chain(zone_records)
             .chain(srv_records_sleds)
             .chain(srv_records_zones)
-            .chain(scrimlet_aaaa_records)
-            .chain(scrimlet_srv_records)
             .collect();
 
         DnsConfigParams {
-            generation: 1,
+            generation: u64::from(self.generation),
             time_created: chrono::Utc::now(),
             zones: vec![DnsConfigZone {
                 zone_name: DNS_ZONE.to_owned(),
diff --git a/internal-dns/src/names.rs b/internal-dns/src/names.rs
index bffe6e829a..8cafe4ac97 100644
--- a/internal-dns/src/names.rs
+++ b/internal-dns/src/names.rs
@@ -4,7 +4,6 @@
 
 //! Well-known DNS names and related types for internal DNS (see RFD 248)
 
-use omicron_common::api::internal::shared::SwitchLocation;
 use uuid::Uuid;
 
 /// Name for the control plane DNS zone
@@ -35,7 +34,6 @@ pub enum ServiceName {
     InternalNtp,
     Maghemite, //TODO change to Dpd - maghemite has several services.
     Mgd,
-    Scrimlet(SwitchLocation),
 }
 
 impl ServiceName {
@@ -59,7 +57,6 @@ impl ServiceName {
             ServiceName::InternalNtp => "internal-ntp",
             ServiceName::Maghemite => "maghemite",
             ServiceName::Mgd => "mgd",
-            ServiceName::Scrimlet(_) => "scrimlet",
         }
     }
 
@@ -91,9 +88,6 @@ impl ServiceName {
             ServiceName::Crucible(id) => {
                 format!("_{}._tcp.{}", self.service_kind(), id)
             }
-            ServiceName::Scrimlet(location) => {
-                format!("_{location}._scrimlet._tcp")
-            }
         }
     }
 
diff --git a/nexus/blueprint-execution/Cargo.toml b/nexus/blueprint-execution/Cargo.toml
index 11d8003599..3284bda27e 100644
--- a/nexus/blueprint-execution/Cargo.toml
+++ b/nexus/blueprint-execution/Cargo.toml
@@ -8,12 +8,17 @@ omicron-rpaths.workspace = true
 
 [dependencies]
 anyhow.workspace = true
+dns-service-client.workspace = true
 futures.workspace = true
+internal-dns.workspace = true
+nexus-db-model.workspace = true
 nexus-db-queries.workspace = true
 nexus-types.workspace = true
+omicron-common.workspace = true
 reqwest.workspace = true
 sled-agent-client.workspace = true
 slog.workspace = true
+slog-error-chain.workspace = true
 uuid.workspace = true
 
 # See omicron-rpaths for more about the "pq-sys" dependency.  This is needed
@@ -26,9 +31,11 @@ omicron-workspace-hack.workspace = true
 [dev-dependencies]
 chrono.workspace = true
 httptest.workspace = true
-nexus-db-model.workspace = true
+ipnet.workspace = true
+nexus-deployment.workspace = true
+nexus-inventory.workspace = true
 nexus-test-utils.workspace = true
 nexus-test-utils-macros.workspace = true
-omicron-common.workspace = true
 omicron-nexus.workspace = true
+omicron-test-utils.workspace = true
 tokio.workspace = true
diff --git a/nexus/blueprint-execution/src/dns.rs b/nexus/blueprint-execution/src/dns.rs
new file mode 100644
index 0000000000..6dd9266f32
--- /dev/null
+++ b/nexus/blueprint-execution/src/dns.rs
@@ -0,0 +1,745 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Propagates internal DNS changes in a given blueprint
+
+use crate::Sled;
+use dns_service_client::DnsDiff;
+use internal_dns::DnsConfigBuilder;
+use internal_dns::ServiceName;
+use nexus_db_model::DnsGroup;
+use nexus_db_queries::context::OpContext;
+use nexus_db_queries::db::datastore::DnsVersionUpdateBuilder;
+use nexus_db_queries::db::DataStore;
+use nexus_types::deployment::Blueprint;
+use nexus_types::deployment::OmicronZoneType;
+use nexus_types::internal_api::params::DnsConfigParams;
+use omicron_common::address::get_switch_zone_address;
+use omicron_common::address::CLICKHOUSE_KEEPER_PORT;
+use omicron_common::address::CLICKHOUSE_PORT;
+use omicron_common::address::COCKROACH_PORT;
+use omicron_common::address::CRUCIBLE_PANTRY_PORT;
+use omicron_common::address::CRUCIBLE_PORT;
+use omicron_common::address::DENDRITE_PORT;
+use omicron_common::address::DNS_HTTP_PORT;
+use omicron_common::address::MGD_PORT;
+use omicron_common::address::MGS_PORT;
+use omicron_common::address::NEXUS_INTERNAL_PORT;
+use omicron_common::address::NTP_PORT;
+use omicron_common::address::OXIMETER_PORT;
+use omicron_common::api::external::Error;
+use omicron_common::api::external::Generation;
+use omicron_common::api::external::InternalContext;
+use slog::{debug, info, o};
+use std::collections::BTreeMap;
+use uuid::Uuid;
+
+pub(crate) async fn deploy_dns(
+    opctx: &OpContext,
+    datastore: &DataStore,
+    creator: String,
+    blueprint: &Blueprint,
+    sleds_by_id: &BTreeMap<Uuid, Sled>,
+) -> Result<(), Error> {
+    // First, fetch the current DNS config.
+    let dns_config_current = datastore
+        .dns_config_read(opctx, DnsGroup::Internal)
+        .await
+        .internal_context("reading current DNS")?;
+
+    // We could check here that the DNS version we found isn't newer than when
+    // the blueprint was generated.  But we have to check later when we try to
+    // update the database anyway.  And we're not wasting much effort allowing
+    // this proceed for now.  This way, we have only one code path for this and
+    // we know it's being hit when we exercise this condition.
+
+    // Next, construct the DNS config represented by the blueprint.
+    let dns_config_blueprint = blueprint_dns_config(blueprint, sleds_by_id);
+
+    // Looking at the current contents of DNS, prepare an update that will make
+    // it match what it should be.
+    let log = opctx.log.new(o!("blueprint_execution" => "DNS"));
+    let comment = format!("blueprint {} ({})", blueprint.id, blueprint.comment);
+    let maybe_update = dns_compute_update(
+        &log,
+        comment,
+        creator,
+        &dns_config_current,
+        &dns_config_blueprint,
+    )?;
+    let Some(update) = maybe_update else {
+        // Nothing to do.
+        return Ok(());
+    };
+
+    // Our goal here is to update the DNS configuration stored in the database
+    // to match the blueprint.  But it's always possible that we're executing a
+    // blueprint that's no longer the current target.  In that case, we want to
+    // fail without making any changes.  We definitely don't want to
+    // accidentally clobber changes that have been made by another instance
+    // executing a newer target blueprint.
+    //
+    // To avoid this problem, before generating a blueprint, Nexus fetches the
+    // current internal DNS generation and stores that into the blueprint
+    // itself.  Here, when we execute the blueprint, we make our database update
+    // conditional on that still being the current internal DNS generation.
+    // If some other instance has already come along and updated the database,
+    // whether for this same blueprint or a newer one, our attempt to update the
+    // database will fail.
+    //
+    // Let's look at a tricky example.  Suppose:
+    //
+    // 1. The system starts with some initial blueprint B1 with DNS version 3.
+    //    The blueprint has been fully executed and all is well.
+    //
+    // 2. Blueprint B2 gets generated.  It stores DNS version 3.  It's made the
+    //    current target.  Execution has not started yet.
+    //
+    // 3. Blueprint B3 gets generated.  It also stores DNS version 3 because
+    //    that's still the current version in DNS.  B3 is made the current
+    //    target.
+    //
+    //    Assume B2 and B3 specify different internal DNS contents (e.g., have a
+    //    different set of Omicron zones in them).
+    //
+    // 4. Nexus instance N1 finds B2 to be the current target and starts
+    //    executing it.  (Assume it found this between 2 and 3 above.)
+    //
+    // 5. Nexus instance N2 finds B3 to be the current target and starts
+    //    executing it.
+    //
+    // During execution:
+    //
+    // * N1 will assemble a new version of DNS called version 4, generate a diff
+    //   between version 3 (which is fixed) and version 4, and attempt to apply
+    //   this to the database conditional on the current version being version
+    //   3.
+    //
+    // * N2 will do the same, but its version 4 will look different.
+    //
+    // Now, one of two things could happen:
+    //
+    // 1. N1 wins.  Its database update applies successfully.  In the database,
+    //    the internal DNS version becomes version 4.  In this case, N2 loses.
+    //    Its database operation fails altogether.  At this point, any
+    //    subsequent attempt to execute blueprint B3 will fail because any DNS
+    //    update will be conditional on the database having version 3.  The only
+    //    way out of this is for the planner to generate a new blueprint B4
+    //    that's exactly equivalent to B3 except that the stored internal DNS
+    //    version is 4.  Then we'll be able to execute that.
+    //
+    // 2. N2 wins.  Its database update applies successfully.  In the database,
+    //    the internal DNS version becomes version 4.  In this case, N1 loses.
+    //    Its database operation fails altogether.  At this point, any
+    //    subsequent attempt to execute blueprint B3 will fail because any DNS
+    //    update will be conditional on the databae having version 3.  No
+    //    further action is needed, though, because we've successfully executed
+    //    the latest target blueprint.
+    //
+    // In both cases, the system will (1) converge to having successfully
+    // executed the target blueprint, and (2) never have rolled any changes back
+    // -- DNS only ever moves forward, closer to the latest desired state.
+    info!(
+        log,
+        "attempting to update from generation {} to generation {}",
+        dns_config_current.generation,
+        dns_config_blueprint.generation,
+    );
+    let generation_u32 =
+        u32::try_from(dns_config_current.generation).map_err(|e| {
+            Error::internal_error(&format!(
+                "internal DNS generation got too large: {}",
+                e,
+            ))
+        })?;
+    let generation =
+        nexus_db_model::Generation::from(Generation::from(generation_u32));
+    datastore.dns_update_from_version(opctx, update, generation).await
+}
+
+/// Returns the expected contents of internal DNS based on the given blueprint
+pub fn blueprint_dns_config(
+    blueprint: &Blueprint,
+    sleds_by_id: &BTreeMap<Uuid, Sled>,
+) -> DnsConfigParams {
+    // The DNS names configured here should match what RSS configures for the
+    // same zones.  It's tricky to have RSS share the same code because it uses
+    // Sled Agent's _internal_ `OmicronZoneConfig` (and friends), whereas we're
+    // using `sled-agent-client`'s version of that type.  However, the
+    // DnsConfigBuilder's interface is high-level enough that it handles most of
+    // the details.
+    let mut dns_builder = DnsConfigBuilder::new();
+
+    // The code below assumes that all zones are using the default port numbers.
+    // That should be true, as those are the only ports ever used today.
+    // In an ideal world, the correct port would be pulled out of the
+    // `OmicronZoneType` variant instead.  Although that information is present,
+    // it's irritatingly non-trivial to do right now because SocketAddrs are
+    // represented as strings, so we'd need to parse all of them and handle all
+    // the errors, even though they should never happen.
+    // See oxidecomputer/omicron#4988.
+    for (_, omicron_zone) in blueprint.all_omicron_zones() {
+        if !blueprint.zones_in_service.contains(&omicron_zone.id) {
+            continue;
+        }
+
+        let (service_name, port) = match omicron_zone.zone_type {
+            OmicronZoneType::BoundaryNtp { .. } => {
+                (ServiceName::BoundaryNtp, NTP_PORT)
+            }
+            OmicronZoneType::InternalNtp { .. } => {
+                (ServiceName::InternalNtp, NTP_PORT)
+            }
+            OmicronZoneType::Clickhouse { .. } => {
+                (ServiceName::Clickhouse, CLICKHOUSE_PORT)
+            }
+            OmicronZoneType::ClickhouseKeeper { .. } => {
+                (ServiceName::ClickhouseKeeper, CLICKHOUSE_KEEPER_PORT)
+            }
+            OmicronZoneType::CockroachDb { .. } => {
+                (ServiceName::Cockroach, COCKROACH_PORT)
+            }
+            OmicronZoneType::Nexus { .. } => {
+                (ServiceName::Nexus, NEXUS_INTERNAL_PORT)
+            }
+            OmicronZoneType::Crucible { .. } => {
+                (ServiceName::Crucible(omicron_zone.id), CRUCIBLE_PORT)
+            }
+            OmicronZoneType::CruciblePantry { .. } => {
+                (ServiceName::CruciblePantry, CRUCIBLE_PANTRY_PORT)
+            }
+            OmicronZoneType::Oximeter { .. } => {
+                (ServiceName::Oximeter, OXIMETER_PORT)
+            }
+            OmicronZoneType::ExternalDns { .. } => {
+                (ServiceName::ExternalDns, DNS_HTTP_PORT)
+            }
+            OmicronZoneType::InternalDns { .. } => {
+                (ServiceName::InternalDns, DNS_HTTP_PORT)
+            }
+        };
+
+        // This unwrap is safe because this function only fails if we provide
+        // the same zone id twice, which should not be possible here.
+        dns_builder
+            .host_zone_with_one_backend(
+                omicron_zone.id,
+                omicron_zone.underlay_address,
+                service_name,
+                port,
+            )
+            .unwrap();
+    }
+
+    let scrimlets = sleds_by_id.values().filter(|sled| sled.is_scrimlet);
+    for scrimlet in scrimlets {
+        let sled_subnet = scrimlet.subnet();
+        let switch_zone_ip = get_switch_zone_address(sled_subnet);
+        // unwrap(): see above.
+        dns_builder
+            .host_zone_switch(
+                scrimlet.id,
+                switch_zone_ip,
+                DENDRITE_PORT,
+                MGS_PORT,
+                MGD_PORT,
+            )
+            .unwrap();
+    }
+
+    // We set the generation number for the internal DNS to be newer than
+    // whatever it was when this blueprint was generated.  This will only be
+    // used if the generated DNS contents are different from what's current.
+    dns_builder.generation(blueprint.internal_dns_version.next());
+    dns_builder.build()
+}
+
+fn dns_compute_update(
+    log: &slog::Logger,
+    comment: String,
+    creator: String,
+    current_config: &DnsConfigParams,
+    new_config: &DnsConfigParams,
+) -> Result<Option<DnsVersionUpdateBuilder>, Error> {
+    let mut update =
+        DnsVersionUpdateBuilder::new(DnsGroup::Internal, comment, creator);
+
+    let diff = DnsDiff::new(&current_config, &new_config)
+        .map_err(|e| Error::internal_error(&format!("{:#}", e)))?;
+    if diff.is_empty() {
+        info!(log, "no changes");
+        return Ok(None);
+    }
+
+    for (name, new_records) in diff.names_added() {
+        debug!(
+            log,
+            "adding name";
+            "name" => name,
+            "new_records" => ?new_records,
+        );
+        update.add_name(
+            name.to_string(),
+            new_records.into_iter().cloned().collect(),
+        )?;
+    }
+
+    for (name, old_records) in diff.names_removed() {
+        debug!(
+            log,
+            "removing name";
+            "name" => name,
+            "old_records" => ?old_records,
+        );
+        update.remove_name(name.to_string())?;
+    }
+
+    for (name, old_records, new_records) in diff.names_changed() {
+        debug!(
+            log,
+            "updating name";
+            "name" => name,
+            "old_records" => ?old_records,
+            "new_records" => ?new_records,
+        );
+        update.remove_name(name.to_string())?;
+        update.add_name(
+            name.to_string(),
+            new_records.into_iter().cloned().collect(),
+        )?;
+    }
+
+    Ok(Some(update))
+}
+
+#[cfg(test)]
+mod test {
+    use super::blueprint_dns_config;
+    use super::dns_compute_update;
+    use crate::Sled;
+    use internal_dns::ServiceName;
+    use internal_dns::DNS_ZONE;
+    use nexus_deployment::blueprint_builder::BlueprintBuilder;
+    use nexus_inventory::CollectionBuilder;
+    use nexus_types::deployment::Blueprint;
+    use nexus_types::deployment::OmicronZoneConfig;
+    use nexus_types::deployment::OmicronZoneType;
+    use nexus_types::deployment::Policy;
+    use nexus_types::deployment::SledResources;
+    use nexus_types::deployment::ZpoolName;
+    use nexus_types::external_api::views::SledProvisionState;
+    use nexus_types::internal_api::params::DnsConfigParams;
+    use nexus_types::internal_api::params::DnsConfigZone;
+    use nexus_types::internal_api::params::DnsRecord;
+    use omicron_common::address::get_sled_address;
+    use omicron_common::address::get_switch_zone_address;
+    use omicron_common::address::Ipv6Subnet;
+    use omicron_common::address::RACK_PREFIX;
+    use omicron_common::address::SLED_PREFIX;
+    use omicron_common::api::external::Generation;
+    use omicron_test_utils::dev::test_setup_log;
+    use std::collections::BTreeMap;
+    use std::collections::BTreeSet;
+    use std::collections::HashMap;
+    use std::net::Ipv4Addr;
+    use std::net::Ipv6Addr;
+    use std::net::SocketAddrV6;
+    use std::str::FromStr;
+    use uuid::Uuid;
+
+    fn blueprint_empty() -> Blueprint {
+        let builder = CollectionBuilder::new("test-suite");
+        let collection = builder.build();
+        let policy = Policy {
+            sleds: BTreeMap::new(),
+            service_ip_pool_ranges: vec![],
+            target_nexus_zone_count: 3,
+        };
+        BlueprintBuilder::build_initial_from_collection(
+            &collection,
+            Generation::new(),
+            &policy,
+            "test-suite",
+        )
+        .expect("failed to generate empty blueprint")
+    }
+
+    fn dns_config_empty() -> DnsConfigParams {
+        DnsConfigParams {
+            generation: 1,
+            time_created: chrono::Utc::now(),
+            zones: vec![DnsConfigZone {
+                zone_name: String::from("internal"),
+                records: HashMap::new(),
+            }],
+        }
+    }
+
+    /// test blueprint_dns_config(): trivial case of an empty blueprint
+    #[test]
+    fn test_blueprint_dns_empty() {
+        let blueprint = blueprint_empty();
+        let blueprint_dns = blueprint_dns_config(&blueprint, &BTreeMap::new());
+        assert!(blueprint_dns.sole_zone().unwrap().records.is_empty());
+    }
+
+    /// test blueprint_dns_config(): exercise various different conditions
+    /// - one of each type of zone in service
+    /// - some zones not in service
+    #[test]
+    fn test_blueprint_dns_basic() {
+        // We'll use the standard representative inventory collection to build a
+        // blueprint.  The main thing we care about here is that we have at
+        // least one zone of each type.  Later, we'll mark a couple of the sleds
+        // as Scrimlets to exercise that case.
+        let representative = nexus_inventory::examples::representative();
+        let collection = representative.builder.build();
+        let rack_subnet_base: Ipv6Addr =
+            "fd00:1122:3344:0100::".parse().unwrap();
+        let rack_subnet =
+            ipnet::Ipv6Net::new(rack_subnet_base, RACK_PREFIX).unwrap();
+        let possible_sled_subnets = rack_subnet.subnets(SLED_PREFIX).unwrap();
+        // Ignore sleds with no associated zones in the inventory.
+        // This is included in the "representative" collection, but it's
+        // not allowed by BlueprintBuilder::build_initial_from_collection().
+        let policy_sleds = collection
+            .omicron_zones
+            .keys()
+            .zip(possible_sled_subnets)
+            .map(|(sled_id, subnet)| {
+                let sled_resources = SledResources {
+                    provision_state: SledProvisionState::Provisionable,
+                    zpools: BTreeSet::from([ZpoolName::from_str(&format!(
+                        "oxp_{}",
+                        Uuid::new_v4()
+                    ))
+                    .unwrap()]),
+                    subnet: Ipv6Subnet::new(subnet.network()),
+                };
+                (*sled_id, sled_resources)
+            })
+            .collect();
+
+        let policy = Policy {
+            sleds: policy_sleds,
+            service_ip_pool_ranges: vec![],
+            target_nexus_zone_count: 3,
+        };
+        let dns_empty = dns_config_empty();
+        let initial_dns_generation =
+            Generation::from(u32::try_from(dns_empty.generation).unwrap());
+        let mut blueprint = BlueprintBuilder::build_initial_from_collection(
+            &collection,
+            initial_dns_generation,
+            &policy,
+            "test-suite",
+        )
+        .expect("failed to build initial blueprint");
+
+        // To make things slightly more interesting, let's add a zone that's
+        // not currently in service.
+        let out_of_service_id = Uuid::new_v4();
+        let out_of_service_addr = Ipv6Addr::LOCALHOST;
+        blueprint.omicron_zones.values_mut().next().unwrap().zones.push(
+            OmicronZoneConfig {
+                id: out_of_service_id,
+                underlay_address: out_of_service_addr,
+                zone_type: OmicronZoneType::Oximeter {
+                    address: SocketAddrV6::new(
+                        out_of_service_addr,
+                        12345,
+                        0,
+                        0,
+                    )
+                    .to_string(),
+                },
+            },
+        );
+
+        // To generate the blueprint's DNS config, we need to make up a
+        // different set of information about the sleds in our fake system.
+        let sleds_by_id = policy
+            .sleds
+            .iter()
+            .enumerate()
+            .map(|(i, (sled_id, sled_resources))| {
+                let sled_info = Sled {
+                    id: *sled_id,
+                    sled_agent_address: get_sled_address(sled_resources.subnet),
+                    // The first two of these (arbitrarily) will be marked
+                    // Scrimlets.
+                    is_scrimlet: i < 2,
+                };
+                (*sled_id, sled_info)
+            })
+            .collect();
+
+        let dns_config_blueprint =
+            blueprint_dns_config(&blueprint, &sleds_by_id);
+        assert_eq!(
+            dns_config_blueprint.generation,
+            u64::from(initial_dns_generation.next())
+        );
+        let blueprint_dns_zone = dns_config_blueprint.sole_zone().unwrap();
+        assert_eq!(blueprint_dns_zone.zone_name, DNS_ZONE);
+
+        // Now, verify a few different properties about the generated DNS
+        // configuration:
+        //
+        // 1. Every zone (except for the one that we added not-in-service)
+        //    should have some DNS name with a AAAA record that points at the
+        //    zone's underlay IP.  (i.e., every Omiron zone is _in_ DNS)
+        //
+        // 2. Every SRV record that we find should have a "target" that points
+        //    to another name within the DNS configuration, and that name should
+        //    be one of the ones with a AAAA record pointing to an Omicron zone.
+        //
+        // 3. There is at least one SRV record for each service that we expect
+        //    to appear in the representative system that we're working with.
+        //
+        // 4. Our out-of-service zone does *not* appear in the DNS config,
+        //    neither with an AAAA record nor in an SRV record.
+        //
+        // Together, this tells us that we have SRV records for all services,
+        // that those SRV records all point to at least one of the Omicron zones
+        // for that service, and that we correctly ignored zones that were not
+        // in service.
+
+        // To start, we need a mapping from underlay IP to the corresponding
+        // Omicron zone.
+        let mut omicron_zones_by_ip: BTreeMap<_, _> = blueprint
+            .all_omicron_zones()
+            .filter(|(_, zone)| zone.id != out_of_service_id)
+            .map(|(_, zone)| (zone.underlay_address, zone.id))
+            .collect();
+        println!("omicron zones by IP: {:#?}", omicron_zones_by_ip);
+
+        // We also want a mapping from underlay IP to the corresponding switch
+        // zone.  In this case, the value is the Scrimlet's sled id.
+        let mut switch_sleds_by_ip: BTreeMap<_, _> = sleds_by_id
+            .iter()
+            .filter_map(|(sled_id, sled)| {
+                if sled.is_scrimlet {
+                    let sled_subnet = policy.sleds.get(sled_id).unwrap().subnet;
+                    let switch_zone_ip = get_switch_zone_address(sled_subnet);
+                    Some((switch_zone_ip, *sled_id))
+                } else {
+                    None
+                }
+            })
+            .collect();
+
+        // Now go through all the DNS names that have AAAA records and remove
+        // any corresponding Omicron zone.  While doing this, construct a set of
+        // the fully-qualified DNS names (i.e., with the zone name suffix
+        // appended) that had AAAA records.  We'll use this later to make sure
+        // all the SRV records' targets that we find are valid.
+        let mut expected_srv_targets: BTreeSet<_> = BTreeSet::new();
+        for (name, records) in &blueprint_dns_zone.records {
+            let addrs: Vec<_> = records
+                .iter()
+                .filter_map(|dns_record| match dns_record {
+                    DnsRecord::Aaaa(addr) => Some(addr),
+                    _ => None,
+                })
+                .collect();
+            for addr in addrs {
+                if let Some(zone_id) = omicron_zones_by_ip.remove(addr) {
+                    println!(
+                        "IP {} found in DNS corresponds with zone {}",
+                        addr, zone_id
+                    );
+                    expected_srv_targets.insert(format!(
+                        "{}.{}",
+                        name, blueprint_dns_zone.zone_name
+                    ));
+                    continue;
+                }
+
+                if let Some(scrimlet_id) = switch_sleds_by_ip.remove(addr) {
+                    println!(
+                        "IP {} found in DNS corresponds with switch zone \
+                        for Scrimlet {}",
+                        addr, scrimlet_id
+                    );
+                    expected_srv_targets.insert(format!(
+                        "{}.{}",
+                        name, blueprint_dns_zone.zone_name
+                    ));
+                    continue;
+                }
+
+                println!(
+                    "note: found IP ({}) not corresponding to any \
+                    Omicron zone or switch zone (name {:?})",
+                    addr, name
+                );
+            }
+        }
+
+        println!(
+            "Omicron zones whose IPs were not found in DNS: {:?}",
+            omicron_zones_by_ip,
+        );
+        assert!(
+            omicron_zones_by_ip.is_empty(),
+            "some Omicron zones' IPs were not found in DNS"
+        );
+
+        println!(
+            "Scrimlets whose switch zone IPs were not found in DNS: {:?}",
+            switch_sleds_by_ip,
+        );
+        assert!(
+            switch_sleds_by_ip.is_empty(),
+            "some switch zones' IPs were not found in DNS"
+        );
+
+        // Now go through all DNS names that have SRV records.  For each one,
+        //
+        // 1. If its name corresponds to the name of one of the SRV services
+        //    that we expect the system to have, record that fact.  At the end
+        //    we'll verify that we found at least one SRV record for each such
+        //    service.
+        //
+        // 2. Make sure that the SRV record points at a name that we found in
+        //    the previous pass (i.e., that corresponds to an Omicron zone).
+        //
+        // There are some ServiceNames missing here because they are not part of
+        // our representative config (e.g., ClickhouseKeeper) or they don't
+        // currently have DNS record at all (e.g., SledAgent, Maghemite, Mgd,
+        // Tfport).
+        let mut srv_kinds_expected = BTreeSet::from([
+            ServiceName::Clickhouse,
+            ServiceName::Cockroach,
+            ServiceName::InternalDns,
+            ServiceName::ExternalDns,
+            ServiceName::Nexus,
+            ServiceName::Oximeter,
+            ServiceName::Dendrite,
+            ServiceName::CruciblePantry,
+            ServiceName::BoundaryNtp,
+            ServiceName::InternalNtp,
+        ]);
+
+        for (name, records) in &blueprint_dns_zone.records {
+            let srvs: Vec<_> = records
+                .iter()
+                .filter_map(|dns_record| match dns_record {
+                    DnsRecord::Srv(srv) => Some(srv),
+                    _ => None,
+                })
+                .collect();
+            for srv in srvs {
+                assert!(
+                    expected_srv_targets.contains(&srv.target),
+                    "found SRV record with target {:?} that does not \
+                    correspond to a name that points to any Omicron zone",
+                    srv.target
+                );
+            }
+
+            let kinds_left: Vec<_> =
+                srv_kinds_expected.iter().copied().collect();
+            for kind in kinds_left {
+                if kind.dns_name() == *name {
+                    srv_kinds_expected.remove(&kind);
+                }
+            }
+        }
+
+        println!("SRV kinds with no records found: {:?}", srv_kinds_expected);
+        assert!(srv_kinds_expected.is_empty());
+    }
+
+    #[test]
+    fn test_dns_compute_update() {
+        let logctx = test_setup_log("dns_compute_update");
+
+        // Start with an empty DNS config.  There's no database update needed
+        // when updating the DNS config to itself.
+        let dns_empty = dns_config_empty();
+        match dns_compute_update(
+            &logctx.log,
+            "test-suite".to_string(),
+            "test-suite".to_string(),
+            &dns_empty,
+            &dns_empty,
+        ) {
+            Ok(None) => (),
+            Err(error) => {
+                panic!("unexpected error generating update: {:?}", error)
+            }
+            Ok(Some(diff)) => panic!("unexpected delta: {:?}", diff),
+        };
+
+        // Now let's do something a little less trivial.  Set up two slightly
+        // different DNS configurations, compute the database update, and make
+        // sure it matches what we expect.
+        let dns_config1 = DnsConfigParams {
+            generation: 4,
+            time_created: chrono::Utc::now(),
+            zones: vec![DnsConfigZone {
+                zone_name: "my-zone".to_string(),
+                records: HashMap::from([
+                    (
+                        "ex1".to_string(),
+                        vec![DnsRecord::A(Ipv4Addr::LOCALHOST)],
+                    ),
+                    (
+                        "ex2".to_string(),
+                        vec![DnsRecord::A("192.168.1.3".parse().unwrap())],
+                    ),
+                ]),
+            }],
+        };
+
+        let dns_config2 = DnsConfigParams {
+            generation: 4,
+            time_created: chrono::Utc::now(),
+            zones: vec![DnsConfigZone {
+                zone_name: "my-zone".to_string(),
+                records: HashMap::from([
+                    (
+                        "ex2".to_string(),
+                        vec![DnsRecord::A("192.168.1.4".parse().unwrap())],
+                    ),
+                    (
+                        "ex3".to_string(),
+                        vec![DnsRecord::A(Ipv4Addr::LOCALHOST)],
+                    ),
+                ]),
+            }],
+        };
+
+        let update = dns_compute_update(
+            &logctx.log,
+            "test-suite".to_string(),
+            "test-suite".to_string(),
+            &dns_config1,
+            &dns_config2,
+        )
+        .expect("failed to compute update")
+        .expect("unexpectedly produced no update");
+
+        let mut removed: Vec<_> = update.names_removed().collect();
+        removed.sort();
+        assert_eq!(removed, vec!["ex1", "ex2"]);
+
+        let mut added: Vec<_> = update.names_added().collect();
+        added.sort_by_key(|n| n.0);
+        assert_eq!(
+            added,
+            vec![
+                (
+                    "ex2",
+                    [DnsRecord::A("192.168.1.4".parse().unwrap())].as_ref()
+                ),
+                ("ex3", [DnsRecord::A(Ipv4Addr::LOCALHOST)].as_ref()),
+            ]
+        );
+
+        logctx.cleanup_successful();
+    }
+}
diff --git a/nexus/blueprint-execution/src/lib.rs b/nexus/blueprint-execution/src/lib.rs
index f7bfd7d30c..a13acdf265 100644
--- a/nexus/blueprint-execution/src/lib.rs
+++ b/nexus/blueprint-execution/src/lib.rs
@@ -6,29 +6,87 @@
 //!
 //! See `nexus_deployment` crate-level docs for background.
 
+use anyhow::{anyhow, Context};
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db::DataStore;
 use nexus_types::deployment::Blueprint;
-use slog::o;
+use nexus_types::identity::Asset;
+use omicron_common::address::Ipv6Subnet;
+use omicron_common::address::SLED_PREFIX;
+use slog::info;
+use slog_error_chain::InlineErrorChain;
+use std::collections::BTreeMap;
+use std::net::SocketAddrV6;
+use uuid::Uuid;
 
+mod dns;
 mod omicron_zones;
 
+struct Sled {
+    id: Uuid,
+    sled_agent_address: SocketAddrV6,
+    is_scrimlet: bool,
+}
+
+impl Sled {
+    pub fn subnet(&self) -> Ipv6Subnet<SLED_PREFIX> {
+        Ipv6Subnet::<SLED_PREFIX>::new(*self.sled_agent_address.ip())
+    }
+}
+
+impl From<nexus_db_model::Sled> for Sled {
+    fn from(value: nexus_db_model::Sled) -> Self {
+        Sled {
+            id: value.id(),
+            sled_agent_address: value.address(),
+            is_scrimlet: value.is_scrimlet(),
+        }
+    }
+}
+
 /// Make one attempt to realize the given blueprint, meaning to take actions to
 /// alter the real system to match the blueprint
 ///
 /// The assumption is that callers are running this periodically or in a loop to
 /// deal with transient errors or changes in the underlying system state.
-pub async fn realize_blueprint(
+pub async fn realize_blueprint<S>(
     opctx: &OpContext,
     datastore: &DataStore,
     blueprint: &Blueprint,
-) -> Result<(), Vec<anyhow::Error>> {
-    let log = opctx.log.new(o!("comment" => blueprint.comment.clone()));
-    omicron_zones::deploy_zones(
-        &log,
-        opctx,
-        datastore,
-        &blueprint.omicron_zones,
+    nexus_label: S,
+) -> Result<(), Vec<anyhow::Error>>
+where
+    String: From<S>,
+{
+    let opctx = opctx.child(BTreeMap::from([(
+        "comment".to_string(),
+        blueprint.comment.clone(),
+    )]));
+
+    info!(
+        opctx.log,
+        "attempting to realize blueprint";
+        "blueprint_id" => ?blueprint.id
+    );
+
+    let sleds_by_id: BTreeMap<Uuid, _> = datastore
+        .sled_list_all_batched(&opctx)
+        .await
+        .context("listing all sleds")
+        .map_err(|e| vec![e])?
+        .into_iter()
+        .map(|db_sled| (db_sled.id(), Sled::from(db_sled)))
+        .collect();
+    omicron_zones::deploy_zones(&opctx, &sleds_by_id, &blueprint.omicron_zones)
+        .await?;
+
+    dns::deploy_dns(
+        &opctx,
+        &datastore,
+        String::from(nexus_label),
+        &blueprint,
+        &sleds_by_id,
     )
     .await
+    .map_err(|e| vec![anyhow!("{}", InlineErrorChain::new(&e))])
 }
diff --git a/nexus/blueprint-execution/src/omicron_zones.rs b/nexus/blueprint-execution/src/omicron_zones.rs
index f3e81d283d..1d5c4444b1 100644
--- a/nexus/blueprint-execution/src/omicron_zones.rs
+++ b/nexus/blueprint-execution/src/omicron_zones.rs
@@ -4,50 +4,49 @@
 
 //! Manges deployment of Omicron zones to Sled Agents
 
+use crate::Sled;
+use anyhow::anyhow;
 use anyhow::Context;
 use futures::stream;
 use futures::StreamExt;
 use nexus_db_queries::context::OpContext;
-use nexus_db_queries::db::lookup::LookupPath;
-use nexus_db_queries::db::DataStore;
 use nexus_types::deployment::OmicronZonesConfig;
 use sled_agent_client::Client as SledAgentClient;
 use slog::info;
 use slog::warn;
-use slog::Logger;
 use std::collections::BTreeMap;
 use uuid::Uuid;
 
 /// Idempotently ensure that the specified Omicron zones are deployed to the
 /// corresponding sleds
 pub(crate) async fn deploy_zones(
-    log: &Logger,
     opctx: &OpContext,
-    datastore: &DataStore,
+    sleds_by_id: &BTreeMap<Uuid, Sled>,
     zones: &BTreeMap<Uuid, OmicronZonesConfig>,
 ) -> Result<(), Vec<anyhow::Error>> {
     let errors: Vec<_> = stream::iter(zones)
         .filter_map(|(sled_id, config)| async move {
-            let client = match sled_client(opctx, datastore, *sled_id).await {
-                Ok(client) => client,
-                Err(err) => {
-                    warn!(log, "{err:#}");
+            let db_sled = match sleds_by_id.get(sled_id) {
+                Some(sled) => sled,
+                None => {
+                    let err = anyhow!("sled not found in db list: {}", sled_id);
+                    warn!(opctx.log, "{err:#}");
                     return Some(err);
                 }
             };
+            let client = sled_client(opctx, &db_sled);
             let result =
                 client.omicron_zones_put(&config).await.with_context(|| {
                     format!("Failed to put {config:#?} to sled {sled_id}")
                 });
-
             match result {
                 Err(error) => {
-                    warn!(log, "{error:#}");
+                    warn!(opctx.log, "{error:#}");
                     Some(error)
                 }
                 Ok(_) => {
                     info!(
-                        log,
+                        opctx.log,
                         "Successfully deployed zones for sled agent";
                         "sled_id" => %sled_id,
                         "generation" => %config.generation,
@@ -71,43 +70,27 @@ pub(crate) async fn deploy_zones(
 // method on the `Nexus` type. We want to have a more constrained type we can
 // pass into background tasks for this type of functionality, but for now we
 // just copy the functionality.
-async fn sled_client(
-    opctx: &OpContext,
-    datastore: &DataStore,
-    sled_id: Uuid,
-) -> Result<SledAgentClient, anyhow::Error> {
-    let (.., sled) = LookupPath::new(opctx, datastore)
-        .sled_id(sled_id)
-        .fetch()
-        .await
-        .with_context(|| {
-            format!(
-                "Failed to create sled_agent::Client for sled_id: {}",
-                sled_id
-            )
-        })?;
+fn sled_client(opctx: &OpContext, sled: &Sled) -> SledAgentClient {
     let dur = std::time::Duration::from_secs(60);
     let client = reqwest::ClientBuilder::new()
         .connect_timeout(dur)
         .timeout(dur)
         .build()
         .unwrap();
-    Ok(SledAgentClient::new_with_client(
-        &format!("http://{}", sled.address()),
+    SledAgentClient::new_with_client(
+        &format!("http://{}", sled.sled_agent_address),
         client,
         opctx.log.clone(),
-    ))
+    )
 }
 
 #[cfg(test)]
 mod test {
     use super::deploy_zones;
+    use crate::Sled;
     use httptest::matchers::{all_of, json_decoded, request};
     use httptest::responders::status_code;
     use httptest::Expectation;
-    use nexus_db_model::{
-        ByteCount, SledBaseboard, SledSystemHardware, SledUpdate,
-    };
     use nexus_db_queries::context::OpContext;
     use nexus_test_utils_macros::nexus_test;
     use nexus_types::deployment::OmicronZonesConfig;
@@ -140,6 +123,7 @@ mod test {
                 omicron_zones,
                 zones_in_service: BTreeSet::new(),
                 parent_blueprint_id: None,
+                internal_dns_version: Generation::new(),
                 time_created: chrono::Utc::now(),
                 creator: "test".to_string(),
                 comment: "test blueprint".to_string(),
@@ -155,14 +139,6 @@ mod test {
             cptestctx.logctx.log.clone(),
             datastore.clone(),
         );
-        let log = opctx.log.clone();
-
-        // Get a success result back when the blueprint has an empty set of
-        // zones.
-        let blueprint = Arc::new(create_blueprint(BTreeMap::new()));
-        deploy_zones(&log, &opctx, &datastore, &blueprint.1.omicron_zones)
-            .await
-            .expect("failed to deploy no zones");
 
         // Create some fake sled-agent servers to respond to zone puts and add
         // sleds to CRDB.
@@ -170,67 +146,61 @@ mod test {
         let mut s2 = httptest::Server::run();
         let sled_id1 = Uuid::new_v4();
         let sled_id2 = Uuid::new_v4();
-        let rack_id = Uuid::new_v4();
-        for (i, (sled_id, server)) in
-            [(sled_id1, &s1), (sled_id2, &s2)].iter().enumerate()
-        {
-            let SocketAddr::V6(addr) = server.addr() else {
-                panic!("Expected Ipv6 address. Got {}", server.addr());
-            };
-            let update = SledUpdate::new(
-                *sled_id,
-                addr,
-                SledBaseboard {
-                    serial_number: i.to_string(),
-                    part_number: "test".into(),
-                    revision: 1,
-                },
-                SledSystemHardware {
-                    is_scrimlet: false,
-                    usable_hardware_threads: 4,
-                    usable_physical_ram: ByteCount(1000.into()),
-                    reservoir_size: ByteCount(999.into()),
-                },
-                rack_id,
-            );
-            datastore
-                .sled_upsert(update)
-                .await
-                .expect("Failed to insert sled to db");
-        }
-
-        // The particular dataset doesn't matter for this test.
-        // We re-use the same one to not obfuscate things
-        let dataset = OmicronZoneDataset {
-            pool_name: format!("oxp_{}", Uuid::new_v4()).parse().unwrap(),
-        };
+        let sleds_by_id: BTreeMap<Uuid, Sled> =
+            [(sled_id1, &s1), (sled_id2, &s2)]
+                .into_iter()
+                .map(|(sled_id, server)| {
+                    let SocketAddr::V6(addr) = server.addr() else {
+                        panic!("Expected Ipv6 address. Got {}", server.addr());
+                    };
+                    let sled = Sled {
+                        id: sled_id,
+                        sled_agent_address: addr,
+                        is_scrimlet: false,
+                    };
+                    (sled_id, sled)
+                })
+                .collect();
 
-        let generation = Generation::new();
+        // Get a success result back when the blueprint has an empty set of
+        // zones.
+        let blueprint = Arc::new(create_blueprint(BTreeMap::new()));
+        deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones)
+            .await
+            .expect("failed to deploy no zones");
 
         // Zones are updated in a particular order, but each request contains
         // the full set of zones that must be running.
         // See `rack_setup::service::ServiceInner::run` for more details.
-        let mut zones = OmicronZonesConfig {
-            generation,
-            zones: vec![OmicronZoneConfig {
-                id: Uuid::new_v4(),
-                underlay_address: "::1".parse().unwrap(),
-                zone_type: OmicronZoneType::InternalDns {
-                    dataset,
-                    dns_address: "oh-hello-internal-dns".into(),
-                    gz_address: "::1".parse().unwrap(),
-                    gz_address_index: 0,
-                    http_address: "some-ipv6-address".into(),
-                },
-            }],
-        };
+        fn make_zones() -> OmicronZonesConfig {
+            OmicronZonesConfig {
+                generation: Generation::new(),
+                zones: vec![OmicronZoneConfig {
+                    id: Uuid::new_v4(),
+                    underlay_address: "::1".parse().unwrap(),
+                    zone_type: OmicronZoneType::InternalDns {
+                        dataset: OmicronZoneDataset {
+                            pool_name: format!("oxp_{}", Uuid::new_v4())
+                                .parse()
+                                .unwrap(),
+                        },
+                        dns_address: "oh-hello-internal-dns".into(),
+                        gz_address: "::1".parse().unwrap(),
+                        gz_address_index: 0,
+                        http_address: "some-ipv6-address".into(),
+                    },
+                }],
+            }
+        }
 
         // Create a blueprint with only the `InternalDns` zone for both servers
         // We reuse the same `OmicronZonesConfig` because the details don't
         // matter for this test.
+        let mut zones1 = make_zones();
+        let mut zones2 = make_zones();
         let blueprint = Arc::new(create_blueprint(BTreeMap::from([
-            (sled_id1, zones.clone()),
-            (sled_id2, zones.clone()),
+            (sled_id1, zones1.clone()),
+            (sled_id2, zones2.clone()),
         ])));
 
         // Set expectations for the initial requests sent to the fake
@@ -250,7 +220,7 @@ mod test {
         }
 
         // Execute it.
-        deploy_zones(&log, &opctx, &datastore, &blueprint.1.omicron_zones)
+        deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones)
             .await
             .expect("failed to deploy initial zones");
 
@@ -267,7 +237,7 @@ mod test {
                 .respond_with(status_code(204)),
             );
         }
-        deploy_zones(&log, &opctx, &datastore, &blueprint.1.omicron_zones)
+        deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones)
             .await
             .expect("failed to deploy same zones");
         s1.verify_and_clear();
@@ -291,7 +261,7 @@ mod test {
         );
 
         let errors =
-            deploy_zones(&log, &opctx, &datastore, &blueprint.1.omicron_zones)
+            deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones)
                 .await
                 .expect_err("unexpectedly succeeded in deploying zones");
 
@@ -304,21 +274,25 @@ mod test {
         s2.verify_and_clear();
 
         // Add an `InternalNtp` zone for our next update
-        zones.generation = generation.next();
-        zones.zones.push(OmicronZoneConfig {
-            id: Uuid::new_v4(),
-            underlay_address: "::1".parse().unwrap(),
-            zone_type: OmicronZoneType::InternalNtp {
-                address: "::1".into(),
-                dns_servers: vec!["::1".parse().unwrap()],
-                domain: None,
-                ntp_servers: vec!["some-ntp-server-addr".into()],
-            },
-        });
+        fn append_zone(zones: &mut OmicronZonesConfig) {
+            zones.generation = zones.generation.next();
+            zones.zones.push(OmicronZoneConfig {
+                id: Uuid::new_v4(),
+                underlay_address: "::1".parse().unwrap(),
+                zone_type: OmicronZoneType::InternalNtp {
+                    address: "::1".into(),
+                    dns_servers: vec!["::1".parse().unwrap()],
+                    domain: None,
+                    ntp_servers: vec!["some-ntp-server-addr".into()],
+                },
+            });
+        }
 
+        append_zone(&mut zones1);
+        append_zone(&mut zones2);
         let blueprint = Arc::new(create_blueprint(BTreeMap::from([
-            (sled_id1, zones.clone()),
-            (sled_id2, zones.clone()),
+            (sled_id1, zones1),
+            (sled_id2, zones2),
         ])));
 
         // Set our new expectations
@@ -337,7 +311,7 @@ mod test {
         }
 
         // Activate the task
-        deploy_zones(&log, &opctx, &datastore, &blueprint.1.omicron_zones)
+        deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones)
             .await
             .expect("failed to deploy last round of zones");
         s1.verify_and_clear();
diff --git a/nexus/db-model/src/deployment.rs b/nexus/db-model/src/deployment.rs
index 34fe08d78c..a1f285fbef 100644
--- a/nexus/db-model/src/deployment.rs
+++ b/nexus/db-model/src/deployment.rs
@@ -23,6 +23,7 @@ use uuid::Uuid;
 pub struct Blueprint {
     pub id: Uuid,
     pub parent_blueprint_id: Option<Uuid>,
+    pub internal_dns_version: Generation,
     pub time_created: DateTime<Utc>,
     pub creator: String,
     pub comment: String,
@@ -33,6 +34,7 @@ impl From<&'_ nexus_types::deployment::Blueprint> for Blueprint {
         Self {
             id: bp.id,
             parent_blueprint_id: bp.parent_blueprint_id,
+            internal_dns_version: Generation(bp.internal_dns_version),
             time_created: bp.time_created,
             creator: bp.creator.clone(),
             comment: bp.comment.clone(),
@@ -45,6 +47,7 @@ impl From<Blueprint> for nexus_types::deployment::BlueprintMetadata {
         Self {
             id: value.id,
             parent_blueprint_id: value.parent_blueprint_id,
+            internal_dns_version: *value.internal_dns_version,
             time_created: value.time_created,
             creator: value.creator,
             comment: value.comment,
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index 3d9050b1af..54755486e5 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(35, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(36, 0, 0);
 
 table! {
     disk (id) {
@@ -1421,6 +1421,8 @@ table! {
         time_created -> Timestamptz,
         creator -> Text,
         comment -> Text,
+
+        internal_dns_version -> Int8,
     }
 }
 
diff --git a/nexus/db-queries/src/context.rs b/nexus/db-queries/src/context.rs
index 835256fc58..72da01a5f1 100644
--- a/nexus/db-queries/src/context.rs
+++ b/nexus/db-queries/src/context.rs
@@ -257,6 +257,65 @@ impl OpContext {
         );
         result
     }
+
+    /// Returns an error if we're currently in a context where expensive or
+    /// complex operations should not be allowed
+    ///
+    /// This is intended for checking that we're not trying to perform expensive
+    /// or complex (multi-step) operations from HTTP request handlers.
+    /// Generally, expensive or complex operations should be broken up into
+    /// multiple requests (e.g., pagination).  That's for a variety reasons:
+    ///
+    /// - DoS mitigation: requests that kick off an arbitrarily-large amount of
+    ///   work can tie up server resources without requiring commensurate
+    ///   resources from the client, which makes it very easy to attack the
+    ///   system
+    ///
+    /// - monitoring: it's easier to reason about metrics for operations that
+    ///   are roughly bounded in size (otherwise, things like "requests per
+    ///   second" can become meaningless)
+    ///
+    /// - stability: very large database queries have outsize effects on the
+    ///   rest of the system (potentially blocking other clients for extended
+    ///   periods of time, or preventing the database from timely cleanup of
+    ///   invalidated data, etc.)
+    ///
+    /// - throttling: separate requests gives us an opportunity to dynamically
+    ///   throttle clients that are hitting the system hard
+    ///
+    /// - failure transparency: when one request kicks off a complex
+    ///   (multi-step) operation, it's harder to communicate programmatically
+    ///   why the request failed
+    ///
+    /// - retries: when failures happen during smaller operations, clients can
+    ///   retry only the part that failed.  When failures happen during complex
+    ///   (multi-step) operations, the client has to retry the whole thing.
+    ///   This is much worse than it sounds: it means that for the request to
+    ///   succeed, _all_ of the suboperations have to succeed.  During a period
+    ///   of transient failures, that could be extremely unlikely.  With smaller
+    ///   requests, clients can just retry each one until it succeeds without
+    ///   having to retry the requests that already succeeded (whose failures
+    ///   would trigger another attempt at the whole thing).
+    ///
+    /// - Simple request-response HTTP is not well-suited to long-running
+    ///   operations.  There's no way to communicate progress or even that the
+    ///   request is still going.  There's no good way to cancel such a request,
+    ///   either.  Clients and proxies often don't expect long requests and
+    ///   apply aggressive timeouts.  Depending on the HTTP version, a
+    ///   long-running request can tie up the TCP connection.
+    pub fn check_complex_operations_allowed(&self) -> Result<(), Error> {
+        let api_handler = match self.kind {
+            OpKind::ExternalApiRequest | OpKind::InternalApiRequest => true,
+            OpKind::Saga | OpKind::Background | OpKind::Test => false,
+        };
+        if api_handler {
+            Err(Error::internal_error(
+                "operation not allowed from API handlers",
+            ))
+        } else {
+            Ok(())
+        }
+    }
 }
 
 impl Session for ConsoleSessionWithSiloId {
diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs
index cde6e7e8c6..d9df143022 100644
--- a/nexus/db-queries/src/db/datastore/deployment.rs
+++ b/nexus/db-queries/src/db/datastore/deployment.rs
@@ -233,7 +233,13 @@ impl DataStore {
 
         // Read the metadata from the primary blueprint row, and ensure that it
         // exists.
-        let (parent_blueprint_id, time_created, creator, comment) = {
+        let (
+            parent_blueprint_id,
+            internal_dns_version,
+            time_created,
+            creator,
+            comment,
+        ) = {
             use db::schema::blueprint::dsl;
 
             let Some(blueprint) = dsl::blueprint
@@ -251,6 +257,7 @@ impl DataStore {
 
             (
                 blueprint.parent_blueprint_id,
+                *blueprint.internal_dns_version,
                 blueprint.time_created,
                 blueprint.creator,
                 blueprint.comment,
@@ -479,6 +486,7 @@ impl DataStore {
             omicron_zones,
             zones_in_service,
             parent_blueprint_id,
+            internal_dns_version,
             time_created,
             creator,
             comment,
@@ -1056,6 +1064,7 @@ mod tests {
     use nexus_types::external_api::views::SledProvisionState;
     use nexus_types::inventory::Collection;
     use omicron_common::address::Ipv6Subnet;
+    use omicron_common::api::external::Generation;
     use omicron_test_utils::dev;
     use rand::thread_rng;
     use rand::Rng;
@@ -1174,6 +1183,7 @@ mod tests {
         let policy = policy_from_collection(&collection);
         let blueprint = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            Generation::new(),
             &policy,
             "test",
         )
@@ -1207,6 +1217,7 @@ mod tests {
             nexus_inventory::CollectionBuilder::new("test").build();
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            Generation::new(),
             &EMPTY_POLICY,
             "test",
         )
@@ -1332,10 +1343,16 @@ mod tests {
         policy.sleds.insert(new_sled_id, fake_sled_resources(None));
         let new_sled_zpools = &policy.sleds.get(&new_sled_id).unwrap().zpools;
 
-        // Create a builder for a child blueprint.
-        let mut builder =
-            BlueprintBuilder::new_based_on(&blueprint1, &policy, "test")
-                .expect("failed to create builder");
+        // Create a builder for a child blueprint.  While we're at it, use a
+        // different DNS version to test that that works.
+        let new_dns_version = blueprint1.internal_dns_version.next();
+        let mut builder = BlueprintBuilder::new_based_on(
+            &blueprint1,
+            new_dns_version,
+            &policy,
+            "test",
+        )
+        .expect("failed to create builder");
 
         // Add zones to our new sled.
         assert_eq!(
@@ -1381,8 +1398,9 @@ mod tests {
             .blueprint_read(&opctx, &authz_blueprint2)
             .await
             .expect("failed to read collection back");
-        println!("diff: {}", blueprint2.diff(&blueprint_read));
+        println!("diff: {}", blueprint2.diff_sleds(&blueprint_read));
         assert_eq!(blueprint2, blueprint_read);
+        assert_eq!(blueprint2.internal_dns_version, new_dns_version);
         {
             let mut expected_ids = [blueprint1.id, blueprint2.id];
             expected_ids.sort();
@@ -1474,18 +1492,27 @@ mod tests {
             nexus_inventory::CollectionBuilder::new("test").build();
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            Generation::new(),
             &EMPTY_POLICY,
             "test1",
         )
         .unwrap();
-        let blueprint2 =
-            BlueprintBuilder::new_based_on(&blueprint1, &EMPTY_POLICY, "test2")
-                .expect("failed to create builder")
-                .build();
-        let blueprint3 =
-            BlueprintBuilder::new_based_on(&blueprint1, &EMPTY_POLICY, "test3")
-                .expect("failed to create builder")
-                .build();
+        let blueprint2 = BlueprintBuilder::new_based_on(
+            &blueprint1,
+            Generation::new(),
+            &EMPTY_POLICY,
+            "test2",
+        )
+        .expect("failed to create builder")
+        .build();
+        let blueprint3 = BlueprintBuilder::new_based_on(
+            &blueprint1,
+            Generation::new(),
+            &EMPTY_POLICY,
+            "test3",
+        )
+        .expect("failed to create builder")
+        .build();
         assert_eq!(blueprint1.parent_blueprint_id, None);
         assert_eq!(blueprint2.parent_blueprint_id, Some(blueprint1.id));
         assert_eq!(blueprint3.parent_blueprint_id, Some(blueprint1.id));
@@ -1574,10 +1601,14 @@ mod tests {
 
         // Create a child of blueprint3, and ensure when we set it as the target
         // with enabled=false, that status is serialized.
-        let blueprint4 =
-            BlueprintBuilder::new_based_on(&blueprint3, &EMPTY_POLICY, "test3")
-                .expect("failed to create builder")
-                .build();
+        let blueprint4 = BlueprintBuilder::new_based_on(
+            &blueprint3,
+            Generation::new(),
+            &EMPTY_POLICY,
+            "test3",
+        )
+        .expect("failed to create builder")
+        .build();
         assert_eq!(blueprint4.parent_blueprint_id, Some(blueprint3.id));
         datastore.blueprint_insert(&opctx, &blueprint4).await.unwrap();
         let bp4_target = BlueprintTarget {
diff --git a/nexus/db-queries/src/db/datastore/dns.rs b/nexus/db-queries/src/db/datastore/dns.rs
index 552ad31487..180764d38c 100644
--- a/nexus/db-queries/src/db/datastore/dns.rs
+++ b/nexus/db-queries/src/db/datastore/dns.rs
@@ -346,8 +346,77 @@ impl DataStore {
         Ok(())
     }
 
+    /// Update the configuration of a DNS zone as specified in `update`,
+    /// conditional on the _current_ DNS version being `old_version`.
+    ///
+    /// Unlike `dns_update_incremental()`, this function assumes the caller has
+    /// already constructed `update` based on a specific DNS version
+    /// (`old_version`) and only wants to apply these changes if the DNS version
+    /// in the database has not changed.
+    ///
+    /// Also unlike `dns_update_incremental()`, this function creates its own
+    /// transaction to apply the update.
+    ///
+    /// Like `dns_update_incremental()`, **callers almost certainly want to wake
+    /// up the corresponding Nexus background task to cause these changes to be
+    /// propagated to the corresponding DNS servers.**
+    pub async fn dns_update_from_version(
+        &self,
+        opctx: &OpContext,
+        update: DnsVersionUpdateBuilder,
+        old_version: Generation,
+    ) -> Result<(), Error> {
+        opctx.authorize(authz::Action::Modify, &authz::DNS_CONFIG).await?;
+        let conn = self.pool_connection_authorized(opctx).await?;
+        conn.transaction_async(|c| async move {
+            let zones = self
+                .dns_zones_list_all_on_connection(opctx, &c, update.dns_group)
+                .await?;
+            // This looks like a time-of-check-to-time-of-use race, but this
+            // approach works because we're inside a transaction and the
+            // isolation level is SERIALIZABLE.
+            let version = self
+                .dns_group_latest_version_conn(opctx, &c, update.dns_group)
+                .await?;
+            if version.version != old_version {
+                return Err(TransactionError::CustomError(Error::conflict(
+                    format!(
+                        "expected current DNS version to be {}, found {}",
+                        *old_version, *version.version,
+                    ),
+                )));
+            }
+
+            self.dns_write_version_internal(
+                &c,
+                update,
+                zones,
+                Generation(old_version.next()),
+            )
+            .await
+        })
+        .await
+        .map_err(|e| match e {
+            TransactionError::CustomError(e) => e,
+            TransactionError::Database(e) => {
+                public_error_from_diesel(e, ErrorHandler::Server)
+            }
+        })
+    }
+
     /// Update the configuration of a DNS zone as specified in `update`
     ///
+    /// Unlike `dns_update_from_version()`, this function assumes that the
+    /// caller's changes are valid regardless of the current DNS configuration.
+    /// This function fetches the latest version and always writes the updated
+    /// config as the next version.  This is appropriate if the caller is adding
+    /// something wholly new or removing something that it knows should be
+    /// present (as in the case when we add or remove DNS names for a Silo,
+    /// since we control exactly when that happens).  This is _not_ appropriate
+    /// if the caller is making arbitrary changes that might conflict with a
+    /// concurrent caller.  For that, you probably want
+    /// `dns_update_from_version()`.
+    ///
     /// This function runs the body inside a transaction (if no transaction is
     /// open) or a nested transaction (savepoint, if a transaction is already
     /// open).  Generally, the caller should invoke this function while already
@@ -358,14 +427,14 @@ impl DataStore {
     ///
     /// It's recommended to put this step last in any transaction because the
     /// more time elapses between running this function and attempting to commit
-    /// the transaction, the greater the change of either transaction failure
+    /// the transaction, the greater the chance of either transaction failure
     /// due to a conflict error (if some other caller attempts to update the
     /// same DNS group) or another client blocking (for the same reason).
     ///
     /// **Callers almost certainly want to wake up the corresponding Nexus
     /// background task to cause these changes to be propagated to the
     /// corresponding DNS servers.**
-    pub async fn dns_update(
+    pub async fn dns_update_incremental(
         &self,
         opctx: &OpContext,
         conn: &async_bb8_diesel::Connection<DbConnection>,
@@ -378,19 +447,32 @@ impl DataStore {
             .await?;
 
         conn.transaction_async(|c| async move {
-            self.dns_update_internal(opctx, &c, update, zones).await
+            let version = self
+                .dns_group_latest_version_conn(opctx, conn, update.dns_group)
+                .await?;
+            self.dns_write_version_internal(
+                &c,
+                update,
+                zones,
+                Generation(version.version.next()),
+            )
+            .await
         })
         .await
     }
 
     // This must only be used inside a transaction.  Otherwise, it may make
-    // invalid changes to the database state.  Use `dns_update()` instead.
-    async fn dns_update_internal(
+    // invalid changes to the database state.  Use one of the `dns_update_*()`
+    // functions instead.
+    //
+    // The caller should already have checked (in the same transaction) that
+    // their version number is the correct next version.
+    async fn dns_write_version_internal(
         &self,
-        opctx: &OpContext,
         conn: &async_bb8_diesel::Connection<DbConnection>,
         update: DnsVersionUpdateBuilder,
         zones: Vec<DnsZone>,
+        new_version_num: Generation,
     ) -> Result<(), TransactionError<Error>> {
         // TODO-scalability TODO-performance This would be much better as a CTE
         // for all the usual reasons described in RFD 192.  Using an interactive
@@ -401,11 +483,6 @@ impl DataStore {
         // operations fail spuriously as far as the client is concerned).  We
         // expect these problems to be small or unlikely at small scale but
         // significant as the system scales up.
-        let dns_group = update.dns_group;
-        let version =
-            self.dns_group_latest_version_conn(opctx, conn, dns_group).await?;
-        let new_version_num =
-            nexus_db_model::Generation(version.version.next());
         let new_version = DnsVersion {
             dns_group: update.dns_group,
             version: new_version_num,
@@ -498,15 +575,16 @@ impl DataStore {
 /// and add the name back with different records.
 ///
 /// You use this object to build up a _description_ of the changes to the DNS
-/// zone's configuration.  Then you call [`DataStore::dns_update()`] to apply
-/// these changes transactionally to the database.  The changes are then
-/// propagated asynchronously to the DNS servers.  No changes are made (to
-/// either the database or the DNS servers) while you modify this object.
+/// zone's configuration.  Then you call [`DataStore::dns_update_incremental()`]
+/// or [`DataStore::dns_update_from_version()`] to apply these changes
+/// transactionally to the database.  The changes are then propagated
+/// asynchronously to the DNS servers.  No changes are made (to either the
+/// database or the DNS servers) while you modify this object.
 ///
 /// This object changes all of the zones associated with a particular DNS group
 /// because the assumption right now is that they're equivalent.  (In practice,
 /// we should only ever have one zone in each group right now.)
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct DnsVersionUpdateBuilder {
     dns_group: DnsGroup,
     comment: String,
@@ -596,6 +674,16 @@ impl DnsVersionUpdateBuilder {
             Ok(())
         }
     }
+
+    pub fn names_removed(&self) -> impl Iterator<Item = &str> {
+        self.names_removed.iter().map(AsRef::as_ref)
+    }
+
+    pub fn names_added(&self) -> impl Iterator<Item = (&str, &[DnsRecord])> {
+        self.names_added
+            .iter()
+            .map(|(name, list)| (name.as_ref(), list.as_ref()))
+    }
 }
 
 #[cfg(test)]
@@ -1360,8 +1448,8 @@ mod test {
     }
 
     #[tokio::test]
-    async fn test_dns_update() {
-        let logctx = dev::test_setup_log("test_dns_update");
+    async fn test_dns_update_incremental() {
+        let logctx = dev::test_setup_log("test_dns_update_incremental");
         let mut db = test_setup_database(&logctx.log).await;
         let (opctx, datastore) = datastore_test(&logctx, &db).await;
         let now = Utc::now();
@@ -1457,7 +1545,10 @@ mod test {
             update.add_name(String::from("n2"), records2.clone()).unwrap();
 
             let conn = datastore.pool_connection_for_tests().await.unwrap();
-            datastore.dns_update(&opctx, &conn, update).await.unwrap();
+            datastore
+                .dns_update_incremental(&opctx, &conn, update)
+                .await
+                .unwrap();
         }
 
         // Verify the new config.
@@ -1490,7 +1581,10 @@ mod test {
             update.add_name(String::from("n1"), records12.clone()).unwrap();
 
             let conn = datastore.pool_connection_for_tests().await.unwrap();
-            datastore.dns_update(&opctx, &conn, update).await.unwrap();
+            datastore
+                .dns_update_incremental(&opctx, &conn, update)
+                .await
+                .unwrap();
         }
 
         let dns_config = datastore
@@ -1520,7 +1614,10 @@ mod test {
             update.remove_name(String::from("n1")).unwrap();
 
             let conn = datastore.pool_connection_for_tests().await.unwrap();
-            datastore.dns_update(&opctx, &conn, update).await.unwrap();
+            datastore
+                .dns_update_incremental(&opctx, &conn, update)
+                .await
+                .unwrap();
         }
 
         let dns_config = datastore
@@ -1547,7 +1644,10 @@ mod test {
             update.add_name(String::from("n1"), records2.clone()).unwrap();
 
             let conn = datastore.pool_connection_for_tests().await.unwrap();
-            datastore.dns_update(&opctx, &conn, update).await.unwrap();
+            datastore
+                .dns_update_incremental(&opctx, &conn, update)
+                .await
+                .unwrap();
         }
 
         let dns_config = datastore
@@ -1586,7 +1686,9 @@ mod test {
             let copctx = opctx.child(std::collections::BTreeMap::new());
             let mut fut = conn1
                 .transaction_async(|c1| async move {
-                    cds.dns_update(&copctx, &c1, update1).await.unwrap();
+                    cds.dns_update_incremental(&copctx, &c1, update1)
+                        .await
+                        .unwrap();
                     // Let the outside scope know we've done the update, but we
                     // haven't committed the transaction yet.  Wait for them to
                     // tell us to proceed.
@@ -1613,7 +1715,10 @@ mod test {
                 String::from("the test suite"),
             );
             update2.add_name(String::from("n1"), records1.clone()).unwrap();
-            datastore.dns_update(&opctx, &conn2, update2).await.unwrap();
+            datastore
+                .dns_update_incremental(&opctx, &conn2, update2)
+                .await
+                .unwrap();
 
             // Now let the first one finish.
             wait2_tx.send(()).unwrap();
@@ -1657,8 +1762,10 @@ mod test {
             update.remove_name(String::from("n4")).unwrap();
 
             let conn = datastore.pool_connection_for_tests().await.unwrap();
-            let error =
-                datastore.dns_update(&opctx, &conn, update).await.unwrap_err();
+            let error = datastore
+                .dns_update_incremental(&opctx, &conn, update)
+                .await
+                .unwrap_err();
             let error = match error {
                 TransactionError::CustomError(err) => err,
                 _ => panic!("Unexpected error: {:?}", error),
@@ -1687,7 +1794,10 @@ mod test {
 
             let conn = datastore.pool_connection_for_tests().await.unwrap();
             let error = Error::from(
-                datastore.dns_update(&opctx, &conn, update).await.unwrap_err(),
+                datastore
+                    .dns_update_incremental(&opctx, &conn, update)
+                    .await
+                    .unwrap_err(),
             );
             let msg = error.to_string();
             assert!(msg.starts_with("Internal Error: "), "Message: {msg:}");
@@ -1714,4 +1824,117 @@ mod test {
         db.cleanup().await.unwrap();
         logctx.cleanup_successful();
     }
+
+    #[tokio::test]
+    async fn test_dns_update_from_version() {
+        let logctx = dev::test_setup_log("test_dns_update_from_version");
+        let mut db = test_setup_database(&logctx.log).await;
+        let (opctx, datastore) = datastore_test(&logctx, &db).await;
+
+        // The guts of `dns_update_from_version()` are shared with
+        // `dns_update_incremental()`.  The main cases worth testing here are
+        // (1) quick check that the happy path works, plus (2) make sure it
+        // fails when the precondition fails (current version doesn't match
+        // what's expected).
+        //
+        // Start by loading some initial data.
+        let conn = datastore.pool_connection_for_tests().await.unwrap();
+        let initial_data = InitialDnsGroup::new(
+            DnsGroup::Internal,
+            "my-zone",
+            "test-suite",
+            "test-suite",
+            HashMap::from([
+                (
+                    "wendell".to_string(),
+                    vec![DnsRecord::Aaaa(Ipv6Addr::LOCALHOST)],
+                ),
+                (
+                    "krabappel".to_string(),
+                    vec![DnsRecord::Aaaa(Ipv6Addr::LOCALHOST)],
+                ),
+            ]),
+        );
+        DataStore::load_dns_data(&conn, initial_data)
+            .await
+            .expect("failed to insert initial data");
+
+        // Construct an update and apply it conditional on the current
+        // generation matching the initial one.  This should succeed.
+        let mut update1 = DnsVersionUpdateBuilder::new(
+            DnsGroup::Internal,
+            String::from("test-suite-1"),
+            String::from("test-suite-1"),
+        );
+        update1.remove_name(String::from("wendell")).unwrap();
+        update1
+            .add_name(
+                String::from("nelson"),
+                vec![DnsRecord::Aaaa(Ipv6Addr::LOCALHOST)],
+            )
+            .unwrap();
+        let gen1 = Generation::new();
+        datastore
+            .dns_update_from_version(&opctx, update1, gen1)
+            .await
+            .expect("failed to update from first generation");
+
+        // Now construct another update based on the _first_ version and try to
+        // apply that.  It should not work because the version has changed from
+        // under us.
+        let mut update2 = DnsVersionUpdateBuilder::new(
+            DnsGroup::Internal,
+            String::from("test-suite-2"),
+            String::from("test-suite-2"),
+        );
+        update2.remove_name(String::from("krabappel")).unwrap();
+        update2
+            .add_name(
+                String::from("hoover"),
+                vec![DnsRecord::Aaaa(Ipv6Addr::LOCALHOST)],
+            )
+            .unwrap();
+        let error = datastore
+            .dns_update_from_version(&opctx, update2.clone(), gen1)
+            .await
+            .expect_err("update unexpectedly succeeded");
+        assert!(error
+            .to_string()
+            .contains("expected current DNS version to be 1, found 2"));
+
+        // At this point, the database state should reflect the first update but
+        // not the second.
+        let config = datastore
+            .dns_config_read(&opctx, DnsGroup::Internal)
+            .await
+            .expect("failed to read config");
+        let gen2 = nexus_db_model::Generation(gen1.next());
+        assert_eq!(u64::from(*gen2), config.generation);
+        assert_eq!(1, config.zones.len());
+        let records = &config.zones[0].records;
+        assert!(records.contains_key("nelson"));
+        assert!(!records.contains_key("wendell"));
+        assert!(records.contains_key("krabappel"));
+
+        // We can apply the second update, as long as we say it's conditional on
+        // the current generation.
+        datastore
+            .dns_update_from_version(&opctx, update2, gen2)
+            .await
+            .expect("failed to update from first generation");
+        let config = datastore
+            .dns_config_read(&opctx, DnsGroup::Internal)
+            .await
+            .expect("failed to read config");
+        assert_eq!(u64::from(gen2.next()), config.generation);
+        assert_eq!(1, config.zones.len());
+        let records = &config.zones[0].records;
+        assert!(records.contains_key("nelson"));
+        assert!(!records.contains_key("wendell"));
+        assert!(!records.contains_key("krabappel"));
+        assert!(records.contains_key("hoover"));
+
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
 }
diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs
index f9e0be81c1..5f05aa1760 100644
--- a/nexus/db-queries/src/db/datastore/mod.rs
+++ b/nexus/db-queries/src/db/datastore/mod.rs
@@ -105,6 +105,7 @@ pub use instance::InstanceAndActiveVmm;
 pub use inventory::DataStoreInventoryTest;
 pub use rack::RackInit;
 pub use silo::Discoverability;
+use std::num::NonZeroU32;
 pub use switch_port::SwitchPortSettingsCombinedResult;
 pub use virtual_provisioning_collection::StorageType;
 pub use volume::read_only_resources_associated_with_volume;
@@ -121,6 +122,14 @@ pub const SERVICE_IP_POOL_NAME: &str = "oxide-service-pool";
 /// The name of the built-in Project and VPC for Oxide services.
 pub const SERVICES_DB_NAME: &str = "oxide-services";
 
+/// "limit" to be used in SQL queries that paginate through large result sets
+///
+/// This value is chosen to be small enough to avoid any queries being too
+/// expensive.
+// unsafe: `new_unchecked` is only unsound if the argument is 0.
+pub const SQL_BATCH_SIZE: NonZeroU32 =
+    unsafe { NonZeroU32::new_unchecked(1000) };
+
 // Represents a query that is ready to be executed.
 //
 // This helper trait lets the statement either be executed or explained.
diff --git a/nexus/db-queries/src/db/datastore/silo.rs b/nexus/db-queries/src/db/datastore/silo.rs
index 6cfda3c093..df10b1c072 100644
--- a/nexus/db-queries/src/db/datastore/silo.rs
+++ b/nexus/db-queries/src/db/datastore/silo.rs
@@ -282,7 +282,8 @@ impl DataStore {
                         .await?;
                 }
 
-                self.dns_update(nexus_opctx, &conn, dns_update).await?;
+                self.dns_update_incremental(nexus_opctx, &conn, dns_update)
+                    .await?;
 
                 self.silo_quotas_create(
                     &conn,
@@ -420,7 +421,7 @@ impl DataStore {
             )
             .await?;
 
-            self.dns_update(dns_opctx, &conn, dns_update).await?;
+            self.dns_update_incremental(dns_opctx, &conn, dns_update).await?;
 
             info!(opctx.log, "deleted silo {}", id);
 
diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs
index 7b94d64418..eb50061272 100644
--- a/nexus/db-queries/src/db/datastore/sled.rs
+++ b/nexus/db-queries/src/db/datastore/sled.rs
@@ -5,6 +5,7 @@
 //! [`DataStore`] methods on [`Sled`]s.
 
 use super::DataStore;
+use super::SQL_BATCH_SIZE;
 use crate::authz;
 use crate::context::OpContext;
 use crate::db;
@@ -14,11 +15,13 @@ use crate::db::model::Sled;
 use crate::db::model::SledResource;
 use crate::db::model::SledUpdate;
 use crate::db::pagination::paginated;
+use crate::db::pagination::Paginator;
 use crate::db::update_and_check::UpdateAndCheck;
 use crate::transaction_retry::OptionalError;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use chrono::Utc;
 use diesel::prelude::*;
+use nexus_types::identity::Asset;
 use omicron_common::api::external;
 use omicron_common::api::external::CreateResult;
 use omicron_common::api::external::DataPageParams;
@@ -77,6 +80,29 @@ impl DataStore {
             .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
     }
 
+    /// List all sleds, making as many queries as needed to get them all
+    ///
+    /// This should generally not be used in API handlers or other
+    /// latency-sensitive contexts, but it can make sense in saga actions or
+    /// background tasks.
+    pub async fn sled_list_all_batched(
+        &self,
+        opctx: &OpContext,
+    ) -> ListResultVec<Sled> {
+        opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?;
+        opctx.check_complex_operations_allowed()?;
+
+        let mut all_sleds = Vec::new();
+        let mut paginator = Paginator::new(SQL_BATCH_SIZE);
+        while let Some(p) = paginator.next() {
+            let batch = self.sled_list(opctx, &p.current_pagparams()).await?;
+            paginator =
+                p.found_batch(&batch, &|s: &nexus_db_model::Sled| s.id());
+            all_sleds.extend(batch);
+        }
+        Ok(all_sleds)
+    }
+
     pub async fn sled_reservation_create(
         &self,
         opctx: &OpContext,
@@ -465,4 +491,54 @@ mod test {
             limit,
         }
     }
+
+    /// Tests listing large numbers of sleds via the batched interface
+    #[tokio::test]
+    async fn sled_list_batch() {
+        let logctx =
+            dev::test_setup_log("sled_reservation_create_non_provisionable");
+        let mut db = test_setup_database(&logctx.log).await;
+        let (opctx, datastore) = datastore_test(&logctx, &db).await;
+
+        let size = usize::try_from(2 * SQL_BATCH_SIZE.get()).unwrap();
+        let mut new_sleds = Vec::with_capacity(size);
+        new_sleds.resize_with(size, test_new_sled_update);
+        let mut expected_ids: Vec<_> =
+            new_sleds.iter().map(|s| s.id()).collect();
+        expected_ids.sort();
+
+        // This is essentially the same as `sled_upsert()`.  But since we know
+        // none of these exist already, we can just insert them.  And that means
+        // we can do them all in one SQL statement.  This is considerably
+        // faster.
+        let values_to_insert: Vec<_> =
+            new_sleds.into_iter().map(|s| s.into_insertable()).collect();
+        let ninserted = {
+            use db::schema::sled::dsl;
+            diesel::insert_into(dsl::sled)
+                .values(values_to_insert)
+                .execute_async(
+                    &*datastore
+                        .pool_connection_for_tests()
+                        .await
+                        .expect("failed to get connection"),
+                )
+                .await
+                .expect("failed to insert sled")
+        };
+        assert_eq!(ninserted, size);
+
+        let sleds = datastore
+            .sled_list_all_batched(&opctx)
+            .await
+            .expect("failed to list all sleds");
+        // We don't need to sort these ids because the sleds are enumerated in
+        // id order.
+        let found_ids: Vec<_> = sleds.into_iter().map(|s| s.id()).collect();
+        assert_eq!(expected_ids, found_ids);
+        assert_eq!(found_ids.len(), size);
+
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
 }
diff --git a/nexus/deployment/src/blueprint_builder.rs b/nexus/deployment/src/blueprint_builder.rs
index 1bf46d34b2..86a9b8da6e 100644
--- a/nexus/deployment/src/blueprint_builder.rs
+++ b/nexus/deployment/src/blueprint_builder.rs
@@ -100,6 +100,7 @@ pub enum EnsureMultiple {
 pub struct BlueprintBuilder<'a> {
     /// previous blueprint, on which this one will be based
     parent_blueprint: &'a Blueprint,
+    internal_dns_version: Generation,
 
     // These fields are used to allocate resources from sleds.
     policy: &'a Policy,
@@ -128,6 +129,7 @@ impl<'a> BlueprintBuilder<'a> {
     /// collection (representing no changes from the collection state)
     pub fn build_initial_from_collection(
         collection: &'a Collection,
+        internal_dns_version: Generation,
         policy: &'a Policy,
         creator: &str,
     ) -> Result<Blueprint, Error> {
@@ -174,6 +176,7 @@ impl<'a> BlueprintBuilder<'a> {
             omicron_zones,
             zones_in_service,
             parent_blueprint_id: None,
+            internal_dns_version,
             time_created: now_db_precision(),
             creator: creator.to_owned(),
             comment: format!("from collection {}", collection.id),
@@ -184,6 +187,7 @@ impl<'a> BlueprintBuilder<'a> {
     /// starting with no changes from that state
     pub fn new_based_on(
         parent_blueprint: &'a Blueprint,
+        internal_dns_version: Generation,
         policy: &'a Policy,
         creator: &str,
     ) -> anyhow::Result<BlueprintBuilder<'a>> {
@@ -284,6 +288,7 @@ impl<'a> BlueprintBuilder<'a> {
 
         Ok(BlueprintBuilder {
             parent_blueprint,
+            internal_dns_version,
             policy,
             sled_ip_allocators: BTreeMap::new(),
             zones: BlueprintZones::new(parent_blueprint),
@@ -307,6 +312,7 @@ impl<'a> BlueprintBuilder<'a> {
             omicron_zones,
             zones_in_service: self.zones_in_service,
             parent_blueprint_id: Some(self.parent_blueprint.id),
+            internal_dns_version: self.internal_dns_version,
             time_created: now_db_precision(),
             creator: self.creator,
             comment: self.comments.join(", "),
@@ -929,6 +935,7 @@ pub mod test {
         let blueprint_initial =
             BlueprintBuilder::build_initial_from_collection(
                 &collection,
+                Generation::new(),
                 &policy,
                 "the_test",
             )
@@ -939,7 +946,7 @@ pub mod test {
         // provide that ourselves.  For our purposes though we don't care.
         let zones_in_service = blueprint_initial.zones_in_service.clone();
         let diff = blueprint_initial
-            .diff_from_collection(&collection, &zones_in_service);
+            .diff_sleds_from_collection(&collection, &zones_in_service);
         println!(
             "collection -> initial blueprint (expected no changes):\n{}",
             diff
@@ -951,13 +958,14 @@ pub mod test {
         // Test a no-op blueprint.
         let builder = BlueprintBuilder::new_based_on(
             &blueprint_initial,
+            Generation::new(),
             &policy,
             "test_basic",
         )
         .expect("failed to create builder");
         let blueprint = builder.build();
         verify_blueprint(&blueprint);
-        let diff = blueprint_initial.diff(&blueprint);
+        let diff = blueprint_initial.diff_sleds(&blueprint);
         println!(
             "initial blueprint -> next blueprint (expected no changes):\n{}",
             diff
@@ -972,15 +980,20 @@ pub mod test {
         let (collection, mut policy) = example();
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            Generation::new(),
             &policy,
             "the_test",
         )
         .expect("failed to create initial blueprint");
         verify_blueprint(&blueprint1);
 
-        let mut builder =
-            BlueprintBuilder::new_based_on(&blueprint1, &policy, "test_basic")
-                .expect("failed to create builder");
+        let mut builder = BlueprintBuilder::new_based_on(
+            &blueprint1,
+            Generation::new(),
+            &policy,
+            "test_basic",
+        )
+        .expect("failed to create builder");
 
         // The initial blueprint should have internal NTP zones on all the
         // existing sleds, plus Crucible zones on all pools.  So if we ensure
@@ -996,7 +1009,7 @@ pub mod test {
 
         let blueprint2 = builder.build();
         verify_blueprint(&blueprint2);
-        let diff = blueprint1.diff(&blueprint2);
+        let diff = blueprint1.diff_sleds(&blueprint2);
         println!(
             "initial blueprint -> next blueprint (expected no changes):\n{}",
             diff
@@ -1008,9 +1021,13 @@ pub mod test {
         // The next step is adding these zones to a new sled.
         let new_sled_id = Uuid::new_v4();
         let _ = policy_add_sled(&mut policy, new_sled_id);
-        let mut builder =
-            BlueprintBuilder::new_based_on(&blueprint2, &policy, "test_basic")
-                .expect("failed to create builder");
+        let mut builder = BlueprintBuilder::new_based_on(
+            &blueprint2,
+            Generation::new(),
+            &policy,
+            "test_basic",
+        )
+        .expect("failed to create builder");
         builder.sled_ensure_zone_ntp(new_sled_id).unwrap();
         let new_sled_resources = policy.sleds.get(&new_sled_id).unwrap();
         for pool_name in &new_sled_resources.zpools {
@@ -1021,7 +1038,7 @@ pub mod test {
 
         let blueprint3 = builder.build();
         verify_blueprint(&blueprint3);
-        let diff = blueprint2.diff(&blueprint3);
+        let diff = blueprint2.diff_sleds(&blueprint3);
         println!("expecting new NTP and Crucible zones:\n{}", diff);
 
         // No sleds were changed or removed.
@@ -1081,6 +1098,9 @@ pub mod test {
     fn test_add_nexus_with_no_existing_nexus_zones() {
         let (mut collection, policy) = example();
 
+        // We don't care about the internal DNS version here.
+        let internal_dns_version = Generation::new();
+
         // Adding a new Nexus zone currently requires copying settings from an
         // existing Nexus zone. If we remove all Nexus zones from the
         // collection, create a blueprint, then try to add a Nexus zone, it
@@ -1093,14 +1113,19 @@ pub mod test {
 
         let parent = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            internal_dns_version,
             &policy,
             "test",
         )
         .expect("failed to create initial blueprint");
 
-        let mut builder =
-            BlueprintBuilder::new_based_on(&parent, &policy, "test")
-                .expect("failed to create builder");
+        let mut builder = BlueprintBuilder::new_based_on(
+            &parent,
+            internal_dns_version,
+            &policy,
+            "test",
+        )
+        .expect("failed to create builder");
 
         let err = builder
             .sled_ensure_zone_multiple_nexus(
@@ -1124,6 +1149,9 @@ pub mod test {
     fn test_add_nexus_error_cases() {
         let (mut collection, policy) = example();
 
+        // We don't care about the internal DNS version here.
+        let internal_dns_version = Generation::new();
+
         // Remove the Nexus zone from one of the sleds so that
         // `sled_ensure_zone_nexus` can attempt to add a Nexus zone to
         // `sled_id`.
@@ -1144,6 +1172,7 @@ pub mod test {
 
         let parent = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            Generation::new(),
             &policy,
             "test",
         )
@@ -1152,9 +1181,13 @@ pub mod test {
         {
             // Attempting to add Nexus to the sled we removed it from (with no
             // other changes to the environment) should succeed.
-            let mut builder =
-                BlueprintBuilder::new_based_on(&parent, &policy, "test")
-                    .expect("failed to create builder");
+            let mut builder = BlueprintBuilder::new_based_on(
+                &parent,
+                internal_dns_version,
+                &policy,
+                "test",
+            )
+            .expect("failed to create builder");
             let added = builder
                 .sled_ensure_zone_multiple_nexus(sled_id, 1)
                 .expect("failed to ensure nexus zone");
@@ -1166,9 +1199,13 @@ pub mod test {
             // Attempting to add multiple Nexus zones to the sled we removed it
             // from (with no other changes to the environment) should also
             // succeed.
-            let mut builder =
-                BlueprintBuilder::new_based_on(&parent, &policy, "test")
-                    .expect("failed to create builder");
+            let mut builder = BlueprintBuilder::new_based_on(
+                &parent,
+                internal_dns_version,
+                &policy,
+                "test",
+            )
+            .expect("failed to create builder");
             let added = builder
                 .sled_ensure_zone_multiple_nexus(sled_id, 3)
                 .expect("failed to ensure nexus zone");
@@ -1194,9 +1231,13 @@ pub mod test {
             assert!(!used_ip_ranges.is_empty());
             policy.service_ip_pool_ranges = used_ip_ranges;
 
-            let mut builder =
-                BlueprintBuilder::new_based_on(&parent, &policy, "test")
-                    .expect("failed to create builder");
+            let mut builder = BlueprintBuilder::new_based_on(
+                &parent,
+                internal_dns_version,
+                &policy,
+                "test",
+            )
+            .expect("failed to create builder");
             let err = builder
                 .sled_ensure_zone_multiple_nexus(sled_id, 1)
                 .unwrap_err();
@@ -1247,12 +1288,18 @@ pub mod test {
 
         let parent = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            Generation::new(),
             &policy,
             "test",
         )
         .unwrap();
 
-        match BlueprintBuilder::new_based_on(&parent, &policy, "test") {
+        match BlueprintBuilder::new_based_on(
+            &parent,
+            Generation::new(),
+            &policy,
+            "test",
+        ) {
             Ok(_) => panic!("unexpected success"),
             Err(err) => assert!(
                 err.to_string().contains("duplicate external IP"),
@@ -1290,12 +1337,18 @@ pub mod test {
 
         let parent = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            Generation::new(),
             &policy,
             "test",
         )
         .unwrap();
 
-        match BlueprintBuilder::new_based_on(&parent, &policy, "test") {
+        match BlueprintBuilder::new_based_on(
+            &parent,
+            Generation::new(),
+            &policy,
+            "test",
+        ) {
             Ok(_) => panic!("unexpected success"),
             Err(err) => assert!(
                 err.to_string().contains("duplicate Nexus NIC IP"),
@@ -1333,12 +1386,18 @@ pub mod test {
 
         let parent = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            Generation::new(),
             &policy,
             "test",
         )
         .unwrap();
 
-        match BlueprintBuilder::new_based_on(&parent, &policy, "test") {
+        match BlueprintBuilder::new_based_on(
+            &parent,
+            Generation::new(),
+            &policy,
+            "test",
+        ) {
             Ok(_) => panic!("unexpected success"),
             Err(err) => assert!(
                 err.to_string().contains("duplicate service vNIC MAC"),
diff --git a/nexus/deployment/src/planner.rs b/nexus/deployment/src/planner.rs
index cbdcfd80c0..7973157068 100644
--- a/nexus/deployment/src/planner.rs
+++ b/nexus/deployment/src/planner.rs
@@ -14,8 +14,8 @@ use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::Policy;
 use nexus_types::external_api::views::SledProvisionState;
 use nexus_types::inventory::Collection;
-use slog::warn;
-use slog::{info, Logger};
+use omicron_common::api::external::Generation;
+use slog::{info, warn, Logger};
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use uuid::Uuid;
@@ -40,14 +40,19 @@ impl<'a> Planner<'a> {
     pub fn new_based_on(
         log: Logger,
         parent_blueprint: &'a Blueprint,
+        internal_dns_version: Generation,
         policy: &'a Policy,
         creator: &str,
         // NOTE: Right now, we just assume that this is the latest inventory
         // collection.  See the comment on the corresponding field in `Planner`.
         inventory: &'a Collection,
     ) -> anyhow::Result<Planner<'a>> {
-        let blueprint =
-            BlueprintBuilder::new_based_on(parent_blueprint, policy, creator)?;
+        let blueprint = BlueprintBuilder::new_based_on(
+            parent_blueprint,
+            internal_dns_version,
+            policy,
+            creator,
+        )?;
         Ok(Planner { log, policy, blueprint, inventory })
     }
 
@@ -319,6 +324,9 @@ mod test {
     fn test_basic_add_sled() {
         let logctx = test_setup_log("planner_basic_add_sled");
 
+        // For our purposes, we don't care about the internal DNS generation.
+        let internal_dns_version = Generation::new();
+
         // Use our example inventory collection.
         let (mut collection, mut policy) = example();
 
@@ -326,6 +334,7 @@ mod test {
         // because there's a separate test for that.
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            internal_dns_version,
             &policy,
             "the_test",
         )
@@ -338,6 +347,7 @@ mod test {
         let blueprint2 = Planner::new_based_on(
             logctx.log.clone(),
             &blueprint1,
+            internal_dns_version,
             &policy,
             "no-op?",
             &collection,
@@ -346,7 +356,7 @@ mod test {
         .plan()
         .expect("failed to plan");
 
-        let diff = blueprint1.diff(&blueprint2);
+        let diff = blueprint1.diff_sleds(&blueprint2);
         println!("1 -> 2 (expected no changes):\n{}", diff);
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
@@ -362,6 +372,7 @@ mod test {
         let blueprint3 = Planner::new_based_on(
             logctx.log.clone(),
             &blueprint2,
+            internal_dns_version,
             &policy,
             "test: add NTP?",
             &collection,
@@ -370,7 +381,7 @@ mod test {
         .plan()
         .expect("failed to plan");
 
-        let diff = blueprint2.diff(&blueprint3);
+        let diff = blueprint2.diff_sleds(&blueprint3);
         println!("2 -> 3 (expect new NTP zone on new sled):\n{}", diff);
         let sleds = diff.sleds_added().collect::<Vec<_>>();
         let (sled_id, sled_zones) = sleds[0];
@@ -394,6 +405,7 @@ mod test {
         let blueprint4 = Planner::new_based_on(
             logctx.log.clone(),
             &blueprint3,
+            internal_dns_version,
             &policy,
             "test: add nothing more",
             &collection,
@@ -401,7 +413,7 @@ mod test {
         .expect("failed to create planner")
         .plan()
         .expect("failed to plan");
-        let diff = blueprint3.diff(&blueprint4);
+        let diff = blueprint3.diff_sleds(&blueprint4);
         println!("3 -> 4 (expected no changes):\n{}", diff);
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
@@ -430,6 +442,7 @@ mod test {
         let blueprint5 = Planner::new_based_on(
             logctx.log.clone(),
             &blueprint3,
+            internal_dns_version,
             &policy,
             "test: add Crucible zones?",
             &collection,
@@ -438,7 +451,7 @@ mod test {
         .plan()
         .expect("failed to plan");
 
-        let diff = blueprint3.diff(&blueprint5);
+        let diff = blueprint3.diff_sleds(&blueprint5);
         println!("3 -> 5 (expect Crucible zones):\n{}", diff);
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
@@ -465,6 +478,7 @@ mod test {
         let blueprint6 = Planner::new_based_on(
             logctx.log.clone(),
             &blueprint5,
+            internal_dns_version,
             &policy,
             "test: no-op?",
             &collection,
@@ -473,7 +487,7 @@ mod test {
         .plan()
         .expect("failed to plan");
 
-        let diff = blueprint5.diff(&blueprint6);
+        let diff = blueprint5.diff_sleds(&blueprint6);
         println!("5 -> 6 (expect no changes):\n{}", diff);
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
@@ -489,6 +503,9 @@ mod test {
     fn test_add_multiple_nexus_to_one_sled() {
         let logctx = test_setup_log("planner_add_multiple_nexus_to_one_sled");
 
+        // For our purposes, we don't care about the internal DNS generation.
+        let internal_dns_version = Generation::new();
+
         // Use our example inventory collection as a starting point, but strip
         // it down to just one sled.
         let (sled_id, collection, mut policy) = {
@@ -510,6 +527,7 @@ mod test {
         // Build the initial blueprint.
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            internal_dns_version,
             &policy,
             "the_test",
         )
@@ -536,6 +554,7 @@ mod test {
         let blueprint2 = Planner::new_based_on(
             logctx.log.clone(),
             &blueprint1,
+            internal_dns_version,
             &policy,
             "add more Nexus",
             &collection,
@@ -544,7 +563,7 @@ mod test {
         .plan()
         .expect("failed to plan");
 
-        let diff = blueprint1.diff(&blueprint2);
+        let diff = blueprint1.diff_sleds(&blueprint2);
         println!("1 -> 2 (added additional Nexus zones):\n{}", diff);
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
@@ -579,6 +598,7 @@ mod test {
         // Build the initial blueprint.
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            Generation::new(),
             &policy,
             "the_test",
         )
@@ -602,6 +622,7 @@ mod test {
         let blueprint2 = Planner::new_based_on(
             logctx.log.clone(),
             &blueprint1,
+            Generation::new(),
             &policy,
             "add more Nexus",
             &collection,
@@ -610,7 +631,7 @@ mod test {
         .plan()
         .expect("failed to plan");
 
-        let diff = blueprint1.diff(&blueprint2);
+        let diff = blueprint1.diff_sleds(&blueprint2);
         println!("1 -> 2 (added additional Nexus zones):\n{}", diff);
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
@@ -658,6 +679,7 @@ mod test {
         // Build the initial blueprint.
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            Generation::new(),
             &policy,
             "the_test",
         )
@@ -689,6 +711,7 @@ mod test {
         let blueprint2 = Planner::new_based_on(
             logctx.log.clone(),
             &blueprint1,
+            Generation::new(),
             &policy,
             "add more Nexus",
             &collection,
@@ -697,7 +720,7 @@ mod test {
         .plan()
         .expect("failed to plan");
 
-        let diff = blueprint1.diff(&blueprint2);
+        let diff = blueprint1.diff_sleds(&blueprint2);
         println!("1 -> 2 (added additional Nexus zones):\n{}", diff);
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
diff --git a/nexus/src/app/background/blueprint_execution.rs b/nexus/src/app/background/blueprint_execution.rs
index 84d4cef212..32797facbf 100644
--- a/nexus/src/app/background/blueprint_execution.rs
+++ b/nexus/src/app/background/blueprint_execution.rs
@@ -19,6 +19,7 @@ use tokio::sync::watch;
 pub struct BlueprintExecutor {
     datastore: Arc<DataStore>,
     rx_blueprint: watch::Receiver<Option<Arc<(BlueprintTarget, Blueprint)>>>,
+    nexus_label: String,
 }
 
 impl BlueprintExecutor {
@@ -27,8 +28,9 @@ impl BlueprintExecutor {
         rx_blueprint: watch::Receiver<
             Option<Arc<(BlueprintTarget, Blueprint)>>,
         >,
+        nexus_label: String,
     ) -> BlueprintExecutor {
-        BlueprintExecutor { datastore, rx_blueprint }
+        BlueprintExecutor { datastore, rx_blueprint, nexus_label }
     }
 }
 
@@ -65,6 +67,7 @@ impl BackgroundTask for BlueprintExecutor {
                 opctx,
                 &self.datastore,
                 blueprint,
+                &self.nexus_label,
             )
             .await;
 
@@ -119,6 +122,7 @@ mod test {
 
     fn create_blueprint(
         omicron_zones: BTreeMap<Uuid, OmicronZonesConfig>,
+        internal_dns_version: Generation,
     ) -> (BlueprintTarget, Blueprint) {
         let id = Uuid::new_v4();
         (
@@ -132,6 +136,7 @@ mod test {
                 omicron_zones,
                 zones_in_service: BTreeSet::new(),
                 parent_blueprint_id: None,
+                internal_dns_version,
                 time_created: chrono::Utc::now(),
                 creator: "test".to_string(),
                 comment: "test blueprint".to_string(),
@@ -185,7 +190,11 @@ mod test {
         }
 
         let (blueprint_tx, blueprint_rx) = watch::channel(None);
-        let mut task = BlueprintExecutor::new(datastore.clone(), blueprint_rx);
+        let mut task = BlueprintExecutor::new(
+            datastore.clone(),
+            blueprint_rx,
+            String::from("test-suite"),
+        );
 
         // Now we're ready.
         //
@@ -196,7 +205,8 @@ mod test {
 
         // With a target blueprint having no zones, the task should trivially
         // complete and report a successful (empty) summary.
-        let blueprint = Arc::new(create_blueprint(BTreeMap::new()));
+        let generation = Generation::new();
+        let blueprint = Arc::new(create_blueprint(BTreeMap::new(), generation));
         blueprint_tx.send(Some(blueprint)).unwrap();
         let value = task.activate(&opctx).await;
         println!("activating with no zones: {:?}", value);
@@ -204,32 +214,35 @@ mod test {
 
         // Create a non-empty blueprint describing two servers and verify that
         // the task correctly winds up making requests to both of them and
-        // reporting success.  We reuse the same `OmicronZonesConfig` in
-        // constructing the blueprint because the details don't matter for this
-        // test.
-        let zones = OmicronZonesConfig {
-            generation: Generation::new(),
-            zones: vec![OmicronZoneConfig {
-                id: Uuid::new_v4(),
-                underlay_address: "::1".parse().unwrap(),
-                zone_type: OmicronZoneType::InternalDns {
-                    dataset: OmicronZoneDataset {
-                        pool_name: format!("oxp_{}", Uuid::new_v4())
-                            .parse()
-                            .unwrap(),
+        // reporting success.
+        fn make_zones() -> OmicronZonesConfig {
+            OmicronZonesConfig {
+                generation: Generation::new(),
+                zones: vec![OmicronZoneConfig {
+                    id: Uuid::new_v4(),
+                    underlay_address: "::1".parse().unwrap(),
+                    zone_type: OmicronZoneType::InternalDns {
+                        dataset: OmicronZoneDataset {
+                            pool_name: format!("oxp_{}", Uuid::new_v4())
+                                .parse()
+                                .unwrap(),
+                        },
+                        dns_address: "oh-hello-internal-dns".into(),
+                        gz_address: "::1".parse().unwrap(),
+                        gz_address_index: 0,
+                        http_address: "some-ipv6-address".into(),
                     },
-                    dns_address: "oh-hello-internal-dns".into(),
-                    gz_address: "::1".parse().unwrap(),
-                    gz_address_index: 0,
-                    http_address: "some-ipv6-address".into(),
-                },
-            }],
-        };
-
-        let mut blueprint = create_blueprint(BTreeMap::from([
-            (sled_id1, zones.clone()),
-            (sled_id2, zones.clone()),
-        ]));
+                }],
+            }
+        }
+        let generation = generation.next();
+        let mut blueprint = create_blueprint(
+            BTreeMap::from([
+                (sled_id1, make_zones()),
+                (sled_id2, make_zones()),
+            ]),
+            generation,
+        );
 
         blueprint_tx.send(Some(Arc::new(blueprint.clone()))).unwrap();
 
@@ -255,6 +268,8 @@ mod test {
 
         // Now, disable the target and make sure that we _don't_ invoke the sled
         // agent.  It's enough to just not set expectations.
+        blueprint.1.internal_dns_version =
+            blueprint.1.internal_dns_version.next();
         blueprint.0.enabled = false;
         blueprint_tx.send(Some(Arc::new(blueprint.clone()))).unwrap();
         let value = task.activate(&opctx).await;
diff --git a/nexus/src/app/background/blueprint_load.rs b/nexus/src/app/background/blueprint_load.rs
index c34d2ab103..8886df81cd 100644
--- a/nexus/src/app/background/blueprint_load.rs
+++ b/nexus/src/app/background/blueprint_load.rs
@@ -184,6 +184,7 @@ mod test {
     use nexus_inventory::now_db_precision;
     use nexus_test_utils_macros::nexus_test;
     use nexus_types::deployment::{Blueprint, BlueprintTarget};
+    use omicron_common::api::external::Generation;
     use serde::Deserialize;
     use std::collections::{BTreeMap, BTreeSet};
     use uuid::Uuid;
@@ -206,6 +207,7 @@ mod test {
                 omicron_zones: BTreeMap::new(),
                 zones_in_service: BTreeSet::new(),
                 parent_blueprint_id,
+                internal_dns_version: Generation::new(),
                 time_created: now_db_precision(),
                 creator: "test".to_string(),
                 comment: "test blueprint".to_string(),
diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs
index 27e58a298c..846051a068 100644
--- a/nexus/src/app/background/init.rs
+++ b/nexus/src/app/background/init.rs
@@ -228,6 +228,7 @@ impl BackgroundTasks {
         let blueprint_executor = blueprint_execution::BlueprintExecutor::new(
             datastore.clone(),
             rx_blueprint.clone(),
+            nexus_id.to_string(),
         );
         let task_blueprint_executor = driver.register(
             String::from("blueprint_executor"),
@@ -628,7 +629,7 @@ pub mod test {
     ) {
         let conn = datastore.pool_connection_for_tests().await.unwrap();
         info!(opctx.log, "writing DNS update...");
-        datastore.dns_update(opctx, &conn, update).await.unwrap();
+        datastore.dns_update_incremental(opctx, &conn, update).await.unwrap();
     }
 
     pub(crate) async fn write_test_dns_generation(
diff --git a/nexus/src/app/deployment.rs b/nexus/src/app/deployment.rs
index b8cb6deabf..61ce803d13 100644
--- a/nexus/src/app/deployment.rs
+++ b/nexus/src/app/deployment.rs
@@ -4,8 +4,10 @@
 
 //! Configuration of the deployment system
 
+use nexus_db_model::DnsGroup;
 use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
+use nexus_db_queries::db::datastore::SQL_BATCH_SIZE;
 use nexus_db_queries::db::pagination::Paginator;
 use nexus_deployment::blueprint_builder::BlueprintBuilder;
 use nexus_deployment::planner::Planner;
@@ -26,6 +28,7 @@ use omicron_common::api::external::CreateResult;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::DeleteResult;
 use omicron_common::api::external::Error;
+use omicron_common::api::external::Generation;
 use omicron_common::api::external::InternalContext;
 use omicron_common::api::external::ListResultVec;
 use omicron_common::api::external::LookupResult;
@@ -33,19 +36,15 @@ use omicron_common::api::external::LookupType;
 use slog_error_chain::InlineErrorChain;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
-use std::num::NonZeroU32;
 use std::str::FromStr;
 use uuid::Uuid;
 
-/// "limit" used in SQL queries that paginate through all sleds, zpools, etc.
-// unsafe: `new_unchecked` is only unsound if the argument is 0.
-const SQL_BATCH_SIZE: NonZeroU32 = unsafe { NonZeroU32::new_unchecked(1000) };
-
 /// Common structure for collecting information that the planner needs
 struct PlanningContext {
     policy: Policy,
     creator: String,
     inventory: Option<Collection>,
+    internal_dns_version: Generation,
 }
 
 impl super::Nexus {
@@ -118,18 +117,7 @@ impl super::Nexus {
         let creator = self.id.to_string();
         let datastore = self.datastore();
 
-        let sled_rows = {
-            let mut all_sleds = Vec::new();
-            let mut paginator = Paginator::new(SQL_BATCH_SIZE);
-            while let Some(p) = paginator.next() {
-                let batch =
-                    datastore.sled_list(opctx, &p.current_pagparams()).await?;
-                paginator =
-                    p.found_batch(&batch, &|s: &nexus_db_model::Sled| s.id());
-                all_sleds.extend(batch);
-            }
-            all_sleds
-        };
+        let sled_rows = datastore.sled_list_all_batched(opctx).await?;
 
         let mut zpools_by_sled_id = {
             let mut zpools = BTreeMap::new();
@@ -222,6 +210,16 @@ impl super::Nexus {
                 "fetching latest inventory collection for blueprint planner",
             )?;
 
+        // Fetch the current internal DNS version.  This could be made part of
+        // inventory, but it's enough of a one-off that there's no particular
+        // advantage to doing that work now.
+        let dns_version = datastore
+            .dns_group_latest_version(opctx, DnsGroup::Internal)
+            .await
+            .internal_context(
+                "fetching internal DNS version for blueprint planning",
+            )?;
+
         Ok(PlanningContext {
             creator,
             policy: Policy {
@@ -230,6 +228,7 @@ impl super::Nexus {
                 target_nexus_zone_count: NEXUS_REDUNDANCY,
             },
             inventory,
+            internal_dns_version: *dns_version.version,
         })
     }
 
@@ -253,6 +252,7 @@ impl super::Nexus {
         let planning_context = self.blueprint_planning_context(opctx).await?;
         let blueprint = BlueprintBuilder::build_initial_from_collection(
             &collection,
+            planning_context.internal_dns_version,
             &planning_context.policy,
             &planning_context.creator,
         )
@@ -287,6 +287,7 @@ impl super::Nexus {
         let planner = Planner::new_based_on(
             opctx.log.clone(),
             &parent_blueprint,
+            planning_context.internal_dns_version,
             &planning_context.policy,
             &planning_context.creator,
             &inventory,
diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs
index 2a37721bb0..271025f7a7 100644
--- a/nexus/test-utils/src/lib.rs
+++ b/nexus/test-utils/src/lib.rs
@@ -781,25 +781,6 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
         );
     }
 
-    pub async fn scrimlet_dns_setup(&mut self) {
-        let sled_agent = self
-            .sled_agent
-            .as_ref()
-            .expect("Cannot set up scrimlet DNS without sled agent");
-
-        let sa = match sled_agent.http_server.local_addr() {
-            SocketAddr::V6(sa) => sa,
-            SocketAddr::V4(_) => panic!("expected SocketAddrV6 for sled agent"),
-        };
-
-        for loc in [SwitchLocation::Switch0, SwitchLocation::Switch1] {
-            self.rack_init_builder
-                .internal_dns_config
-                .host_scrimlet(loc, sa)
-                .expect("add switch0 scrimlet dns entry");
-        }
-    }
-
     // Set up an external DNS server.
     pub async fn start_external_dns(&mut self) {
         let log = self.logctx.log.new(o!("component" => "external_dns_server"));
@@ -1063,10 +1044,6 @@ async fn setup_with_config_impl<N: NexusServer>(
                     "start_crucible_pantry",
                     Box::new(|builder| builder.start_crucible_pantry().boxed()),
                 ),
-                (
-                    "scrimlet_dns_setup",
-                    Box::new(|builder| builder.scrimlet_dns_setup().boxed()),
-                ),
                 (
                     "populate_internal_dns",
                     Box::new(|builder| builder.populate_internal_dns().boxed()),
diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs
index 490097d46d..2e683878be 100644
--- a/nexus/types/src/deployment.rs
+++ b/nexus/types/src/deployment.rs
@@ -135,6 +135,10 @@ pub struct Blueprint {
     /// which blueprint this blueprint is based on
     pub parent_blueprint_id: Option<Uuid>,
 
+    /// internal DNS version when this blueprint was created
+    // See blueprint generation for more on this.
+    pub internal_dns_version: Generation,
+
     /// when this blueprint was generated (for debugging)
     pub time_created: chrono::DateTime<chrono::Utc>,
     /// identity of the component that generated the blueprint (for debugging)
@@ -161,8 +165,11 @@ impl Blueprint {
         self.omicron_zones.keys().copied()
     }
 
-    /// Summarize the difference between two blueprints
-    pub fn diff<'a>(&'a self, other: &'a Blueprint) -> OmicronZonesDiff<'a> {
+    /// Summarize the difference between sleds and zones between two blueprints
+    pub fn diff_sleds<'a>(
+        &'a self,
+        other: &'a Blueprint,
+    ) -> OmicronZonesDiff<'a> {
         OmicronZonesDiff {
             before_label: format!("blueprint {}", self.id),
             before_zones: self.omicron_zones.clone(),
@@ -173,14 +180,15 @@ impl Blueprint {
         }
     }
 
-    /// Summarize the difference between a collection and a blueprint
+    /// Summarize the differences in sleds and zones between a collection and a
+    /// blueprint
     ///
     /// This gives an idea about what would change about a running system if one
     /// were to execute the blueprint.
     ///
     /// Note that collections do not currently include information about what
     /// zones are in-service, so the caller must provide that information.
-    pub fn diff_from_collection<'a>(
+    pub fn diff_sleds_from_collection<'a>(
         &'a self,
         collection: &'a Collection,
         before_zones_in_service: &'a BTreeSet<Uuid>,
@@ -212,6 +220,8 @@ pub struct BlueprintMetadata {
 
     /// which blueprint this blueprint is based on
     pub parent_blueprint_id: Option<Uuid>,
+    /// internal DNS version when this blueprint was created
+    pub internal_dns_version: Generation,
 
     /// when this blueprint was generated (for debugging)
     pub time_created: chrono::DateTime<chrono::Utc>,
diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs
index 50e8b380b3..71e8e64d97 100644
--- a/nexus/types/src/inventory.rs
+++ b/nexus/types/src/inventory.rs
@@ -134,6 +134,14 @@ impl Collection {
     ) -> impl Iterator<Item = &OmicronZoneConfig> {
         self.omicron_zones.values().flat_map(|z| z.zones.zones.iter())
     }
+
+    /// Iterate over the sled ids of sleds identified as Scrimlets
+    pub fn scrimlets(&self) -> impl Iterator<Item = Uuid> + '_ {
+        self.sled_agents
+            .iter()
+            .filter(|(_, inventory)| inventory.sled_role == SledRole::Scrimlet)
+            .map(|(sled_id, _)| *sled_id)
+    }
 }
 
 /// A unique baseboard id found during a collection
diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json
index b8b45aa08e..53a53fb219 100644
--- a/openapi/nexus-internal.json
+++ b/openapi/nexus-internal.json
@@ -2124,6 +2124,14 @@
             "type": "string",
             "format": "uuid"
           },
+          "internal_dns_version": {
+            "description": "internal DNS version when this blueprint was created",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Generation"
+              }
+            ]
+          },
           "omicron_zones": {
             "description": "mapping: sled id -> zones deployed on each sled A sled is considered part of the control plane cluster iff it has an entry in this map.",
             "type": "object",
@@ -2156,6 +2164,7 @@
           "comment",
           "creator",
           "id",
+          "internal_dns_version",
           "omicron_zones",
           "time_created",
           "zones_in_service"
@@ -2178,6 +2187,14 @@
             "type": "string",
             "format": "uuid"
           },
+          "internal_dns_version": {
+            "description": "internal DNS version when this blueprint was created",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Generation"
+              }
+            ]
+          },
           "parent_blueprint_id": {
             "nullable": true,
             "description": "which blueprint this blueprint is based on",
@@ -2194,6 +2211,7 @@
           "comment",
           "creator",
           "id",
+          "internal_dns_version",
           "time_created"
         ]
       },
diff --git a/schema/crdb/36.0.0/up1.sql b/schema/crdb/36.0.0/up1.sql
new file mode 100644
index 0000000000..38e3843bfc
--- /dev/null
+++ b/schema/crdb/36.0.0/up1.sql
@@ -0,0 +1,8 @@
+-- Add the "internal_dns_version" column to the "blueprint" table.
+-- This query will end up setting the internal DNS version for any existing
+-- blueprints to 1.  This is always safe because it's the smallest possible
+-- value and if a value is too small, the end result is simply needing to
+-- regenerate the blueprint in order to be able to execute it.  (On the other
+-- hand, using a value that's too large could cause corruption.)
+ALTER TABLE omicron.public.blueprint
+    ADD COLUMN IF NOT EXISTS internal_dns_version INT8 NOT NULL DEFAULT 1;
diff --git a/schema/crdb/36.0.0/up2.sql b/schema/crdb/36.0.0/up2.sql
new file mode 100644
index 0000000000..7c0082fbb0
--- /dev/null
+++ b/schema/crdb/36.0.0/up2.sql
@@ -0,0 +1,2 @@
+ALTER TABLE omicron.public.blueprint
+    ALTER COLUMN internal_dns_version DROP DEFAULT;
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 6c82b63e6e..87a22d1adc 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -3096,7 +3096,10 @@ CREATE TABLE IF NOT EXISTS omicron.public.blueprint (
     -- These fields are for debugging only.
     time_created TIMESTAMPTZ NOT NULL,
     creator TEXT NOT NULL,
-    comment TEXT NOT NULL
+    comment TEXT NOT NULL,
+
+    -- identifies the latest internal DNS version when blueprint planning began
+    internal_dns_version INT8 NOT NULL
 );
 
 -- table describing both the current and historical target blueprints of the
@@ -3515,7 +3518,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '35.0.0', NULL)
+    ( TRUE, NOW(), NOW(), '36.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;
diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs
index 0b633c2057..77fd8a39de 100644
--- a/sled-agent/src/rack_setup/plan/service.rs
+++ b/sled-agent/src/rack_setup/plan/service.rs
@@ -19,7 +19,6 @@ use omicron_common::address::{
     RSS_RESERVED_ADDRESSES, SLED_PREFIX,
 };
 use omicron_common::api::external::{MacAddr, Vni};
-use omicron_common::api::internal::shared::SwitchLocation;
 use omicron_common::api::internal::shared::{
     NetworkInterface, NetworkInterfaceKind, SourceNatConfig,
 };
@@ -295,36 +294,17 @@ impl Plan {
                 "No scrimlets observed".to_string(),
             ));
         }
-        for (i, sled) in scrimlets.iter().enumerate() {
+        for sled in scrimlets.iter() {
             let address = get_switch_zone_address(sled.subnet);
-            let zone =
-                dns_builder.host_dendrite(sled.sled_id, address).unwrap();
             dns_builder
-                .service_backend_zone(
-                    ServiceName::Dendrite,
-                    &zone,
+                .host_zone_switch(
+                    sled.sled_id,
+                    address,
                     DENDRITE_PORT,
-                )
-                .unwrap();
-            dns_builder
-                .service_backend_zone(
-                    ServiceName::ManagementGatewayService,
-                    &zone,
                     MGS_PORT,
+                    MGD_PORT,
                 )
                 .unwrap();
-            dns_builder
-                .service_backend_zone(ServiceName::Mgd, &zone, MGD_PORT)
-                .unwrap();
-
-            // TODO only works for single rack
-            let sled_address = get_sled_address(sled.subnet);
-            let switch_location = if i == 0 {
-                SwitchLocation::Switch0
-            } else {
-                SwitchLocation::Switch1
-            };
-            dns_builder.host_scrimlet(switch_location, sled_address).unwrap();
         }
 
         // We'll stripe most services across all available Sleds, round-robin
@@ -354,11 +334,11 @@ impl Plan {
             let dns_address = SocketAddrV6::new(ip, DNS_PORT, 0, 0);
 
             let id = Uuid::new_v4();
-            let zone = dns_builder.host_zone(id, ip).unwrap();
             dns_builder
-                .service_backend_zone(
+                .host_zone_with_one_backend(
+                    id,
+                    ip,
                     ServiceName::InternalDns,
-                    &zone,
                     DNS_HTTP_PORT,
                 )
                 .unwrap();
@@ -391,9 +371,13 @@ impl Plan {
             let ip = sled.addr_alloc.next().expect("Not enough addrs");
             let port = omicron_common::address::COCKROACH_PORT;
             let address = SocketAddrV6::new(ip, port, 0, 0);
-            let zone = dns_builder.host_zone(id, ip).unwrap();
             dns_builder
-                .service_backend_zone(ServiceName::Cockroach, &zone, port)
+                .host_zone_with_one_backend(
+                    id,
+                    ip,
+                    ServiceName::Cockroach,
+                    port,
+                )
                 .unwrap();
             let dataset_name =
                 sled.alloc_from_u2_zpool(DatasetKind::CockroachDb)?;
@@ -427,11 +411,11 @@ impl Plan {
             let internal_ip = sled.addr_alloc.next().expect("Not enough addrs");
             let http_port = omicron_common::address::DNS_HTTP_PORT;
             let http_address = SocketAddrV6::new(internal_ip, http_port, 0, 0);
-            let zone = dns_builder.host_zone(id, internal_ip).unwrap();
             dns_builder
-                .service_backend_zone(
+                .host_zone_with_one_backend(
+                    id,
+                    internal_ip,
                     ServiceName::ExternalDns,
-                    &zone,
                     http_port,
                 )
                 .unwrap();
@@ -463,11 +447,11 @@ impl Plan {
             };
             let id = Uuid::new_v4();
             let address = sled.addr_alloc.next().expect("Not enough addrs");
-            let zone = dns_builder.host_zone(id, address).unwrap();
             dns_builder
-                .service_backend_zone(
+                .host_zone_with_one_backend(
+                    id,
+                    address,
                     ServiceName::Nexus,
-                    &zone,
                     omicron_common::address::NEXUS_INTERNAL_PORT,
                 )
                 .unwrap();
@@ -506,11 +490,11 @@ impl Plan {
             };
             let id = Uuid::new_v4();
             let address = sled.addr_alloc.next().expect("Not enough addrs");
-            let zone = dns_builder.host_zone(id, address).unwrap();
             dns_builder
-                .service_backend_zone(
+                .host_zone_with_one_backend(
+                    id,
+                    address,
                     ServiceName::Oximeter,
-                    &zone,
                     omicron_common::address::OXIMETER_PORT,
                 )
                 .unwrap();
@@ -540,9 +524,13 @@ impl Plan {
             let ip = sled.addr_alloc.next().expect("Not enough addrs");
             let port = omicron_common::address::CLICKHOUSE_PORT;
             let address = SocketAddrV6::new(ip, port, 0, 0);
-            let zone = dns_builder.host_zone(id, ip).unwrap();
             dns_builder
-                .service_backend_zone(ServiceName::Clickhouse, &zone, port)
+                .host_zone_with_one_backend(
+                    id,
+                    ip,
+                    ServiceName::Clickhouse,
+                    port,
+                )
                 .unwrap();
             let dataset_name =
                 sled.alloc_from_u2_zpool(DatasetKind::Clickhouse)?;
@@ -572,11 +560,11 @@ impl Plan {
             let ip = sled.addr_alloc.next().expect("Not enough addrs");
             let port = omicron_common::address::CLICKHOUSE_KEEPER_PORT;
             let address = SocketAddrV6::new(ip, port, 0, 0);
-            let zone = dns_builder.host_zone(id, ip).unwrap();
             dns_builder
-                .service_backend_zone(
+                .host_zone_with_one_backend(
+                    id,
+                    ip,
                     ServiceName::ClickhouseKeeper,
-                    &zone,
                     port,
                 )
                 .unwrap();
@@ -605,9 +593,13 @@ impl Plan {
             let address = sled.addr_alloc.next().expect("Not enough addrs");
             let port = omicron_common::address::CRUCIBLE_PANTRY_PORT;
             let id = Uuid::new_v4();
-            let zone = dns_builder.host_zone(id, address).unwrap();
             dns_builder
-                .service_backend_zone(ServiceName::CruciblePantry, &zone, port)
+                .host_zone_with_one_backend(
+                    id,
+                    address,
+                    ServiceName::CruciblePantry,
+                    port,
+                )
                 .unwrap();
             sled.request.zones.push(OmicronZoneConfig {
                 id,
@@ -626,11 +618,11 @@ impl Plan {
                 let port = omicron_common::address::CRUCIBLE_PORT;
                 let address = SocketAddrV6::new(ip, port, 0, 0);
                 let id = Uuid::new_v4();
-                let zone = dns_builder.host_zone(id, ip).unwrap();
                 dns_builder
-                    .service_backend_zone(
+                    .host_zone_with_one_backend(
+                        id,
+                        ip,
                         ServiceName::Crucible(id),
-                        &zone,
                         port,
                     )
                     .unwrap();
@@ -653,7 +645,6 @@ impl Plan {
         for (idx, sled) in sled_info.iter_mut().enumerate() {
             let id = Uuid::new_v4();
             let address = sled.addr_alloc.next().expect("Not enough addrs");
-            let zone = dns_builder.host_zone(id, address).unwrap();
             let ntp_address = SocketAddrV6::new(address, NTP_PORT, 0, 0);
 
             let (zone_type, svcname) = if idx < BOUNDARY_NTP_COUNT {
@@ -683,7 +674,9 @@ impl Plan {
                 )
             };
 
-            dns_builder.service_backend_zone(svcname, &zone, NTP_PORT).unwrap();
+            dns_builder
+                .host_zone_with_one_backend(id, address, svcname, NTP_PORT)
+                .unwrap();
 
             sled.request.zones.push(OmicronZoneConfig {
                 id,

From e56e2394eb5f581a1957c47c520344d85533e704 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Karen=20C=C3=A1rcamo?= <karencfv@users.noreply.github.com>
Date: Wed, 21 Feb 2024 16:54:39 +1300
Subject: [PATCH 026/157] Create composite crucible and pantry packages (#4927)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Create new packages for crucible and pantry to include the zone network
config service.

Depends on https://github.com/oxidecomputer/crucible/pull/1096.

These two PRs should be merged in coordination

Related: https://github.com/oxidecomputer/omicron/issues/1898

### Crucible updates

This PR also merges a few changes from Crucible:

* fe0c5c7 - [smf] Use new zone network config service
* 3d48060 - (upstream/main) Move a few methods into downstairs
* b01e15c - Remove extra clone in upstairs read
* b4f37b4 - Make `crucible-downstairs` not depend on upstairs
* 733b7f9 - Update Rust crate rusqlite to 0.31
* 961e971 - Update Rust crate reedline to 0.29.0
* b946a04 - Update Rust crate clap to 4.5
* 39f1f3f - Update Rust crate indicatif to 0.17.8
* 4ea9387 - Update progenitor to bc0bb4b
* ace10f4 - Do not 500 on snapshot delete for deleted region
* 4105133 - Drop jobs from Offline downstairs.
* 43dace9 - `Mutex<Work>` → `Work`
* a1f3207 - Added a contributing.md
* 13b8669 - Remove ExtentFlushClose::source_downstairs
* 9b3f366 - Remove unnecessary mutexes from Downstairs
---
 .github/buildomat/jobs/package.sh |  4 ++--
 package-manifest.toml             | 29 +++++++++++++++++++++++------
 sled-agent/src/services.rs        | 22 ++++++++++++++--------
 3 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
index 8546a67a4f..13f374779c 100755
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -111,8 +111,8 @@ zones=(
   out/clickhouse.tar.gz
   out/clickhouse_keeper.tar.gz
   out/cockroachdb.tar.gz
-  out/crucible-pantry.tar.gz
-  out/crucible.tar.gz
+  out/crucible-pantry-zone.tar.gz
+  out/crucible-zone.tar.gz
   out/external-dns.tar.gz
   out/internal-dns.tar.gz
   out/omicron-nexus.tar.gz
diff --git a/package-manifest.toml b/package-manifest.toml
index 9b72dd7d18..d7f42794ee 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -389,13 +389,28 @@ source.packages = [ "sp-sim.tar.gz" ]
 output.type = "zone"
 output.intermediate_only = true
 
+[package.crucible-zone]
+service_name = "crucible"
+only_for_targets.image = "standard"
+source.type = "composite"
+source.packages = [ "crucible.tar.gz", "zone-network-setup.tar.gz" ]
+output.type = "zone"
+
+
+[package.crucible-pantry-zone]
+service_name = "crucible_pantry"
+only_for_targets.image = "standard"
+source.type = "composite"
+source.packages = [ "crucible-pantry.tar.gz", "zone-network-setup.tar.gz" ]
+output.type = "zone"
+
 # Packages not built within Omicron, but which must be imported.
 
 # Refer to
 #   https://github.com/oxidecomputer/crucible/blob/main/package/README.md
 # for instructions on building this manually.
 [package.crucible]
-service_name = "crucible"
+service_name = "crucible_prebuilt"
 only_for_targets.image = "standard"
 # To manually override the package source (for example, to test a change in
 # both Crucible and Omicron simultaneously):
@@ -405,22 +420,24 @@ only_for_targets.image = "standard"
 # 3. Use source.type = "manual" instead of "prebuilt"
 source.type = "prebuilt"
 source.repo = "crucible"
-source.commit = "796dce526dd7ed7b52a0429a486ccba4a9da1ce5"
+source.commit = "fe0c5c7909707a0f826025be4fe2bbf5f6e0206f"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image/<commit>/crucible.sha256.txt
-source.sha256 = "8b654627a4250e8d444133cf3130838d224b13e53f3e48cf0d031314d6f05ee0"
+source.sha256 = "5da4f93b16fc7c0f3cc3a67919dbaa3f143cc07b703183a236f5c98b61504d15"
 output.type = "zone"
+output.intermediate_only = true
 
 [package.crucible-pantry]
-service_name = "crucible_pantry"
+service_name = "crucible_pantry_prebuilt"
 only_for_targets.image = "standard"
 source.type = "prebuilt"
 source.repo = "crucible"
-source.commit = "796dce526dd7ed7b52a0429a486ccba4a9da1ce5"
+source.commit = "fe0c5c7909707a0f826025be4fe2bbf5f6e0206f"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image/<commit>/crucible-pantry.sha256.txt
-source.sha256 = "8602b2d6e7beb0731ae2be481715c94795657306d6013cc6d81fd60c4784a6ed"
+source.sha256 = "ed5027cc37c5ba4f2b9a568528f5bb49deedccaaa60bd770311c7bface6aa02b"
 output.type = "zone"
+output.intermediate_only = true
 
 # Refer to
 #   https://github.com/oxidecomputer/propolis/blob/master/package/README.md
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
index bc40187b38..bcd648cd2d 100644
--- a/sled-agent/src/services.rs
+++ b/sled-agent/src/services.rs
@@ -1708,11 +1708,15 @@ impl ServiceManager {
                 let Some(info) = self.inner.sled_info.get() else {
                     return Err(Error::SledAgentNotReady);
                 };
-                let datalink = installed_zone.get_control_vnic_name();
-                let gateway = &info.underlay_address.to_string();
                 let listen_addr = &underlay_address.to_string();
                 let listen_port = &CRUCIBLE_PORT.to_string();
 
+                let nw_setup_service = Self::zone_network_setup_install(
+                    info,
+                    &installed_zone,
+                    listen_addr,
+                )?;
+
                 let dataset_name = DatasetName::new(
                     dataset.pool_name.clone(),
                     DatasetKind::Crucible,
@@ -1720,8 +1724,6 @@ impl ServiceManager {
                 .full_name();
                 let uuid = &Uuid::new_v4().to_string();
                 let config = PropertyGroupBuilder::new("config")
-                    .add_property("datalink", "astring", datalink)
-                    .add_property("gateway", "astring", gateway)
                     .add_property("dataset", "astring", &dataset_name)
                     .add_property("listen_addr", "astring", listen_addr)
                     .add_property("listen_port", "astring", listen_port)
@@ -1729,6 +1731,7 @@ impl ServiceManager {
                     .add_property("store", "astring", "/data");
 
                 let profile = ProfileBuilder::new("omicron")
+                    .add_service(nw_setup_service)
                     .add_service(disabled_ssh_service)
                     .add_service(
                         ServiceBuilder::new("oxide/crucible/agent")
@@ -1759,18 +1762,21 @@ impl ServiceManager {
                     return Err(Error::SledAgentNotReady);
                 };
 
-                let datalink = installed_zone.get_control_vnic_name();
-                let gateway = &info.underlay_address.to_string();
                 let listen_addr = &underlay_address.to_string();
                 let listen_port = &CRUCIBLE_PANTRY_PORT.to_string();
 
+                let nw_setup_service = Self::zone_network_setup_install(
+                    info,
+                    &installed_zone,
+                    listen_addr,
+                )?;
+
                 let config = PropertyGroupBuilder::new("config")
-                    .add_property("datalink", "astring", datalink)
-                    .add_property("gateway", "astring", gateway)
                     .add_property("listen_addr", "astring", listen_addr)
                     .add_property("listen_port", "astring", listen_port);
 
                 let profile = ProfileBuilder::new("omicron")
+                    .add_service(nw_setup_service)
                     .add_service(disabled_ssh_service)
                     .add_service(
                         ServiceBuilder::new("oxide/crucible/pantry")

From 6993b410c04df7c9295720d6e68b297517be451b Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Wed, 21 Feb 2024 05:23:47 +0000
Subject: [PATCH 027/157] chore(deps): update taiki-e/install-action digest to
 12af778 (#5109)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [taiki-e/install-action](https://togithub.com/taiki-e/install-action)
| action | digest | [`6943331` ->
`12af778`](https://togithub.com/taiki-e/install-action/compare/6943331...12af778)
|

---

### Configuration

📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone
America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone
America/Los_Angeles.

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update
again.

---

- [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check
this box

---

This PR has been generated by [Renovate
Bot](https://togithub.com/renovatebot/renovate).

<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNy4yMDMuMSIsInVwZGF0ZWRJblZlciI6IjM3LjIwMy4xIiwidGFyZ2V0QnJhbmNoIjoibWFpbiJ9-->

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index 5927cb43e6..68bc323a07 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@6943331e01261cdff7420bbc2508cb463574e404 # v2
+        uses: taiki-e/install-action@12af778b97addf4c562c75a0564dc7e7dc5339a5 # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From 55ef8daa573669c69e9730ac6b18139f9a648698 Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Wed, 21 Feb 2024 17:13:36 -0500
Subject: [PATCH 028/157] Reconfigurator: Record external networking
 allocations when realizing a blueprint (#5045)

This PR expands blueprint execution to record external IPs and created
NICs for Nexus, Boundary NTP, and External DNS in crdb _before_ sending
new zone requests to sled-agents.

The implementation has a very obvious TOCTOU race, but I think it's okay
(as explained in the comments inline). If two Nexuses try to realize the
same blueprint simultaneously and both see no records present, only one
will succeed to insert, and the other will spuriously fail. Assuming
that failure causes a retry, subsequent attempts to realize the same
blueprint will succeed, as the required records will be present. If this
seems wrong, please holler!

I'd like to give this a spin on either madrid or one of the software
testbeds before merging, but I think this is ready for review.
---
 nexus/blueprint-execution/src/lib.rs          |  13 +-
 .../src/resource_allocation.rs                | 992 ++++++++++++++++++
 nexus/db-model/src/external_ip.rs             |  14 +-
 nexus/db-model/src/network_interface.rs       |   2 +-
 .../src/db/datastore/external_ip.rs           |  23 +-
 .../src/db/datastore/network_interface.rs     |  65 +-
 .../db-queries/src/db/queries/external_ip.rs  |  44 +
 7 files changed, 1140 insertions(+), 13 deletions(-)
 create mode 100644 nexus/blueprint-execution/src/resource_allocation.rs

diff --git a/nexus/blueprint-execution/src/lib.rs b/nexus/blueprint-execution/src/lib.rs
index a13acdf265..d6f5f8fc31 100644
--- a/nexus/blueprint-execution/src/lib.rs
+++ b/nexus/blueprint-execution/src/lib.rs
@@ -21,6 +21,7 @@ use uuid::Uuid;
 
 mod dns;
 mod omicron_zones;
+mod resource_allocation;
 
 struct Sled {
     id: Uuid,
@@ -69,6 +70,14 @@ where
         "blueprint_id" => ?blueprint.id
     );
 
+    resource_allocation::ensure_zone_resources_allocated(
+        &opctx,
+        datastore,
+        &blueprint.omicron_zones,
+    )
+    .await
+    .map_err(|err| vec![err])?;
+
     let sleds_by_id: BTreeMap<Uuid, _> = datastore
         .sled_list_all_batched(&opctx)
         .await
@@ -82,9 +91,9 @@ where
 
     dns::deploy_dns(
         &opctx,
-        &datastore,
+        datastore,
         String::from(nexus_label),
-        &blueprint,
+        blueprint,
         &sleds_by_id,
     )
     .await
diff --git a/nexus/blueprint-execution/src/resource_allocation.rs b/nexus/blueprint-execution/src/resource_allocation.rs
new file mode 100644
index 0000000000..7f3ebb9876
--- /dev/null
+++ b/nexus/blueprint-execution/src/resource_allocation.rs
@@ -0,0 +1,992 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Manges allocation of resources required for blueprint realization
+
+use anyhow::bail;
+use anyhow::Context;
+use nexus_db_model::IncompleteNetworkInterface;
+use nexus_db_model::Name;
+use nexus_db_model::SqlU16;
+use nexus_db_model::VpcSubnet;
+use nexus_db_queries::context::OpContext;
+use nexus_db_queries::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET;
+use nexus_db_queries::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET;
+use nexus_db_queries::db::fixed_data::vpc_subnet::NTP_VPC_SUBNET;
+use nexus_db_queries::db::DataStore;
+use nexus_types::deployment::NetworkInterface;
+use nexus_types::deployment::NetworkInterfaceKind;
+use nexus_types::deployment::OmicronZoneType;
+use nexus_types::deployment::OmicronZonesConfig;
+use nexus_types::deployment::SourceNatConfig;
+use omicron_common::api::external::IdentityMetadataCreateParams;
+use slog::info;
+use slog::warn;
+use std::collections::BTreeMap;
+use std::net::IpAddr;
+use std::net::SocketAddr;
+use uuid::Uuid;
+
+pub(crate) async fn ensure_zone_resources_allocated(
+    opctx: &OpContext,
+    datastore: &DataStore,
+    zones: &BTreeMap<Uuid, OmicronZonesConfig>,
+) -> anyhow::Result<()> {
+    let allocator = ResourceAllocator { opctx, datastore };
+
+    for config in zones.values() {
+        for z in &config.zones {
+            match &z.zone_type {
+                OmicronZoneType::Nexus { external_ip, nic, .. } => {
+                    allocator
+                        .ensure_nexus_external_networking_allocated(
+                            z.id,
+                            *external_ip,
+                            nic,
+                        )
+                        .await?;
+                }
+                OmicronZoneType::ExternalDns { dns_address, nic, .. } => {
+                    allocator
+                        .ensure_external_dns_external_networking_allocated(
+                            z.id,
+                            dns_address,
+                            nic,
+                        )
+                        .await?;
+                }
+                OmicronZoneType::BoundaryNtp { snat_cfg, nic, .. } => {
+                    allocator
+                        .ensure_boundary_ntp_external_networking_allocated(
+                            z.id, snat_cfg, nic,
+                        )
+                        .await?;
+                }
+                OmicronZoneType::InternalNtp { .. }
+                | OmicronZoneType::Clickhouse { .. }
+                | OmicronZoneType::ClickhouseKeeper { .. }
+                | OmicronZoneType::CockroachDb { .. }
+                | OmicronZoneType::Crucible { .. }
+                | OmicronZoneType::CruciblePantry { .. }
+                | OmicronZoneType::InternalDns { .. }
+                | OmicronZoneType::Oximeter { .. } => (),
+            }
+        }
+    }
+
+    Ok(())
+}
+
+struct ResourceAllocator<'a> {
+    opctx: &'a OpContext,
+    datastore: &'a DataStore,
+}
+
+impl<'a> ResourceAllocator<'a> {
+    // Helper function to determine whether a given external IP address is
+    // already allocated to a specific service zone.
+    async fn is_external_ip_already_allocated(
+        &self,
+        zone_type: &'static str,
+        zone_id: Uuid,
+        external_ip: IpAddr,
+        port_range: Option<(u16, u16)>,
+    ) -> anyhow::Result<bool> {
+        let allocated_ips = self
+            .datastore
+            .service_lookup_external_ips(self.opctx, zone_id)
+            .await
+            .with_context(|| {
+                format!(
+                    "failed to look up external IPs for {zone_type} {zone_id}"
+                )
+            })?;
+
+        if !allocated_ips.is_empty() {
+            // All the service zones that want external IP addresses only expect
+            // to have a single IP. This service already has (at least) one:
+            // make sure this list includes the one we want, or return an error.
+            for allocated_ip in &allocated_ips {
+                if allocated_ip.ip.ip() == external_ip
+                    && port_range
+                        .map(|(first, last)| {
+                            allocated_ip.first_port == SqlU16(first)
+                                && allocated_ip.last_port == SqlU16(last)
+                        })
+                        .unwrap_or(true)
+                {
+                    info!(
+                        self.opctx.log, "found already-allocated external IP";
+                        "zone_type" => zone_type,
+                        "zone_id" => %zone_id,
+                        "ip" => %external_ip,
+                    );
+                    return Ok(true);
+                }
+            }
+
+            warn!(
+                self.opctx.log, "zone has unexpected IPs allocated";
+                "zone_type" => zone_type,
+                "zone_id" => %zone_id,
+                "want_ip" => %external_ip,
+                "allocated_ips" => ?allocated_ips,
+            );
+            bail!(
+                "zone {zone_id} already has {} non-matching IP(s) allocated",
+                allocated_ips.len()
+            );
+        }
+
+        info!(
+            self.opctx.log, "external IP allocation required for zone";
+            "zone_type" => zone_type,
+            "zone_id" => %zone_id,
+            "ip" => %external_ip,
+        );
+
+        Ok(false)
+    }
+
+    // Helper function to determine whether a given NIC is already allocated to
+    // a specific service zone.
+    async fn is_nic_already_allocated(
+        &self,
+        zone_type: &'static str,
+        zone_id: Uuid,
+        nic: &NetworkInterface,
+    ) -> anyhow::Result<bool> {
+        let allocated_nics = self
+            .datastore
+            .service_list_network_interfaces(self.opctx, zone_id)
+            .await
+            .with_context(|| {
+                format!("failed to look up NICs for {zone_type} {zone_id}")
+            })?;
+
+        if !allocated_nics.is_empty() {
+            // All the service zones that want NICs only expect to have a single
+            // one. Bail out here if this zone already has one or more allocated
+            // NICs but not the one we think it needs.
+            //
+            // This doesn't check the allocated NIC's subnet against our NICs,
+            // because that would require an extra DB lookup. We'll assume if
+            // these main properties are correct, the subnet is too.
+            for allocated_nic in &allocated_nics {
+                if allocated_nic.ip.ip() == nic.ip
+                    && *allocated_nic.mac == nic.mac
+                    && allocated_nic.slot == i16::from(nic.slot)
+                    && allocated_nic.primary == nic.primary
+                {
+                    info!(
+                        self.opctx.log, "found already-allocated NIC";
+                        "zone_type" => zone_type,
+                        "zone_id" => %zone_id,
+                        "nic" => ?allocated_nic,
+                    );
+                    return Ok(true);
+                }
+            }
+
+            warn!(
+                self.opctx.log, "zone has unexpected NICs allocated";
+                "zone_type" => zone_type,
+                "zone_id" => %zone_id,
+                "want_nic" => ?nic,
+                "allocated_nics" => ?allocated_nics,
+            );
+
+            bail!(
+                "zone {zone_id} already has {} non-matching NIC(s) allocated",
+                allocated_nics.len()
+            );
+        }
+
+        info!(
+            self.opctx.log, "NIC allocation required for zone";
+            "zone_type" => zone_type,
+            "zone_id" => %zone_id,
+            "nid" => ?nic,
+        );
+
+        Ok(false)
+    }
+
+    // Nexus and ExternalDns both use non-SNAT service IPs; this method is used
+    // to allocate external networking for both of them.
+    async fn ensure_external_service_ip(
+        &self,
+        zone_type: &'static str,
+        service_id: Uuid,
+        external_ip: IpAddr,
+        ip_name: &Name,
+    ) -> anyhow::Result<()> {
+        // Only attempt to allocate `external_ip` if it isn't already assigned
+        // to this zone.
+        //
+        // Checking for the existing of the external IP and then creating it
+        // if not found inserts a classic TOCTOU race: what if another Nexus
+        // is running concurrently, we both check and see that the IP is not
+        // allocated, then both attempt to create it? We believe this is
+        // okay: the loser of the race (i.e., the one whose create tries to
+        // commit second) will fail to allocate the IP, which will bubble
+        // out and prevent realization of the current blueprint. That's
+        // exactly what we want if two Nexuses try to realize the same
+        // blueprint at the same time.
+        if self
+            .is_external_ip_already_allocated(
+                zone_type,
+                service_id,
+                external_ip,
+                None,
+            )
+            .await?
+        {
+            return Ok(());
+        }
+        let ip_id = Uuid::new_v4();
+        let description = zone_type;
+        self.datastore
+            .allocate_explicit_service_ip(
+                self.opctx,
+                ip_id,
+                ip_name,
+                description,
+                service_id,
+                external_ip,
+            )
+            .await
+            .with_context(|| {
+                format!(
+                    "failed to allocate IP to {zone_type} {service_id}: \
+                         {external_ip}"
+                )
+            })?;
+
+        info!(
+            self.opctx.log, "successfully allocated external IP";
+            "zone_type" => zone_type,
+            "zone_id" => %service_id,
+            "ip" => %external_ip,
+            "ip_id" => %ip_id,
+        );
+
+        Ok(())
+    }
+
+    // BoundaryNtp uses a SNAT service IPs; this method is similar to
+    // `ensure_external_service_ip` but accounts for that.
+    async fn ensure_external_service_snat_ip(
+        &self,
+        zone_type: &'static str,
+        service_id: Uuid,
+        snat: &SourceNatConfig,
+    ) -> anyhow::Result<()> {
+        // Only attempt to allocate `external_ip` if it isn't already assigned
+        // to this zone.
+        //
+        // This is subject to the same kind of TOCTOU race as described for IP
+        // allocation in `ensure_external_service_ip`, and we believe it's okay
+        // for the same reasons as described there.
+        if self
+            .is_external_ip_already_allocated(
+                zone_type,
+                service_id,
+                snat.ip,
+                Some((snat.first_port, snat.last_port)),
+            )
+            .await?
+        {
+            return Ok(());
+        }
+
+        let ip_id = Uuid::new_v4();
+        self.datastore
+            .allocate_explicit_service_snat_ip(
+                self.opctx,
+                ip_id,
+                service_id,
+                snat.ip,
+                (snat.first_port, snat.last_port),
+            )
+            .await
+            .with_context(|| {
+                format!(
+                    "failed to allocate snat IP to {zone_type} {service_id}: \
+                     {snat:?}"
+                )
+            })?;
+
+        info!(
+            self.opctx.log, "successfully allocated external SNAT IP";
+            "zone_type" => zone_type,
+            "zone_id" => %service_id,
+            "snat" => ?snat,
+            "ip_id" => %ip_id,
+        );
+
+        Ok(())
+    }
+
+    // All service zones with external connectivity get service vNICs.
+    async fn ensure_service_nic(
+        &self,
+        zone_type: &'static str,
+        service_id: Uuid,
+        nic: &NetworkInterface,
+        nic_subnet: &VpcSubnet,
+    ) -> anyhow::Result<()> {
+        // We don't pass `nic.kind` into the database below, but instead
+        // explicitly call `service_create_network_interface`. Ensure this is
+        // indeed a service NIC.
+        match &nic.kind {
+            NetworkInterfaceKind::Instance { .. } => {
+                bail!("invalid NIC kind (expected service, got instance)")
+            }
+            NetworkInterfaceKind::Service { .. } => (),
+        }
+
+        // Only attempt to allocate `nic` if it isn't already assigned to this
+        // zone.
+        //
+        // This is subject to the same kind of TOCTOU race as described for IP
+        // allocation in `ensure_external_service_ip`, and we believe it's okay
+        // for the same reasons as described there.
+        if self.is_nic_already_allocated(zone_type, service_id, nic).await? {
+            return Ok(());
+        }
+        let nic_arg = IncompleteNetworkInterface::new_service(
+            nic.id,
+            service_id,
+            nic_subnet.clone(),
+            IdentityMetadataCreateParams {
+                name: nic.name.clone(),
+                description: format!("{zone_type} service vNIC"),
+            },
+            nic.ip,
+            nic.mac,
+            nic.slot,
+        )
+        .with_context(|| {
+            format!(
+                "failed to convert NIC into IncompleteNetworkInterface: {nic:?}"
+            )
+        })?;
+        let created_nic = self
+            .datastore
+            .service_create_network_interface(self.opctx, nic_arg)
+            .await
+            .map_err(|err| err.into_external())
+            .with_context(|| {
+                format!(
+                    "failed to allocate NIC to {zone_type} {service_id}: \
+                     {nic:?}"
+                )
+            })?;
+
+        // We don't pass all the properties of `nic` into the create request
+        // above. Double-check that the properties the DB assigned match
+        // what we expect.
+        //
+        // We do not check `nic.vni`, because it's not stored in the
+        // database. (All services are given the constant vni
+        // `Vni::SERVICES_VNI`.)
+        if created_nic.primary != nic.primary
+            || created_nic.slot != i16::from(nic.slot)
+        {
+            warn!(
+                self.opctx.log, "unexpected property on allocated NIC";
+                "db_primary" => created_nic.primary,
+                "expected_primary" => nic.primary,
+                "db_slot" => created_nic.slot,
+                "expected_slot" => nic.slot,
+            );
+
+            // Now what? We've allocated a NIC in the database but it's
+            // incorrect. Should we try to delete it? That would be best
+            // effort (we could fail to delete, or we could crash between
+            // creation and deletion).
+            //
+            // We only expect services to have one NIC, so the only way it
+            // should be possible to get a different primary/slot value is
+            // if somehow this same service got a _different_ NIC allocated
+            // to it in the TOCTOU race window above. That should be
+            // impossible with the way we generate blueprints, so we'll just
+            // return a scary error here and expect to never see it.
+            bail!(
+                "database cleanup required: \
+                 unexpected NIC ({created_nic:?}) \
+                 allocated for {zone_type} {service_id}"
+            );
+        }
+
+        info!(
+            self.opctx.log, "successfully allocated service vNIC";
+            "zone_type" => zone_type,
+            "zone_id" => %service_id,
+            "nic" => ?nic,
+        );
+
+        Ok(())
+    }
+
+    async fn ensure_nexus_external_networking_allocated(
+        &self,
+        zone_id: Uuid,
+        external_ip: IpAddr,
+        nic: &NetworkInterface,
+    ) -> anyhow::Result<()> {
+        self.ensure_external_service_ip(
+            "nexus",
+            zone_id,
+            external_ip,
+            &Name(nic.name.clone()),
+        )
+        .await?;
+        self.ensure_service_nic("nexus", zone_id, nic, &NEXUS_VPC_SUBNET)
+            .await?;
+        Ok(())
+    }
+
+    async fn ensure_external_dns_external_networking_allocated(
+        &self,
+        zone_id: Uuid,
+        dns_address: &str,
+        nic: &NetworkInterface,
+    ) -> anyhow::Result<()> {
+        let dns_address =
+            dns_address.parse::<SocketAddr>().with_context(|| {
+                format!("failed to parse ExternalDns address {dns_address}")
+            })?;
+        self.ensure_external_service_ip(
+            "external_dns",
+            zone_id,
+            dns_address.ip(),
+            &Name(nic.name.clone()),
+        )
+        .await?;
+        self.ensure_service_nic("external_dns", zone_id, nic, &DNS_VPC_SUBNET)
+            .await?;
+        Ok(())
+    }
+
+    async fn ensure_boundary_ntp_external_networking_allocated(
+        &self,
+        zone_id: Uuid,
+        snat: &SourceNatConfig,
+        nic: &NetworkInterface,
+    ) -> anyhow::Result<()> {
+        self.ensure_external_service_snat_ip("ntp", zone_id, snat).await?;
+        self.ensure_service_nic("ntp", zone_id, nic, &NTP_VPC_SUBNET).await?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use nexus_test_utils_macros::nexus_test;
+    use nexus_types::deployment::OmicronZoneConfig;
+    use nexus_types::deployment::OmicronZoneDataset;
+    use nexus_types::identity::Resource;
+    use omicron_common::address::IpRange;
+    use omicron_common::address::DNS_OPTE_IPV4_SUBNET;
+    use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET;
+    use omicron_common::address::NTP_OPTE_IPV4_SUBNET;
+    use omicron_common::address::NUM_SOURCE_NAT_PORTS;
+    use omicron_common::api::external::Generation;
+    use omicron_common::api::external::IpNet;
+    use omicron_common::api::external::MacAddr;
+    use omicron_common::api::external::Vni;
+    use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
+    use std::net::IpAddr;
+    use std::net::Ipv6Addr;
+    use std::net::SocketAddrV6;
+
+    type ControlPlaneTestContext =
+        nexus_test_utils::ControlPlaneTestContext<omicron_nexus::Server>;
+
+    #[nexus_test]
+    async fn test_allocate_external_networking(
+        cptestctx: &ControlPlaneTestContext,
+    ) {
+        // Set up.
+        let nexus = &cptestctx.server.apictx().nexus;
+        let datastore = nexus.datastore();
+        let opctx = OpContext::for_tests(
+            cptestctx.logctx.log.clone(),
+            datastore.clone(),
+        );
+
+        // Create an external IP range we can use for our services.
+        let external_ip_range = IpRange::try_from((
+            "192.0.2.1".parse::<IpAddr>().unwrap(),
+            "192.0.2.100".parse::<IpAddr>().unwrap(),
+        ))
+        .expect("bad IP range");
+        let mut external_ips = external_ip_range.iter();
+
+        // Add the external IP range to the services IP pool.
+        let (ip_pool, _) = datastore
+            .ip_pools_service_lookup(&opctx)
+            .await
+            .expect("failed to find service IP pool");
+        datastore
+            .ip_pool_add_range(&opctx, &ip_pool, &external_ip_range)
+            .await
+            .expect("failed to expand service IP pool");
+
+        // Generate the values we care about. (Other required zone config params
+        // that we don't care about will be filled in below arbitrarily.)
+
+        // Nexus:
+        let nexus_id = Uuid::new_v4();
+        let nexus_external_ip =
+            external_ips.next().expect("exhausted external_ips");
+        let nexus_nic = NetworkInterface {
+            id: Uuid::new_v4(),
+            kind: NetworkInterfaceKind::Service(nexus_id),
+            name: "test-nexus".parse().expect("bad name"),
+            ip: NEXUS_OPTE_IPV4_SUBNET
+                .iter()
+                .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES)
+                .unwrap()
+                .into(),
+            mac: MacAddr::random_system(),
+            subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET).into(),
+            vni: Vni::SERVICES_VNI,
+            primary: true,
+            slot: 0,
+        };
+
+        // External DNS:
+        let dns_id = Uuid::new_v4();
+        let dns_external_ip =
+            external_ips.next().expect("exhausted external_ips");
+        let dns_nic = NetworkInterface {
+            id: Uuid::new_v4(),
+            kind: NetworkInterfaceKind::Service(dns_id),
+            name: "test-external-dns".parse().expect("bad name"),
+            ip: DNS_OPTE_IPV4_SUBNET
+                .iter()
+                .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES)
+                .unwrap()
+                .into(),
+            mac: MacAddr::random_system(),
+            subnet: IpNet::from(*DNS_OPTE_IPV4_SUBNET).into(),
+            vni: Vni::SERVICES_VNI,
+            primary: true,
+            slot: 0,
+        };
+
+        // Boundary NTP:
+        let ntp_id = Uuid::new_v4();
+        let ntp_snat = SourceNatConfig {
+            ip: external_ips.next().expect("exhausted external_ips"),
+            first_port: NUM_SOURCE_NAT_PORTS,
+            last_port: 2 * NUM_SOURCE_NAT_PORTS - 1,
+        };
+        let ntp_nic = NetworkInterface {
+            id: Uuid::new_v4(),
+            kind: NetworkInterfaceKind::Service(ntp_id),
+            name: "test-external-ntp".parse().expect("bad name"),
+            ip: NTP_OPTE_IPV4_SUBNET
+                .iter()
+                .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES)
+                .unwrap()
+                .into(),
+            mac: MacAddr::random_system(),
+            subnet: IpNet::from(*NTP_OPTE_IPV4_SUBNET).into(),
+            vni: Vni::SERVICES_VNI,
+            primary: true,
+            slot: 0,
+        };
+
+        // Build the `zones` map needed by `ensure_zone_resources_allocated`,
+        // with an arbitrary sled_id.
+        let mut zones = BTreeMap::new();
+        let sled_id = Uuid::new_v4();
+        zones.insert(
+            sled_id,
+            OmicronZonesConfig {
+                generation: Generation::new().next(),
+                zones: vec![
+                    OmicronZoneConfig {
+                        id: nexus_id,
+                        underlay_address: Ipv6Addr::LOCALHOST,
+                        zone_type: OmicronZoneType::Nexus {
+                            internal_address: Ipv6Addr::LOCALHOST.to_string(),
+                            external_ip: nexus_external_ip,
+                            nic: nexus_nic.clone(),
+                            external_tls: false,
+                            external_dns_servers: Vec::new(),
+                        },
+                    },
+                    OmicronZoneConfig {
+                        id: dns_id,
+                        underlay_address: Ipv6Addr::LOCALHOST,
+                        zone_type: OmicronZoneType::ExternalDns {
+                            dataset: OmicronZoneDataset {
+                                pool_name: format!("oxp_{}", Uuid::new_v4())
+                                    .parse()
+                                    .expect("bad name"),
+                            },
+                            http_address: SocketAddrV6::new(
+                                Ipv6Addr::LOCALHOST,
+                                0,
+                                0,
+                                0,
+                            )
+                            .to_string(),
+                            dns_address: SocketAddr::new(dns_external_ip, 0)
+                                .to_string(),
+                            nic: dns_nic.clone(),
+                        },
+                    },
+                    OmicronZoneConfig {
+                        id: ntp_id,
+                        underlay_address: Ipv6Addr::LOCALHOST,
+                        zone_type: OmicronZoneType::BoundaryNtp {
+                            address: SocketAddr::new(dns_external_ip, 0)
+                                .to_string(),
+                            ntp_servers: Vec::new(),
+                            dns_servers: Vec::new(),
+                            domain: None,
+                            nic: ntp_nic.clone(),
+                            snat_cfg: ntp_snat,
+                        },
+                    },
+                ],
+            },
+        );
+
+        // Initialize resource allocation: this should succeed and create all
+        // the relevant db records.
+        ensure_zone_resources_allocated(&opctx, datastore, &zones)
+            .await
+            .with_context(|| format!("{zones:#?}"))
+            .unwrap();
+
+        // Check that the external IP records were created.
+        let db_nexus_ips = datastore
+            .service_lookup_external_ips(&opctx, nexus_id)
+            .await
+            .expect("failed to get external IPs");
+        assert_eq!(db_nexus_ips.len(), 1);
+        assert!(db_nexus_ips[0].is_service);
+        assert_eq!(db_nexus_ips[0].parent_id, Some(nexus_id));
+        assert_eq!(db_nexus_ips[0].ip, nexus_external_ip.into());
+        assert_eq!(db_nexus_ips[0].first_port, SqlU16(0));
+        assert_eq!(db_nexus_ips[0].last_port, SqlU16(65535));
+
+        let db_dns_ips = datastore
+            .service_lookup_external_ips(&opctx, dns_id)
+            .await
+            .expect("failed to get external IPs");
+        assert_eq!(db_dns_ips.len(), 1);
+        assert!(db_dns_ips[0].is_service);
+        assert_eq!(db_dns_ips[0].parent_id, Some(dns_id));
+        assert_eq!(db_dns_ips[0].ip, dns_external_ip.into());
+        assert_eq!(db_dns_ips[0].first_port, SqlU16(0));
+        assert_eq!(db_dns_ips[0].last_port, SqlU16(65535));
+
+        let db_ntp_ips = datastore
+            .service_lookup_external_ips(&opctx, ntp_id)
+            .await
+            .expect("failed to get external IPs");
+        assert_eq!(db_ntp_ips.len(), 1);
+        assert!(db_ntp_ips[0].is_service);
+        assert_eq!(db_ntp_ips[0].parent_id, Some(ntp_id));
+        assert_eq!(db_ntp_ips[0].ip, ntp_snat.ip.into());
+        assert_eq!(db_ntp_ips[0].first_port, SqlU16(ntp_snat.first_port));
+        assert_eq!(db_ntp_ips[0].last_port, SqlU16(ntp_snat.last_port));
+
+        // Check that the NIC records were created.
+        let db_nexus_nics = datastore
+            .service_list_network_interfaces(&opctx, nexus_id)
+            .await
+            .expect("failed to get NICs");
+        assert_eq!(db_nexus_nics.len(), 1);
+        assert_eq!(db_nexus_nics[0].id(), nexus_nic.id);
+        assert_eq!(db_nexus_nics[0].service_id, nexus_id);
+        assert_eq!(db_nexus_nics[0].vpc_id, NEXUS_VPC_SUBNET.vpc_id);
+        assert_eq!(db_nexus_nics[0].subnet_id, NEXUS_VPC_SUBNET.id());
+        assert_eq!(*db_nexus_nics[0].mac, nexus_nic.mac);
+        assert_eq!(db_nexus_nics[0].ip, nexus_nic.ip.into());
+        assert_eq!(db_nexus_nics[0].slot, i16::from(nexus_nic.slot));
+        assert_eq!(db_nexus_nics[0].primary, nexus_nic.primary);
+
+        let db_dns_nics = datastore
+            .service_list_network_interfaces(&opctx, dns_id)
+            .await
+            .expect("failed to get NICs");
+        assert_eq!(db_dns_nics.len(), 1);
+        assert_eq!(db_dns_nics[0].id(), dns_nic.id);
+        assert_eq!(db_dns_nics[0].service_id, dns_id);
+        assert_eq!(db_dns_nics[0].vpc_id, DNS_VPC_SUBNET.vpc_id);
+        assert_eq!(db_dns_nics[0].subnet_id, DNS_VPC_SUBNET.id());
+        assert_eq!(*db_dns_nics[0].mac, dns_nic.mac);
+        assert_eq!(db_dns_nics[0].ip, dns_nic.ip.into());
+        assert_eq!(db_dns_nics[0].slot, i16::from(dns_nic.slot));
+        assert_eq!(db_dns_nics[0].primary, dns_nic.primary);
+
+        let db_ntp_nics = datastore
+            .service_list_network_interfaces(&opctx, ntp_id)
+            .await
+            .expect("failed to get NICs");
+        assert_eq!(db_ntp_nics.len(), 1);
+        assert_eq!(db_ntp_nics[0].id(), ntp_nic.id);
+        assert_eq!(db_ntp_nics[0].service_id, ntp_id);
+        assert_eq!(db_ntp_nics[0].vpc_id, NTP_VPC_SUBNET.vpc_id);
+        assert_eq!(db_ntp_nics[0].subnet_id, NTP_VPC_SUBNET.id());
+        assert_eq!(*db_ntp_nics[0].mac, ntp_nic.mac);
+        assert_eq!(db_ntp_nics[0].ip, ntp_nic.ip.into());
+        assert_eq!(db_ntp_nics[0].slot, i16::from(ntp_nic.slot));
+        assert_eq!(db_ntp_nics[0].primary, ntp_nic.primary);
+
+        // We should be able to run the function again with the same inputs, and
+        // it should succeed without inserting any new records.
+        ensure_zone_resources_allocated(&opctx, datastore, &zones)
+            .await
+            .with_context(|| format!("{zones:#?}"))
+            .unwrap();
+        assert_eq!(
+            db_nexus_ips,
+            datastore
+                .service_lookup_external_ips(&opctx, nexus_id)
+                .await
+                .expect("failed to get external IPs")
+        );
+        assert_eq!(
+            db_dns_ips,
+            datastore
+                .service_lookup_external_ips(&opctx, dns_id)
+                .await
+                .expect("failed to get external IPs")
+        );
+        assert_eq!(
+            db_ntp_ips,
+            datastore
+                .service_lookup_external_ips(&opctx, ntp_id)
+                .await
+                .expect("failed to get external IPs")
+        );
+        assert_eq!(
+            db_nexus_nics,
+            datastore
+                .service_list_network_interfaces(&opctx, nexus_id)
+                .await
+                .expect("failed to get NICs")
+        );
+        assert_eq!(
+            db_dns_nics,
+            datastore
+                .service_list_network_interfaces(&opctx, dns_id)
+                .await
+                .expect("failed to get NICs")
+        );
+        assert_eq!(
+            db_ntp_nics,
+            datastore
+                .service_list_network_interfaces(&opctx, ntp_id)
+                .await
+                .expect("failed to get NICs")
+        );
+
+        // Now that we've tested the happy path, try some requests that ought to
+        // fail because the request includes an external IP that doesn't match
+        // the already-allocated external IPs from above.
+        let bogus_ip = external_ips.next().expect("exhausted external_ips");
+        for mutate_zones_fn in [
+            // non-matching IP on Nexus
+            (&|config: &mut OmicronZonesConfig| {
+                for zone in &mut config.zones {
+                    if let OmicronZoneType::Nexus {
+                        ref mut external_ip, ..
+                    } = &mut zone.zone_type
+                    {
+                        *external_ip = bogus_ip;
+                        return format!(
+                            "zone {} already has 1 non-matching IP",
+                            zone.id
+                        );
+                    }
+                }
+                panic!("didn't find expected zone");
+            }) as &dyn Fn(&mut OmicronZonesConfig) -> String,
+            // non-matching IP on External DNS
+            &|config| {
+                for zone in &mut config.zones {
+                    if let OmicronZoneType::ExternalDns {
+                        ref mut dns_address,
+                        ..
+                    } = &mut zone.zone_type
+                    {
+                        *dns_address = SocketAddr::new(bogus_ip, 0).to_string();
+                        return format!(
+                            "zone {} already has 1 non-matching IP",
+                            zone.id
+                        );
+                    }
+                }
+                panic!("didn't find expected zone");
+            },
+            // non-matching SNAT port range on Boundary NTP
+            &|config| {
+                for zone in &mut config.zones {
+                    if let OmicronZoneType::BoundaryNtp {
+                        ref mut snat_cfg,
+                        ..
+                    } = &mut zone.zone_type
+                    {
+                        snat_cfg.first_port += NUM_SOURCE_NAT_PORTS;
+                        snat_cfg.last_port += NUM_SOURCE_NAT_PORTS;
+                        return format!(
+                            "zone {} already has 1 non-matching IP",
+                            zone.id
+                        );
+                    }
+                }
+                panic!("didn't find expected zone");
+            },
+        ] {
+            // Run `mutate_zones_fn` on our config...
+            let mut config =
+                zones.remove(&sled_id).expect("missing zone config");
+            let orig_config = config.clone();
+            let expected_error = mutate_zones_fn(&mut config);
+            zones.insert(sled_id, config);
+
+            // ... check that we get the error we expect
+            let err =
+                ensure_zone_resources_allocated(&opctx, datastore, &zones)
+                    .await
+                    .expect_err("unexpected success");
+            assert!(
+                err.to_string().contains(&expected_error),
+                "expected {expected_error:?}, got {err:#}"
+            );
+
+            // ... and restore the original, valid config before iterating.
+            zones.insert(sled_id, orig_config);
+        }
+
+        // Also try some requests that ought to fail because the request
+        // includes a NIC that doesn't match the already-allocated NICs from
+        // above.
+        //
+        // All three zone types have a `nic` property, so here our mutating
+        // function only modifies that, and the body of our loop tries it on all
+        // three to ensure we get the errors we expect no matter the zone type.
+        for mutate_nic_fn in [
+            // switch kind from Service to Instance
+            (&|_: Uuid, nic: &mut NetworkInterface| {
+                match &nic.kind {
+                    NetworkInterfaceKind::Instance { .. } => {
+                        panic!(
+                            "invalid NIC kind (expected service, got instance)"
+                        )
+                    }
+                    NetworkInterfaceKind::Service(id) => {
+                        let id = *id;
+                        nic.kind = NetworkInterfaceKind::Instance(id);
+                    }
+                }
+                "invalid NIC kind".to_string()
+            }) as &dyn Fn(Uuid, &mut NetworkInterface) -> String,
+            // non-matching IP
+            &|zone_id, nic| {
+                nic.ip = bogus_ip;
+                format!("zone {zone_id} already has 1 non-matching NIC")
+            },
+        ] {
+            // Try this NIC mutation on Nexus...
+            let mut mutated_zones = zones.clone();
+            for zone in &mut mutated_zones
+                .get_mut(&sled_id)
+                .expect("missing sled")
+                .zones
+            {
+                if let OmicronZoneType::Nexus { ref mut nic, .. } =
+                    &mut zone.zone_type
+                {
+                    let expected_error = mutate_nic_fn(zone.id, nic);
+
+                    let err = ensure_zone_resources_allocated(
+                        &opctx,
+                        datastore,
+                        &mutated_zones,
+                    )
+                    .await
+                    .expect_err("unexpected success");
+
+                    assert!(
+                        err.to_string().contains(&expected_error),
+                        "expected {expected_error:?}, got {err:#}"
+                    );
+
+                    break;
+                }
+            }
+
+            // ... and again on ExternalDns
+            let mut mutated_zones = zones.clone();
+            for zone in &mut mutated_zones
+                .get_mut(&sled_id)
+                .expect("missing sled")
+                .zones
+            {
+                if let OmicronZoneType::ExternalDns { ref mut nic, .. } =
+                    &mut zone.zone_type
+                {
+                    let expected_error = mutate_nic_fn(zone.id, nic);
+
+                    let err = ensure_zone_resources_allocated(
+                        &opctx,
+                        datastore,
+                        &mutated_zones,
+                    )
+                    .await
+                    .expect_err("unexpected success");
+
+                    assert!(
+                        err.to_string().contains(&expected_error),
+                        "expected {expected_error:?}, got {err:#}"
+                    );
+
+                    break;
+                }
+            }
+
+            // ... and again on BoundaryNtp
+            let mut mutated_zones = zones.clone();
+            for zone in &mut mutated_zones
+                .get_mut(&sled_id)
+                .expect("missing sled")
+                .zones
+            {
+                if let OmicronZoneType::BoundaryNtp { ref mut nic, .. } =
+                    &mut zone.zone_type
+                {
+                    let expected_error = mutate_nic_fn(zone.id, nic);
+
+                    let err = ensure_zone_resources_allocated(
+                        &opctx,
+                        datastore,
+                        &mutated_zones,
+                    )
+                    .await
+                    .expect_err("unexpected success");
+
+                    assert!(
+                        err.to_string().contains(&expected_error),
+                        "expected {expected_error:?}, got {err:#}"
+                    );
+
+                    break;
+                }
+            }
+        }
+    }
+}
diff --git a/nexus/db-model/src/external_ip.rs b/nexus/db-model/src/external_ip.rs
index 1e9def4182..b30f91c7c0 100644
--- a/nexus/db-model/src/external_ip.rs
+++ b/nexus/db-model/src/external_ip.rs
@@ -33,7 +33,7 @@ impl_enum_type!(
     #[diesel(postgres_type(name = "ip_kind", schema = "public"))]
      pub struct IpKindEnum;
 
-     #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Deserialize, Serialize)]
+     #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Deserialize, Serialize)]
      #[diesel(sql_type = IpKindEnum)]
      pub enum IpKind;
 
@@ -47,7 +47,7 @@ impl_enum_type!(
     #[diesel(postgres_type(name = "ip_attach_state"))]
      pub struct IpAttachStateEnum;
 
-     #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Deserialize, Serialize)]
+     #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Deserialize, Serialize)]
      #[diesel(sql_type = IpAttachStateEnum)]
      pub enum IpAttachState;
 
@@ -89,7 +89,15 @@ impl std::fmt::Display for IpKind {
 /// API at all, and only provide outbound connectivity to instances, not
 /// inbound.
 #[derive(
-    Debug, Clone, Selectable, Queryable, Insertable, Deserialize, Serialize,
+    Debug,
+    Clone,
+    Selectable,
+    Queryable,
+    Insertable,
+    Deserialize,
+    Serialize,
+    PartialEq,
+    Eq,
 )]
 #[diesel(table_name = external_ip)]
 pub struct ExternalIp {
diff --git a/nexus/db-model/src/network_interface.rs b/nexus/db-model/src/network_interface.rs
index 01317fc160..72752ae3f8 100644
--- a/nexus/db-model/src/network_interface.rs
+++ b/nexus/db-model/src/network_interface.rs
@@ -91,7 +91,7 @@ pub struct InstanceNetworkInterface {
 /// The underlying "table" (`service_network_interface`) is actually a view
 /// over the `network_interface` table, that contains only rows with
 /// `kind = 'service'`.
-#[derive(Selectable, Queryable, Clone, Debug, Resource)]
+#[derive(Selectable, Queryable, Clone, Debug, PartialEq, Eq, Resource)]
 #[diesel(table_name = service_network_interface)]
 pub struct ServiceNetworkInterface {
     #[diesel(embed)]
diff --git a/nexus/db-queries/src/db/datastore/external_ip.rs b/nexus/db-queries/src/db/datastore/external_ip.rs
index 9d4d947476..24439aa3a0 100644
--- a/nexus/db-queries/src/db/datastore/external_ip.rs
+++ b/nexus/db-queries/src/db/datastore/external_ip.rs
@@ -167,6 +167,23 @@ impl DataStore {
         }
     }
 
+    /// Fetch all external IP addresses of any kind for the provided service.
+    pub async fn service_lookup_external_ips(
+        &self,
+        opctx: &OpContext,
+        service_id: Uuid,
+    ) -> LookupResult<Vec<ExternalIp>> {
+        use db::schema::external_ip::dsl;
+        dsl::external_ip
+            .filter(dsl::is_service.eq(true))
+            .filter(dsl::parent_id.eq(service_id))
+            .filter(dsl::time_deleted.is_null())
+            .select(ExternalIp::as_select())
+            .get_results_async(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
+    }
+
     /// Allocates an IP address for internal service usage.
     pub async fn allocate_service_ip(
         &self,
@@ -337,7 +354,8 @@ impl DataStore {
         service_id: Uuid,
         ip: IpAddr,
     ) -> CreateResult<ExternalIp> {
-        let (.., pool) = self.ip_pools_service_lookup(opctx).await?;
+        let (authz_pool, pool) = self.ip_pools_service_lookup(opctx).await?;
+        opctx.authorize(authz::Action::CreateChild, &authz_pool).await?;
         let data = IncompleteExternalIp::for_service_explicit(
             ip_id,
             name,
@@ -361,7 +379,8 @@ impl DataStore {
         ip: IpAddr,
         port_range: (u16, u16),
     ) -> CreateResult<ExternalIp> {
-        let (.., pool) = self.ip_pools_service_lookup(opctx).await?;
+        let (authz_pool, pool) = self.ip_pools_service_lookup(opctx).await?;
+        opctx.authorize(authz::Action::CreateChild, &authz_pool).await?;
         let data = IncompleteExternalIp::for_service_explicit_snat(
             ip_id,
             service_id,
diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs
index d715bf3889..f2782e8f67 100644
--- a/nexus/db-queries/src/db/datastore/network_interface.rs
+++ b/nexus/db-queries/src/db/datastore/network_interface.rs
@@ -29,6 +29,7 @@ use async_bb8_diesel::AsyncRunQueryDsl;
 use chrono::Utc;
 use diesel::prelude::*;
 use diesel::result::Error as DieselError;
+use nexus_db_model::ServiceNetworkInterface;
 use omicron_common::api::external;
 use omicron_common::api::external::http_pagination::PaginatedBy;
 use omicron_common::api::external::DeleteResult;
@@ -126,15 +127,69 @@ impl DataStore {
             .map(NetworkInterface::as_instance)
     }
 
-    #[cfg(test)]
+    /// List network interfaces associated with a given service.
+    pub async fn service_list_network_interfaces(
+        &self,
+        opctx: &OpContext,
+        service_id: Uuid,
+    ) -> ListResultVec<ServiceNetworkInterface> {
+        // See the comment in `service_create_network_interface`. There's no
+        // obvious parent for a service network interface (as opposed to
+        // instance network interfaces, which require ListChildren on the
+        // instance to list). As a logical proxy, we check for listing children
+        // of the service IP pool.
+        let (authz_service_ip_pool, _) =
+            self.ip_pools_service_lookup(opctx).await?;
+        opctx
+            .authorize(authz::Action::ListChildren, &authz_service_ip_pool)
+            .await?;
+
+        use db::schema::service_network_interface::dsl;
+        dsl::service_network_interface
+            .filter(dsl::time_deleted.is_null())
+            .filter(dsl::service_id.eq(service_id))
+            .select(ServiceNetworkInterface::as_select())
+            .get_results_async::<ServiceNetworkInterface>(
+                &*self.pool_connection_authorized(opctx).await?,
+            )
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
+    }
+
+    /// Create a network interface attached to the provided service zone.
+    pub async fn service_create_network_interface(
+        &self,
+        opctx: &OpContext,
+        interface: IncompleteNetworkInterface,
+    ) -> Result<ServiceNetworkInterface, network_interface::InsertError> {
+        // In `instance_create_network_interface`, the authz checks are for
+        // creating children of the VpcSubnet and the instance. We don't have an
+        // instance. We do have a VpcSubet, but for services these are all
+        // fixed data subnets.
+        //
+        // As a proxy auth check that isn't really guarding the right resource
+        // but should logically be equivalent, we can insert a authz check for
+        // creating children of the service IP pool. For any service zone with
+        // external networking, we create an external IP (in the service IP
+        // pool) and a network interface (in the relevant VpcSubnet). Putting
+        // this check here ensures that the caller can't proceed if they also
+        // couldn't proceed with creating the corresponding external IP.
+        let (authz_service_ip_pool, _) = self
+            .ip_pools_service_lookup(opctx)
+            .await
+            .map_err(network_interface::InsertError::External)?;
+        opctx
+            .authorize(authz::Action::CreateChild, &authz_service_ip_pool)
+            .await
+            .map_err(network_interface::InsertError::External)?;
+        self.service_create_network_interface_raw(opctx, interface).await
+    }
+
     pub(crate) async fn service_create_network_interface_raw(
         &self,
         opctx: &OpContext,
         interface: IncompleteNetworkInterface,
-    ) -> Result<
-        db::model::ServiceNetworkInterface,
-        network_interface::InsertError,
-    > {
+    ) -> Result<ServiceNetworkInterface, network_interface::InsertError> {
         if interface.kind != NetworkInterfaceKind::Service {
             return Err(network_interface::InsertError::External(
                 Error::invalid_request(
diff --git a/nexus/db-queries/src/db/queries/external_ip.rs b/nexus/db-queries/src/db/queries/external_ip.rs
index 54595a8444..7456e6e6bb 100644
--- a/nexus/db-queries/src/db/queries/external_ip.rs
+++ b/nexus/db-queries/src/db/queries/external_ip.rs
@@ -1331,6 +1331,18 @@ mod tests {
         // Allocate an IP address as we would for an external, rack-associated
         // service.
         let service1_id = Uuid::new_v4();
+
+        // Check that `service_lookup_external_ips` returns an empty vector for
+        // a service with no external IPs.
+        assert_eq!(
+            context
+                .db_datastore
+                .service_lookup_external_ips(&context.opctx, service1_id)
+                .await
+                .expect("Failed to look up service external IPs"),
+            Vec::new(),
+        );
+
         let id1 = Uuid::new_v4();
         let ip1 = context
             .db_datastore
@@ -1349,6 +1361,14 @@ mod tests {
         assert_eq!(ip1.first_port.0, 0);
         assert_eq!(ip1.last_port.0, u16::MAX);
         assert_eq!(ip1.parent_id, Some(service1_id));
+        assert_eq!(
+            context
+                .db_datastore
+                .service_lookup_external_ips(&context.opctx, service1_id)
+                .await
+                .expect("Failed to look up service external IPs"),
+            vec![ip1],
+        );
 
         // Allocate an SNat IP
         let service2_id = Uuid::new_v4();
@@ -1364,6 +1384,14 @@ mod tests {
         assert_eq!(ip2.first_port.0, 0);
         assert_eq!(ip2.last_port.0, 16383);
         assert_eq!(ip2.parent_id, Some(service2_id));
+        assert_eq!(
+            context
+                .db_datastore
+                .service_lookup_external_ips(&context.opctx, service2_id)
+                .await
+                .expect("Failed to look up service external IPs"),
+            vec![ip2],
+        );
 
         // Allocate the next IP address
         let service3_id = Uuid::new_v4();
@@ -1385,6 +1413,14 @@ mod tests {
         assert_eq!(ip3.first_port.0, 0);
         assert_eq!(ip3.last_port.0, u16::MAX);
         assert_eq!(ip3.parent_id, Some(service3_id));
+        assert_eq!(
+            context
+                .db_datastore
+                .service_lookup_external_ips(&context.opctx, service3_id)
+                .await
+                .expect("Failed to look up service external IPs"),
+            vec![ip3],
+        );
 
         // Once we're out of IP addresses, test that we see the right error.
         let service3_id = Uuid::new_v4();
@@ -1422,6 +1458,14 @@ mod tests {
         assert_eq!(ip4.first_port.0, 16384);
         assert_eq!(ip4.last_port.0, 32767);
         assert_eq!(ip4.parent_id, Some(service4_id));
+        assert_eq!(
+            context
+                .db_datastore
+                .service_lookup_external_ips(&context.opctx, service4_id)
+                .await
+                .expect("Failed to look up service external IPs"),
+            vec![ip4],
+        );
 
         context.success().await;
     }

From 4de45c8e8138ab50394b8fe0290791d8c707f48e Mon Sep 17 00:00:00 2001
From: David Pacheco <dap@oxidecomputer.com>
Date: Wed, 21 Feb 2024 17:26:10 -0800
Subject: [PATCH 029/157] "expensive operation" check was too aggressive
 (#5113)

---
 nexus/db-queries/src/context.rs | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/nexus/db-queries/src/context.rs b/nexus/db-queries/src/context.rs
index 72da01a5f1..aea4a60e66 100644
--- a/nexus/db-queries/src/context.rs
+++ b/nexus/db-queries/src/context.rs
@@ -303,10 +303,17 @@ impl OpContext {
     ///   either.  Clients and proxies often don't expect long requests and
     ///   apply aggressive timeouts.  Depending on the HTTP version, a
     ///   long-running request can tie up the TCP connection.
+    ///
+    /// We shouldn't allow these in either internal or external API handlers,
+    /// but we currently have some internal APIs for exercising some expensive
+    /// blueprint operations and so we allow these cases here.
     pub fn check_complex_operations_allowed(&self) -> Result<(), Error> {
         let api_handler = match self.kind {
-            OpKind::ExternalApiRequest | OpKind::InternalApiRequest => true,
-            OpKind::Saga | OpKind::Background | OpKind::Test => false,
+            OpKind::ExternalApiRequest => true,
+            OpKind::InternalApiRequest
+            | OpKind::Saga
+            | OpKind::Background
+            | OpKind::Test => false,
         };
         if api_handler {
             Err(Error::internal_error(

From 828c79c10e75d7529d13836b6fcc08a155b6fe59 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Thu, 22 Feb 2024 05:11:33 +0000
Subject: [PATCH 030/157] chore(deps): update taiki-e/install-action digest to
 875aa7b (#5114)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [taiki-e/install-action](https://togithub.com/taiki-e/install-action)
| action | digest | [`12af778` ->
`875aa7b`](https://togithub.com/taiki-e/install-action/compare/12af778...875aa7b)
|

---

### Configuration

📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone
America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone
America/Los_Angeles.

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update
again.

---

- [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check
this box

---

This PR has been generated by [Renovate
Bot](https://togithub.com/renovatebot/renovate).

<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNy4yMDQuMiIsInVwZGF0ZWRJblZlciI6IjM3LjIwNC4yIiwidGFyZ2V0QnJhbmNoIjoibWFpbiJ9-->

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index 68bc323a07..dc907e93e7 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@12af778b97addf4c562c75a0564dc7e7dc5339a5 # v2
+        uses: taiki-e/install-action@875aa7bb88c61a90732e8d159a915df57f27e475 # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From 42844311eb688e63ea23361dfb4279b44e65f792 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Thu, 22 Feb 2024 07:28:45 +0000
Subject: [PATCH 031/157] chore(deps): update taiki-e/install-action digest to
 19e9b54 (#5117)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [taiki-e/install-action](https://togithub.com/taiki-e/install-action)
| action | digest | [`875aa7b` ->
`19e9b54`](https://togithub.com/taiki-e/install-action/compare/875aa7b...19e9b54)
|

---

### Configuration

📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone
America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone
America/Los_Angeles.

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update
again.

---

- [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check
this box

---

This PR has been generated by [Renovate
Bot](https://togithub.com/renovatebot/renovate).

<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNy4yMDYuMCIsInVwZGF0ZWRJblZlciI6IjM3LjIwNi4wIiwidGFyZ2V0QnJhbmNoIjoibWFpbiJ9-->

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index dc907e93e7..1f55f2f255 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@875aa7bb88c61a90732e8d159a915df57f27e475 # v2
+        uses: taiki-e/install-action@19e9b549a48620cc50fcf6e6e866b8fb4eca1b01 # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From dea0ea5a1e26464dc9780ae908f028d210f5f637 Mon Sep 17 00:00:00 2001
From: Laura Abbott <labbott@users.noreply.github.com>
Date: Thu, 22 Feb 2024 09:25:23 -0500
Subject: [PATCH 032/157] RoT staging/dev and prod/rel 1.0.6 and SP 1.0.8
 (#5057)

---
 .github/buildomat/jobs/tuf-repo.sh |  4 ++--
 tools/dvt_dock_version             |  2 +-
 tools/hubris_checksums             | 16 ++++++++--------
 tools/hubris_version               |  2 +-
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh
index 59fc0fd7a7..14c2293f5b 100644
--- a/.github/buildomat/jobs/tuf-repo.sh
+++ b/.github/buildomat/jobs/tuf-repo.sh
@@ -278,8 +278,8 @@ EOF
     done
 }
 # usage:              SERIES           ROT_DIR      ROT_VERSION              BOARDS...
-add_hubris_artifacts  rot-staging-dev  staging/dev  cert-staging-dev-v1.0.5  "${ALL_BOARDS[@]}"
-add_hubris_artifacts  rot-prod-rel     prod/rel     cert-prod-rel-v1.0.5     "${ALL_BOARDS[@]}"
+add_hubris_artifacts  rot-staging-dev  staging/dev  cert-staging-dev-v1.0.6  "${ALL_BOARDS[@]}"
+add_hubris_artifacts  rot-prod-rel     prod/rel     cert-prod-rel-v1.0.6     "${ALL_BOARDS[@]}"
 
 for series in "${SERIES_LIST[@]}"; do
     /work/tufaceous assemble --no-generate-key /work/manifest-"$series".toml /work/repo-"$series".zip
diff --git a/tools/dvt_dock_version b/tools/dvt_dock_version
index 047065135b..fd988dc876 100644
--- a/tools/dvt_dock_version
+++ b/tools/dvt_dock_version
@@ -1 +1 @@
-COMMIT=e384836415e05ae0ba648810ab1c87e9093cdabb
+COMMIT=9e8e16f508bb41f02455aeddcece0a7edae2f672
diff --git a/tools/hubris_checksums b/tools/hubris_checksums
index d451f7a86c..dca8ea0ab6 100644
--- a/tools/hubris_checksums
+++ b/tools/hubris_checksums
@@ -1,8 +1,8 @@
-e1b3dc5c7da643b27c0dd5bf8e915d13661446e711bfdeb1d8274eed63fa5843  build-gimlet-c-image-default-v1.0.6.zip
-3002444307047429531ef862435a034c64b89a698921bf19794ac97b777a2f95  build-gimlet-d-image-default-v1.0.6.zip
-9e783bc92fb1c8a91f4b117241ed4c0ff2818f32f46c5193cdcdbbe02d56af9a  build-gimlet-e-image-default-v1.0.6.zip
-458c4f02310fe79f27841ce87b2a7c163494f0196890e6420fac17dc4803b51c  build-gimlet-f-image-default-v1.0.6.zip
-dece7d39f7fcd2f15dc62d91e94046b1f438a3e0fd2c804efd5f67e12ce0dd58  build-psc-b-image-default-v1.0.6.zip
-7e94035b52f1dcb137b477750bf9e215d4fcd07fe95b2cfdbbc0d7fada79eb28  build-psc-c-image-default-v1.0.6.zip
-ccf09dc7c9c2a946b89bcfafb391100504880fa395c9079dfb7a3b28635a4abb  build-sidecar-b-image-default-v1.0.6.zip
-b5d91c212f813dbdba06c1f5b098fd37fe6cb93fe33fd3c58325cb6504dc6d05  build-sidecar-c-image-default-v1.0.6.zip
+28f432735a15f40101c133e4e9974d1681a33bb7b3386ccbe15b465b613f4826  build-gimlet-c-image-default-v1.0.8.zip
+db927999398f0723d5d614db78a5abb4a1d515c711ffba944477bdac10c48907  build-gimlet-d-image-default-v1.0.8.zip
+629a53b5d9d4bf3d410687d0ecedf4837a54233ce62b6223d209494777cc7ebc  build-gimlet-e-image-default-v1.0.8.zip
+e3c2a243257929a65de638f3be425370f084aeeefafbd1773d01ee71cf0b8ea7  build-gimlet-f-image-default-v1.0.8.zip
+556595b42d05508ebfdac9dd71b38fe9b72e0cb30f6aa4be626c02141e375a71  build-psc-b-image-default-v1.0.8.zip
+39fbf92cbc935b4eaecb81a9768357828cf3e79b5c74d36c9a655ae9023cc50c  build-psc-c-image-default-v1.0.8.zip
+4225dff721b034fe7cf1dc26277557e1f15f2710014dd46dfa7c92ff04c7e054  build-sidecar-b-image-default-v1.0.8.zip
+ae8f12d7b66d0bcc372dd79abf515255f6ca97bb0339c570a058684e04e12cf8  build-sidecar-c-image-default-v1.0.8.zip
diff --git a/tools/hubris_version b/tools/hubris_version
index f2c1e74f2b..4ee8ac61fe 100644
--- a/tools/hubris_version
+++ b/tools/hubris_version
@@ -1 +1 @@
-TAGS=(gimlet-v1.0.6 psc-v1.0.6 sidecar-v1.0.6)
+TAGS=(gimlet-v1.0.8 psc-v1.0.8 sidecar-v1.0.8)

From a5be09fe0f20466ff66ae546df4895b785129fb0 Mon Sep 17 00:00:00 2001
From: David Pacheco <dap@oxidecomputer.com>
Date: Thu, 22 Feb 2024 11:06:14 -0800
Subject: [PATCH 033/157] "add sled" needs a longer timeout (#5116)

In #5111:

- the "add sled" Nexus external API call invokes `PUT /sleds` to some
sled agent
- `PUT /sleds` itself blocks until the new sled's sled agent has started
- sled agent startup blocks on setting the reservoir
- on production hardware, setting the reservoir took 115s
- the default Progenitor (reqwest) timeout is only 15s

So as a result, the "add sled" request failed, even though the operation
ultimately succeeded.

In this PR, I bump the timeout to 5 minutes. I do wonder if we should
remove it altogether, or if we should consider the other changes
mentioned in #5111 (like not blocking sled agent startup on this, or not
blocking these API calls in this way). But for now, this seems like a
low-risk way to improve this situation.
---
 nexus/src/app/rack.rs | 41 +++++++++++++++++++++++++++++++++--------
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs
index 7a1ad0e6a9..a137f19434 100644
--- a/nexus/src/app/rack.rs
+++ b/nexus/src/app/rack.rs
@@ -61,6 +61,7 @@ use sled_agent_client::types::{
     BgpConfig, BgpPeerConfig as SledBgpPeerConfig, EarlyNetworkConfig,
     PortConfigV1, RackNetworkConfigV1, RouteConfig as SledRouteConfig,
 };
+use slog_error_chain::InlineErrorChain;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::collections::HashMap;
@@ -647,7 +648,7 @@ impl super::Nexus {
         if rack.rack_subnet.is_some() {
             return Ok(());
         }
-        let sa = self.get_any_sled_agent(opctx).await?;
+        let sa = self.get_any_sled_agent_client(opctx).await?;
         let result = sa
             .read_network_bootstore_config_cache()
             .await
@@ -883,7 +884,27 @@ impl super::Nexus {
                 },
             },
         };
-        let sa = self.get_any_sled_agent(opctx).await?;
+
+        // This timeout value is fairly arbitrary (as they usually are).  As of
+        // this writing, this operation is known to take close to two minutes on
+        // production hardware.
+        let dur = std::time::Duration::from_secs(300);
+        let sa_url = self.get_any_sled_agent_url(opctx).await?;
+        let reqwest_client = reqwest::ClientBuilder::new()
+            .connect_timeout(dur)
+            .timeout(dur)
+            .build()
+            .map_err(|e| {
+                Error::internal_error(&format!(
+                    "failed to create reqwest client for sled agent: {}",
+                    InlineErrorChain::new(&e)
+                ))
+            })?;
+        let sa = sled_agent_client::Client::new_with_client(
+            &sa_url,
+            reqwest_client,
+            self.log.new(o!("sled_agent_url" => sa_url.clone())),
+        );
         sa.sled_add(&req).await.map_err(|e| Error::InternalError {
             internal_message: format!(
                 "failed to add sled with baseboard {:?} to rack {}: {e}",
@@ -899,10 +920,10 @@ impl super::Nexus {
         Ok(())
     }
 
-    async fn get_any_sled_agent(
+    async fn get_any_sled_agent_url(
         &self,
         opctx: &OpContext,
-    ) -> Result<sled_agent_client::Client, Error> {
+    ) -> Result<String, Error> {
         let addr = self
             .sled_list(opctx, &DataPageParams::max_page())
             .await?
@@ -911,11 +932,15 @@ impl super::Nexus {
                 internal_message: "no sled agents available".into(),
             })?
             .address();
+        Ok(format!("http://{}", addr))
+    }
 
-        Ok(sled_agent_client::Client::new(
-            &format!("http://{}", addr),
-            self.log.clone(),
-        ))
+    async fn get_any_sled_agent_client(
+        &self,
+        opctx: &OpContext,
+    ) -> Result<sled_agent_client::Client, Error> {
+        let url = self.get_any_sled_agent_url(opctx).await?;
+        Ok(sled_agent_client::Client::new(&url, self.log.clone()))
     }
 }
 

From 119bcd1f3d376dabc90a398b4c8ce9f384bbd17d Mon Sep 17 00:00:00 2001
From: "oxide-reflector-bot[bot]"
 <130185838+oxide-reflector-bot[bot]@users.noreply.github.com>
Date: Thu, 22 Feb 2024 22:53:12 +0000
Subject: [PATCH 034/157] Update maghemite to 4b0e584 (#5123)

Updated maghemite to commit 4b0e584.

Co-authored-by: reflector[bot] <130185838+reflector[bot]@users.noreply.github.com>
---
 package-manifest.toml               | 12 ++++++------
 tools/maghemite_ddm_openapi_version |  2 +-
 tools/maghemite_mg_openapi_version  |  2 +-
 tools/maghemite_mgd_checksums       |  4 ++--
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/package-manifest.toml b/package-manifest.toml
index d7f42794ee..1a749c5b61 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -463,10 +463,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "41a69a11db6cfa8fc0c8686dc2d725708e0586ce"
+source.commit = "4b0e584eec455a43c36af08ae207086965cef833"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/maghemite.sha256.txt
-source.sha256 = "19d5eaa744257c32ccdca52af79d718aeb88a0af188345d33a4564a69b259632"
+source.sha256 = "f1407cb9aac188d6493d2b0f948c75aad2c36668ddf4ae2a1ed80e9dd395b35d"
 output.type = "tarball"
 
 [package.mg-ddm]
@@ -479,10 +479,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "41a69a11db6cfa8fc0c8686dc2d725708e0586ce"
+source.commit = "4b0e584eec455a43c36af08ae207086965cef833"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mg-ddm.sha256.txt
-source.sha256 = "ffb647b3297ec616d3d9ea93396ad9edd16ed146048a660b34e9b86e85d466b7"
+source.sha256 = "fae53cb39536dc92d97cb9610de65b0acbce285e685d7167b719ea6311844fec"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -494,10 +494,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "41a69a11db6cfa8fc0c8686dc2d725708e0586ce"
+source.commit = "4b0e584eec455a43c36af08ae207086965cef833"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mg-ddm.sha256.txt
-source.sha256 = "26d34f61589f63be64eaa77a6e9e2db4c95d6675798386a1d61721c1ccc59d4d"
+source.sha256 = "22996a6f3353296b848be729f14e78a42e7d3d6e62a4a918a5c2358ae011c8eb"
 output.type = "zone"
 output.intermediate_only = true
 
diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version
index 6c58d83ea3..d161091fa8 100644
--- a/tools/maghemite_ddm_openapi_version
+++ b/tools/maghemite_ddm_openapi_version
@@ -1,2 +1,2 @@
-COMMIT="41a69a11db6cfa8fc0c8686dc2d725708e0586ce"
+COMMIT="4b0e584eec455a43c36af08ae207086965cef833"
 SHA2="0b0dbc2f8bbc5d2d9be92d64c4865f8f9335355aae62f7de9f67f81dfb3f1803"
diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version
index 896be8d38c..475d273f4a 100644
--- a/tools/maghemite_mg_openapi_version
+++ b/tools/maghemite_mg_openapi_version
@@ -1,2 +1,2 @@
-COMMIT="41a69a11db6cfa8fc0c8686dc2d725708e0586ce"
+COMMIT="4b0e584eec455a43c36af08ae207086965cef833"
 SHA2="0ac038bbaa54d0ae0ac5ccaeff48f03070618372cca26c9d09b716b909bf9355"
diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums
index 8fc4d083f8..ab84fafc01 100644
--- a/tools/maghemite_mgd_checksums
+++ b/tools/maghemite_mgd_checksums
@@ -1,2 +1,2 @@
-CIDL_SHA256="26d34f61589f63be64eaa77a6e9e2db4c95d6675798386a1d61721c1ccc59d4d"
-MGD_LINUX_SHA256="b2c823dd714fad67546a0e0c0d4ae56f2fe2e7c43434469b38e13b78de9f6968"
\ No newline at end of file
+CIDL_SHA256="22996a6f3353296b848be729f14e78a42e7d3d6e62a4a918a5c2358ae011c8eb"
+MGD_LINUX_SHA256="943b0a52d279bde55a419e2cdb24873acc32703bc97bd599376117ee0edc1511"
\ No newline at end of file

From 2088693046c7a666a50a5e6a8b0a14e99f7d01a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Karen=20C=C3=A1rcamo?= <karencfv@users.noreply.github.com>
Date: Fri, 23 Feb 2024 19:39:35 +1300
Subject: [PATCH 035/157] [sled-agent] Self assembling external DNS zone
 (#5059)

### Overview

In addition to implementing the external DNS self assembling zone, this
PR contains a new SMF service called `opte-interface-setup`.

Closes: https://github.com/oxidecomputer/omicron/issues/2881
Related: https://github.com/oxidecomputer/omicron/issues/1898
### Implementation

This service makes use of the zone-network CLI tool to avoid having too
many CLIs doing things.

The CLI is now shipped independently so it can be called by two
different services.

The [`zone-networking opte-interface-set-up`](
https://github.com/oxidecomputer/omicron/pull/5059/files#diff-5fb7b70dc87176e02517181b0887ce250b6a4e4079e495990551deeca741dc8bR181-R202)
command sets up what the `ensure_address_for_port()` method used to set
up.

### Justification

The reasoning behind this new service is to avoid setting up too many
things via the method_script.sh file, and to avoid code duplication. The
Nexus zone will also be using this service to set up the OPTE interface.
---
 illumos-utils/src/ipadm.rs                    |  25 +++
 illumos-utils/src/opte/port.rs                |   8 +-
 illumos-utils/src/route.rs                    |  67 +++++++-
 illumos-utils/src/running_zone.rs             |   7 +-
 package-manifest.toml                         |  56 +++++--
 sled-agent/src/services.rs                    | 140 ++++++++++++-----
 smf/external-dns/manifest.xml                 |  12 +-
 smf/opte-interface-setup/manifest.xml         |  46 ++++++
 smf/zone-network-setup/manifest.xml           |   2 +-
 zone-network-setup/src/bin/zone-networking.rs | 145 ++++++++++++++----
 10 files changed, 417 insertions(+), 91 deletions(-)
 create mode 100644 smf/opte-interface-setup/manifest.xml

diff --git a/illumos-utils/src/ipadm.rs b/illumos-utils/src/ipadm.rs
index f4d884d452..1c9e1e234e 100644
--- a/illumos-utils/src/ipadm.rs
+++ b/illumos-utils/src/ipadm.rs
@@ -107,4 +107,29 @@ impl Ipadm {
         };
         Ok(())
     }
+
+    // Create gateway on the IP interface if it doesn't already exist
+    pub fn create_opte_gateway(
+        opte_iface: &String,
+    ) -> Result<(), ExecutionError> {
+        let addrobj = format!("{}/public", opte_iface);
+        let mut cmd = std::process::Command::new(PFEXEC);
+        let cmd = cmd.args(&[IPADM, "show-addr", &addrobj]);
+        match execute(cmd) {
+            Err(_) => {
+                let mut cmd = std::process::Command::new(PFEXEC);
+                let cmd = cmd.args(&[
+                    IPADM,
+                    "create-addr",
+                    "-t",
+                    "-T",
+                    "dhcp",
+                    &addrobj,
+                ]);
+                execute(cmd)?;
+            }
+            Ok(_) => (),
+        };
+        Ok(())
+    }
 }
diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs
index d06cdfe1ec..6fbb89c450 100644
--- a/illumos-utils/src/opte/port.rs
+++ b/illumos-utils/src/opte/port.rs
@@ -15,7 +15,7 @@ struct PortInner {
     // Name of the port as identified by OPTE
     name: String,
     // IP address within the VPC Subnet
-    _ip: IpAddr,
+    ip: IpAddr,
     // VPC-private MAC address
     mac: MacAddr6,
     // Emulated PCI slot for the guest NIC, passed to Propolis
@@ -95,7 +95,7 @@ impl Port {
         Self {
             inner: Arc::new(PortInner {
                 name,
-                _ip: ip,
+                ip,
                 mac,
                 slot,
                 vni,
@@ -105,6 +105,10 @@ impl Port {
         }
     }
 
+    pub fn ip(&self) -> &IpAddr {
+        &self.inner.ip
+    }
+
     pub fn name(&self) -> &str {
         &self.inner.name
     }
diff --git a/illumos-utils/src/route.rs b/illumos-utils/src/route.rs
index 2b6af9a9fd..ceff2b3d9e 100644
--- a/illumos-utils/src/route.rs
+++ b/illumos-utils/src/route.rs
@@ -7,27 +7,76 @@
 use crate::zone::ROUTE;
 use crate::{execute, inner, output_to_exec_error, ExecutionError, PFEXEC};
 use libc::ESRCH;
-use std::net::Ipv6Addr;
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 
 /// Wraps commands for interacting with routing tables.
 pub struct Route {}
 
+pub enum Gateway {
+    Ipv4(Ipv4Addr),
+    Ipv6(Ipv6Addr),
+}
+
 #[cfg_attr(any(test, feature = "testing"), mockall::automock)]
 impl Route {
     pub fn ensure_default_route_with_gateway(
-        gateway: &Ipv6Addr,
+        gateway: Gateway,
     ) -> Result<(), ExecutionError> {
+        let inet;
+        let gw;
+        match gateway {
+            Gateway::Ipv4(addr) => {
+                inet = "-inet";
+                gw = addr.to_string();
+            }
+            Gateway::Ipv6(addr) => {
+                inet = "-inet6";
+                gw = addr.to_string();
+            }
+        }
         // Add the desired route if it doesn't already exist
         let destination = "default";
         let mut cmd = std::process::Command::new(PFEXEC);
+        let cmd = cmd.args(&[ROUTE, "-n", "get", inet, destination, inet, &gw]);
+
+        let out =
+            cmd.output().map_err(|err| ExecutionError::ExecutionStart {
+                command: inner::to_string(cmd),
+                err,
+            })?;
+        match out.status.code() {
+            Some(0) => (),
+            // If the entry is not found in the table,
+            // the exit status of the command will be 3 (ESRCH).
+            // When that is the case, we'll add the route.
+            Some(ESRCH) => {
+                let mut cmd = std::process::Command::new(PFEXEC);
+                let cmd =
+                    cmd.args(&[ROUTE, "add", inet, destination, inet, &gw]);
+                execute(cmd)?;
+            }
+            Some(_) | None => return Err(output_to_exec_error(cmd, &out)),
+        };
+        Ok(())
+    }
+
+    pub fn ensure_opte_route(
+        gateway: &Ipv4Addr,
+        iface: &String,
+        opte_ip: &IpAddr,
+    ) -> Result<(), ExecutionError> {
+        // Add the desired route if it doesn't already exist
+        let mut cmd = std::process::Command::new(PFEXEC);
         let cmd = cmd.args(&[
             ROUTE,
             "-n",
             "get",
-            "-inet6",
-            destination,
-            "-inet6",
+            "-host",
             &gateway.to_string(),
+            &opte_ip.to_string(),
+            "-interface",
+            "-ifp",
+            &iface.to_string(),
         ]);
 
         let out =
@@ -45,10 +94,12 @@ impl Route {
                 let cmd = cmd.args(&[
                     ROUTE,
                     "add",
-                    "-inet6",
-                    destination,
-                    "-inet6",
+                    "-host",
                     &gateway.to_string(),
+                    &opte_ip.to_string(),
+                    "-interface",
+                    "-ifp",
+                    &iface.to_string(),
                 ]);
                 execute(cmd)?;
             }
diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs
index 4b4107f529..1c1df01980 100644
--- a/illumos-utils/src/running_zone.rs
+++ b/illumos-utils/src/running_zone.rs
@@ -888,7 +888,7 @@ impl RunningZone {
 
     /// Return references to the OPTE ports for this zone.
     pub fn opte_ports(&self) -> impl Iterator<Item = &Port> {
-        self.inner.opte_ports.iter().map(|(port, _)| port)
+        self.inner.opte_ports()
     }
 
     /// Remove the OPTE ports on this zone from the port manager.
@@ -1130,6 +1130,11 @@ impl InstalledZone {
         path.push("root/var/svc/profile/site.xml");
         path
     }
+
+    /// Returns references to the OPTE ports for this zone.
+    pub fn opte_ports(&self) -> impl Iterator<Item = &Port> {
+        self.opte_ports.iter().map(|(port, _)| port)
+    }
 }
 
 #[derive(Clone)]
diff --git a/package-manifest.toml b/package-manifest.toml
index 1a749c5b61..c474a52736 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -120,7 +120,7 @@ setup_hint = """
 service_name = "oximeter"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "oximeter-collector.tar.gz", "zone-network-setup.tar.gz" ]
+source.packages = [ "oximeter-collector.tar.gz", "zone-network-setup.tar.gz", "zone-network-install.tar.gz" ]
 output.type = "zone"
 
 [package.oximeter-collector]
@@ -140,7 +140,12 @@ output.intermediate_only = true
 service_name = "clickhouse"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "clickhouse_svc.tar.gz", "internal-dns-cli.tar.gz", "zone-network-setup.tar.gz" ]
+source.packages = [
+  "clickhouse_svc.tar.gz",
+  "internal-dns-cli.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz"
+]
 output.type = "zone"
 
 [package.clickhouse_svc]
@@ -161,7 +166,12 @@ setup_hint = "Run `./tools/ci_download_clickhouse` to download the necessary bin
 service_name = "clickhouse_keeper"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "clickhouse_keeper_svc.tar.gz", "internal-dns-cli.tar.gz", "zone-network-setup.tar.gz" ]
+source.packages = [
+  "clickhouse_keeper_svc.tar.gz",
+  "internal-dns-cli.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz"
+]
 output.type = "zone"
 
 [package.clickhouse_keeper_svc]
@@ -182,7 +192,12 @@ setup_hint = "Run `./tools/ci_download_clickhouse` to download the necessary bin
 service_name = "cockroachdb"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "cockroachdb-service.tar.gz", "internal-dns-cli.tar.gz", "zone-network-setup.tar.gz" ]
+source.packages = [ 
+  "cockroachdb-service.tar.gz",
+  "internal-dns-cli.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz"
+]
 output.type = "zone"
 
 [package.cockroachdb-service]
@@ -220,7 +235,13 @@ output.type = "zone"
 service_name = "external_dns"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "dns-server.tar.gz", "external-dns-customizations.tar.gz" ]
+source.packages = [ 
+  "dns-server.tar.gz",
+  "external-dns-customizations.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz",
+  "opte-interface-setup.tar.gz"
+]
 output.type = "zone"
 
 [package.dns-server]
@@ -393,7 +414,7 @@ output.intermediate_only = true
 service_name = "crucible"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "crucible.tar.gz", "zone-network-setup.tar.gz" ]
+source.packages = [ "crucible.tar.gz", "zone-network-setup.tar.gz", "zone-network-install.tar.gz" ]
 output.type = "zone"
 
 
@@ -401,7 +422,7 @@ output.type = "zone"
 service_name = "crucible_pantry"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "crucible-pantry.tar.gz", "zone-network-setup.tar.gz" ]
+source.packages = [ "crucible-pantry.tar.gz", "zone-network-setup.tar.gz", "zone-network-install.tar.gz" ]
 output.type = "zone"
 
 # Packages not built within Omicron, but which must be imported.
@@ -643,14 +664,31 @@ source.packages = [
 ]
 output.type = "zone"
 
-[package.zone-network-setup]
+[package.zone-network-install]
 service_name = "zone-network-setup"
 only_for_targets.image = "standard"
 source.type = "local"
+source.paths = [
+    { from = "smf/zone-network-setup/manifest.xml", to = "/var/svc/manifest/site/zone-network-setup/manifest.xml" },
+]
+output.type = "zone"
+output.intermediate_only = true
+
+[package.zone-network-setup]
+service_name = "zone-network-cli"
+only_for_targets.image = "standard"
+source.type = "local"
 source.rust.binary_names = ["zone-networking"]
 source.rust.release = true
+output.type = "zone"
+output.intermediate_only = true
+
+[package.opte-interface-setup]
+service_name = "opte-interface-setup"
+only_for_targets.image = "standard"
+source.type = "local"
 source.paths = [
-    { from = "smf/zone-network-setup/manifest.xml", to = "/var/svc/manifest/site/zone-network-setup/manifest.xml" },
+    { from = "smf/opte-interface-setup/manifest.xml", to = "/var/svc/manifest/site/opte-interface-setup/manifest.xml" },
 ]
 output.type = "zone"
 output.intermediate_only = true
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
index bcd648cd2d..f3ddfdbf89 100644
--- a/sled-agent/src/services.rs
+++ b/sled-agent/src/services.rs
@@ -53,7 +53,8 @@ use illumos_utils::dladm::{
 use illumos_utils::link::{Link, VnicAllocator};
 use illumos_utils::opte::{DhcpCfg, Port, PortManager, PortTicket};
 use illumos_utils::running_zone::{
-    InstalledZone, RunCommandError, RunningZone, ZoneBuilderFactory,
+    EnsureAddressError, InstalledZone, RunCommandError, RunningZone,
+    ZoneBuilderFactory,
 };
 use illumos_utils::zfs::ZONE_ZFS_RAMDISK_DATASET_MOUNTPOINT;
 use illumos_utils::zone::AddressRequest;
@@ -68,6 +69,8 @@ use omicron_common::address::COCKROACH_PORT;
 use omicron_common::address::CRUCIBLE_PANTRY_PORT;
 use omicron_common::address::CRUCIBLE_PORT;
 use omicron_common::address::DENDRITE_PORT;
+use omicron_common::address::DNS_HTTP_PORT;
+use omicron_common::address::DNS_PORT;
 use omicron_common::address::MGS_PORT;
 use omicron_common::address::RACK_PREFIX;
 use omicron_common::address::SLED_PREFIX;
@@ -1444,6 +1447,32 @@ impl ServiceManager {
             .add_instance(ServiceInstanceBuilder::new("default")))
     }
 
+    fn opte_interface_set_up_install(
+        zone: &InstalledZone,
+    ) -> Result<ServiceBuilder, Error> {
+        let port_idx = 0;
+        let port = zone.opte_ports().nth(port_idx).ok_or_else(|| {
+            Error::ZoneEnsureAddress(EnsureAddressError::MissingOptePort {
+                zone: String::from(zone.name()),
+                port_idx,
+            })
+        })?;
+
+        let opte_interface = port.vnic_name();
+        let opte_gateway = port.gateway().ip().to_string();
+        let opte_ip = port.ip().to_string();
+
+        let mut config_builder = PropertyGroupBuilder::new("config");
+        config_builder = config_builder
+            .add_property("interface", "astring", opte_interface)
+            .add_property("gateway", "astring", &opte_gateway)
+            .add_property("ip", "astring", &opte_ip);
+
+        Ok(ServiceBuilder::new("oxide/opte-interface-setup")
+            .add_property_group(config_builder)
+            .add_instance(ServiceInstanceBuilder::new("default")))
+    }
+
     async fn initialize_zone(
         &self,
         request: ZoneArgs<'_>,
@@ -1841,6 +1870,73 @@ impl ServiceManager {
                     })?;
                 return Ok(RunningZone::boot(installed_zone).await?);
             }
+            ZoneArgs::Omicron(OmicronZoneConfigLocal {
+                zone:
+                    OmicronZoneConfig {
+                        zone_type: OmicronZoneType::ExternalDns { .. },
+                        underlay_address,
+                        ..
+                    },
+                ..
+            }) => {
+                let Some(info) = self.inner.sled_info.get() else {
+                    return Err(Error::SledAgentNotReady);
+                };
+
+                let static_addr = underlay_address.to_string();
+
+                let nw_setup_service = Self::zone_network_setup_install(
+                    info,
+                    &installed_zone,
+                    &static_addr.clone(),
+                )?;
+
+                // Like Nexus, we need to be reachable externally via
+                // `dns_address` but we don't listen on that address
+                // directly but instead on a VPC private IP. OPTE will
+                // en/decapsulate as appropriate.
+                let opte_interface_setup =
+                    Self::opte_interface_set_up_install(&installed_zone)?;
+
+                let port_idx = 0;
+                let port = installed_zone
+                    .opte_ports()
+                    .nth(port_idx)
+                    .ok_or_else(|| {
+                        Error::ZoneEnsureAddress(
+                            EnsureAddressError::MissingOptePort {
+                                zone: String::from(installed_zone.name()),
+                                port_idx,
+                            },
+                        )
+                    })?;
+                let opte_ip = port.ip();
+
+                let http_addr = format!("[{}]:{}", static_addr, DNS_HTTP_PORT);
+                let dns_addr = format!("{}:{}", opte_ip, DNS_PORT);
+
+                let external_dns_config = PropertyGroupBuilder::new("config")
+                    .add_property("http_address", "astring", &http_addr)
+                    .add_property("dns_address", "astring", &dns_addr);
+                let external_dns_service =
+                    ServiceBuilder::new("oxide/external_dns").add_instance(
+                        ServiceInstanceBuilder::new("default")
+                            .add_property_group(external_dns_config),
+                    );
+
+                let profile = ProfileBuilder::new("omicron")
+                    .add_service(nw_setup_service)
+                    .add_service(opte_interface_setup)
+                    .add_service(disabled_ssh_service)
+                    .add_service(external_dns_service);
+                profile
+                    .add_to_zone(&self.inner.log, &installed_zone)
+                    .await
+                    .map_err(|err| {
+                        Error::io("Failed to setup External DNS profile", err)
+                    })?;
+                return Ok(RunningZone::boot(installed_zone).await?);
+            }
             _ => {}
         }
 
@@ -2081,47 +2177,6 @@ impl ServiceManager {
                             .await
                             .map_err(|err| Error::io_path(&config_path, err))?;
                     }
-
-                    OmicronZoneType::ExternalDns {
-                        http_address,
-                        dns_address,
-                        ..
-                    } => {
-                        info!(
-                            self.inner.log,
-                            "Setting up external-dns service"
-                        );
-
-                        // Like Nexus, we need to be reachable externally via
-                        // `dns_address` but we don't listen on that address
-                        // directly but instead on a VPC private IP. OPTE will
-                        // en/decapsulate as appropriate.
-                        let port_ip = running_zone
-                            .ensure_address_for_port("public", 0)
-                            .await?
-                            .ip();
-                        let dns_address =
-                            SocketAddr::new(port_ip, dns_address.port());
-
-                        smfh.setprop(
-                            "config/http_address",
-                            format!(
-                                "[{}]:{}",
-                                http_address.ip(),
-                                http_address.port(),
-                            ),
-                        )?;
-                        smfh.setprop(
-                            "config/dns_address",
-                            dns_address.to_string(),
-                        )?;
-
-                        // Refresh the manifest with the new properties we set,
-                        // so they become "effective" properties when the
-                        // service is enabled.
-                        smfh.refresh()?;
-                    }
-
                     OmicronZoneType::InternalDns {
                         http_address,
                         dns_address,
@@ -2262,6 +2317,7 @@ impl ServiceManager {
                     | OmicronZoneType::CockroachDb { .. }
                     | OmicronZoneType::Crucible { .. }
                     | OmicronZoneType::CruciblePantry { .. }
+                    | OmicronZoneType::ExternalDns { .. }
                     | OmicronZoneType::Oximeter { .. } => {
                         panic!(
                             "{} is a service which exists as part of a \
diff --git a/smf/external-dns/manifest.xml b/smf/external-dns/manifest.xml
index 05c3c02b4e..1b6ee2cba0 100644
--- a/smf/external-dns/manifest.xml
+++ b/smf/external-dns/manifest.xml
@@ -4,13 +4,23 @@
 <service_bundle type='manifest' name='external_dns'>
 
   <service name='oxide/external_dns' type='service' version='1'>
-  <create_default_instance enabled='false' />
+  <create_default_instance enabled='true' />
 
   <dependency name='multi_user' grouping='require_all' restart_on='none'
     type='service'>
   <service_fmri value='svc:/milestone/multi-user:default' />
   </dependency>
 
+  <dependency name='zone_network_setup' grouping='require_all' restart_on='none'
+    type='service'>
+  <service_fmri value='svc:/oxide/zone-network-setup:default' />
+  </dependency>
+
+  <dependency name='opte_interface_setup' grouping='require_all' restart_on='none'
+    type='service'>
+  <service_fmri value='svc:/oxide/opte-interface-setup:default' />
+  </dependency>
+
   <exec_method type='method' name='start'
       exec='ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/external_dns/config.toml --http-address %{config/http_address} --dns-address %{config/dns_address} &amp;'
     timeout_seconds='0' />
diff --git a/smf/opte-interface-setup/manifest.xml b/smf/opte-interface-setup/manifest.xml
new file mode 100644
index 0000000000..5b886c8a71
--- /dev/null
+++ b/smf/opte-interface-setup/manifest.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0"?>
+<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
+
+<service_bundle type='manifest' name='opte-interface-setup'>
+
+<service name='oxide/opte-interface-setup' type='service' version='1'>
+  <create_default_instance enabled='true' />
+
+  <!-- Run after the operating system's svc:/network/physical service is done. -->
+  <dependency name='physical' grouping='require_all' restart_on='none'
+    type='service'>
+  <service_fmri value='svc:/network/physical:default' />
+  </dependency>
+
+  <dependency name='multi_user' grouping='require_all' restart_on='none'
+    type='service'>
+  <service_fmri value='svc:/milestone/multi-user:default' />
+  </dependency>
+
+  <exec_method type='method' name='start'
+    exec='/opt/oxide/zone-network-cli/bin/zone-networking opte-interface-set-up -i %{config/interface} -g %{config/gateway} -p %{config/ip}'
+    timeout_seconds='0' />
+  
+  <property_group name='startd' type='framework'>
+    <propval name='duration' type='astring' value='transient' />
+  </property_group>
+
+  <property_group name='config' type='application'>
+    <propval name='gateway' type='astring' value='unknown' />
+    <propval name='interface' type='astring' value='unknown' />
+    <propval name='ip' type='astring' value='unknown' />
+  </property_group>
+
+  <stability value='Unstable' />
+
+  <template>
+    <common_name>
+      <loctext xml:lang='C'>Oxide OPTE interface set up</loctext>
+    </common_name>
+    <description>
+      <loctext xml:lang='C'>Sets up an OPTE interface for control plane zones that require it</loctext>
+    </description>
+  </template>
+</service>
+
+</service_bundle>
diff --git a/smf/zone-network-setup/manifest.xml b/smf/zone-network-setup/manifest.xml
index 0776329749..a20ff949a4 100644
--- a/smf/zone-network-setup/manifest.xml
+++ b/smf/zone-network-setup/manifest.xml
@@ -18,7 +18,7 @@
   </dependency>
 
   <exec_method type='method' name='start'
-    exec='/opt/oxide/zone-network-setup/bin/zone-networking -d %{config/datalink} -s %{config/static_addr} -g %{config/gateway}'
+    exec='/opt/oxide/zone-network-cli/bin/zone-networking set-up -d %{config/datalink} -s %{config/static_addr} -g %{config/gateway}'
     timeout_seconds='0' />
   
   <property_group name='startd' type='framework'>
diff --git a/zone-network-setup/src/bin/zone-networking.rs b/zone-network-setup/src/bin/zone-networking.rs
index f3d18832c5..e36afd6b03 100644
--- a/zone-network-setup/src/bin/zone-networking.rs
+++ b/zone-network-setup/src/bin/zone-networking.rs
@@ -5,17 +5,31 @@
 //! CLI to set up zone networking
 
 use anyhow::anyhow;
-use clap::{arg, command};
+use clap::{arg, command, ArgMatches, Command};
 use illumos_utils::ipadm::Ipadm;
-use illumos_utils::route::Route;
+use illumos_utils::route::{Gateway, Route};
 use omicron_common::cmd::fatal;
 use omicron_common::cmd::CmdError;
-use slog::info;
+use slog::{info, Logger};
 use std::fs;
-use std::net::Ipv6Addr;
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 
 pub const HOSTS_FILE: &str = "/etc/inet/hosts";
 
+fn parse_ip(s: &str) -> anyhow::Result<IpAddr> {
+    if s == "unknown" {
+        return Err(anyhow!("ERROR: Missing input value"));
+    };
+    s.parse().map_err(|_| anyhow!("ERROR: Invalid IP address"))
+}
+
+fn parse_ipv4(s: &str) -> anyhow::Result<Ipv4Addr> {
+    if s == "unknown" {
+        return Err(anyhow!("ERROR: Missing input value"));
+    };
+    s.parse().map_err(|_| anyhow!("ERROR: Invalid IPv4 address"))
+}
+
 fn parse_ipv6(s: &str) -> anyhow::Result<Ipv6Addr> {
     if s == "unknown" {
         return Err(anyhow!("ERROR: Missing input value"));
@@ -30,6 +44,13 @@ fn parse_datalink(s: &str) -> anyhow::Result<String> {
     s.parse().map_err(|_| anyhow!("ERROR: Invalid data link"))
 }
 
+fn parse_opte_iface(s: &str) -> anyhow::Result<String> {
+    if s == "unknown" {
+        return Err(anyhow!("ERROR: Missing OPTE interface"));
+    };
+    s.parse().map_err(|_| anyhow!("ERROR: Invalid OPTE interface"))
+}
+
 #[tokio::main]
 async fn main() {
     if let Err(message) = do_run().await {
@@ -47,34 +68,81 @@ async fn do_run() -> Result<(), CmdError> {
     .map_err(|err| CmdError::Failure(anyhow!(err)))?;
 
     let matches = command!()
-        .arg(
-            arg!(
-                -d --datalink <STRING> "datalink"
-            )
-            .required(true)
-            .value_parser(parse_datalink),
-        )
-        .arg(
-            arg!(
-                -g --gateway <Ipv6Addr> "gateway"
-            )
-            .required(true)
-            .value_parser(parse_ipv6),
+        .subcommand(
+            Command::new("set-up")
+                .about(
+                    "Sets up common networking configuration across all zones",
+                )
+                .arg(
+                    arg!(
+                        -d --datalink <STRING> "datalink"
+                    )
+                    .required(true)
+                    .value_parser(parse_datalink),
+                )
+                .arg(
+                    arg!(
+                        -g --gateway <Ipv6Addr> "gateway"
+                    )
+                    .required(true)
+                    .value_parser(parse_ipv6),
+                )
+                .arg(
+                    arg!(
+                        -s --static_addr <Ipv6Addr> "static_addr"
+                    )
+                    .required(true)
+                    .value_parser(parse_ipv6),
+                ),
         )
-        .arg(
-            arg!(
-                -s --static_addr <Ipv6Addr> "static_addr"
-            )
-            .required(true)
-            .value_parser(parse_ipv6),
+        .subcommand(
+            Command::new("opte-interface-set-up")
+                .about("Sets up OPTE interface")
+                .arg(
+                    arg!(
+                        -i --opte_interface <STRING> "opte_interface"
+                    )
+                    .required(true)
+                    .value_parser(parse_opte_iface),
+                )
+                .arg(
+                    arg!(
+                        -g --opte_gateway <Ipv4Addr> "opte_gateway"
+                    )
+                    .required(true)
+                    .value_parser(parse_ipv4),
+                )
+                .arg(
+                    arg!(
+                        -p --opte_ip <IpAddr> "opte_ip"
+                    )
+                    .required(true)
+                    .value_parser(parse_ip),
+                ),
         )
         .get_matches();
 
-    let zonename =
-        zone::current().await.expect("Could not determine local zone name");
+    if let Some(matches) = matches.subcommand_matches("set-up") {
+        set_up(matches, log.clone()).await?;
+    }
+
+    if let Some(matches) = matches.subcommand_matches("opte-interface-set-up") {
+        opte_interface_set_up(matches, log.clone()).await?;
+    }
+
+    Ok(())
+}
+
+async fn set_up(matches: &ArgMatches, log: Logger) -> Result<(), CmdError> {
     let datalink: &String = matches.get_one("datalink").unwrap();
     let static_addr: &Ipv6Addr = matches.get_one("static_addr").unwrap();
-    let gateway: &Ipv6Addr = matches.get_one("gateway").unwrap();
+    let gateway: Ipv6Addr = *matches.get_one("gateway").unwrap();
+    let zonename = zone::current().await.map_err(|err| {
+        CmdError::Failure(anyhow!(
+            "Could not determine local zone name: {}",
+            err
+        ))
+    })?;
 
     // TODO: remove when https://github.com/oxidecomputer/stlouis/issues/435 is
     // addressed
@@ -91,7 +159,7 @@ async fn do_run() -> Result<(), CmdError> {
         .map_err(|err| CmdError::Failure(anyhow!(err)))?;
 
     info!(&log, "Ensuring there is a default route"; "gateway" => ?gateway);
-    Route::ensure_default_route_with_gateway(gateway)
+    Route::ensure_default_route_with_gateway(Gateway::Ipv6(gateway))
         .map_err(|err| CmdError::Failure(anyhow!(err)))?;
 
     info!(&log, "Populating hosts file for zone"; "zonename" => ?zonename);
@@ -109,3 +177,26 @@ async fn do_run() -> Result<(), CmdError> {
 
     Ok(())
 }
+
+async fn opte_interface_set_up(
+    matches: &ArgMatches,
+    log: Logger,
+) -> Result<(), CmdError> {
+    let interface: &String = matches.get_one("opte_interface").unwrap();
+    let gateway: Ipv4Addr = *matches.get_one("opte_gateway").unwrap();
+    let opte_ip: &IpAddr = matches.get_one("opte_ip").unwrap();
+
+    info!(&log, "Creating gateway on the OPTE IP interface if it doesn't already exist"; "OPTE interface" => ?interface);
+    Ipadm::create_opte_gateway(interface)
+        .map_err(|err| CmdError::Failure(anyhow!(err)))?;
+
+    info!(&log, "Ensuring there is a gateway route"; "OPTE gateway" => ?gateway, "OPTE interface" => ?interface, "OPTE IP" => ?opte_ip);
+    Route::ensure_opte_route(&gateway, interface, &opte_ip)
+        .map_err(|err| CmdError::Failure(anyhow!(err)))?;
+
+    info!(&log, "Ensuring there is a default route"; "gateway" => ?gateway);
+    Route::ensure_default_route_with_gateway(Gateway::Ipv4(gateway))
+        .map_err(|err| CmdError::Failure(anyhow!(err)))?;
+
+    Ok(())
+}

From 65ebf72288b16467dc6457bb4c3bfa90830000db Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 23 Feb 2024 01:59:41 -0800
Subject: [PATCH 036/157] [test-utils] Give a little hint about running
 CockroachDB on test failures (#5125)

This is a pattern I use a lot, but I want to help make it more visible
to folks during test failures.

Digging into the state of Cockroach after a test failure is a useful
debugging tool, so I want to make
it obvious "how to do that".
---
 test-utils/src/dev/db.rs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/test-utils/src/dev/db.rs b/test-utils/src/dev/db.rs
index da6cc90b03..3c1b46b4c2 100644
--- a/test-utils/src/dev/db.rs
+++ b/test-utils/src/dev/db.rs
@@ -640,8 +640,13 @@ impl Drop for CockroachInstance {
                 // Do NOT clean up the temporary directory in this case.
                 let path = temp_dir.into_path();
                 eprintln!(
-                    "WARN: temporary directory leaked: {}",
-                    path.display()
+                    "WARN: temporary directory leaked: {path:?}\n\
+                     \tIf you would like to access the database for debugging, run the following:\n\n\
+                     \t# Run the database\n\
+                     \tcargo run --bin omicron-dev db-run --no-populate --store-dir {data_path:?}\n\
+                     \t# Access the database. Note the port may change if you run multiple databases.\n\
+                     \tcockroach sql --host=localhost:32221 --insecure",
+                     data_path = path.join("data"),
                 );
             }
         }

From a07cae6485c3f7babbd4824b021c47601b99f92a Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Fri, 23 Feb 2024 14:55:47 -0500
Subject: [PATCH 037/157] Fix backwards counts in bad partition layout log
 message (#5129)

---
 sled-hardware/src/illumos/partitions.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/sled-hardware/src/illumos/partitions.rs b/sled-hardware/src/illumos/partitions.rs
index de62e25cfe..29b2466ad9 100644
--- a/sled-hardware/src/illumos/partitions.rs
+++ b/sled-hardware/src/illumos/partitions.rs
@@ -46,9 +46,8 @@ fn parse_partition_types<const N: usize>(
         return Err(PooledDiskError::BadPartitionLayout {
             path: path.to_path_buf(),
             why: format!(
-                "Expected {} partitions, only saw {}",
+                "Expected {N} partitions, only saw {}",
                 partitions.len(),
-                N
             ),
         });
     }

From a6ef7f9ed791893d1333f080b15ae4109e3b858d Mon Sep 17 00:00:00 2001
From: Rain <rain@oxide.computer>
Date: Fri, 23 Feb 2024 15:52:27 -0800
Subject: [PATCH 038/157] [nexus] add basic support for expunged sled policy
 and decommissioned sled state (#5032)

This PR does a few things:

* Migrates our current `sled_provision_state` to a new `sled_policy` enum, which has more information (per [RFD 457](https://rfd.shared.oxide.computer/rfd/0457)). This PR implements the expunged state, not the graceful removal state.
* Adds a `sled_state` enum, which describes Nexus's view of the sled. This PR adds the `active` and `decommissioned` states.
* Adds **internal** code to move around between valid states.
* Makes the blueprint execution code aware of sleds eligible for discretionary services.
* Adds tests for all of this new stuff, as well as valid and invalid state transitions -- and also makes sure that if we _do_ end up in an invalid state, things don't break down.

Not done here, but in future PRs (to try and keep this PR a manageable size):

* We'll add the endpoint to mark the sled as expunged (this is an irreversible operation and will need the appropriate warnings): https://github.com/oxidecomputer/omicron/issues/5134
* We'll add blueprint code to start removing sleds.
* We'll also remove the sled `time_deleted` because it has a lifecycle too complicated to be described that way -- instead, we'll add a `time_decommissioned` field: https://github.com/oxidecomputer/omicron/issues/5131
---
 Cargo.lock                                    |   1 +
 nexus/blueprint-execution/src/dns.rs          |   6 +-
 nexus/db-model/src/lib.rs                     |   6 +-
 nexus/db-model/src/schema.rs                  |   5 +-
 nexus/db-model/src/sled.rs                    |  31 +-
 nexus/db-model/src/sled_policy.rs             |  68 ++
 nexus/db-model/src/sled_provision_state.rs    |  53 --
 nexus/db-model/src/sled_state.rs              |  59 ++
 nexus/db-queries/Cargo.toml                   |   3 +-
 nexus/db-queries/src/authz/context.rs         |   3 +-
 nexus/db-queries/src/authz/policy_test/mod.rs |   6 +-
 nexus/db-queries/src/context.rs               |   9 +-
 .../db-queries/src/db/datastore/deployment.rs |   8 +-
 nexus/db-queries/src/db/datastore/disk.rs     |   2 +-
 nexus/db-queries/src/db/datastore/dns.rs      |   2 +-
 .../db-queries/src/db/datastore/inventory.rs  |   2 +-
 nexus/db-queries/src/db/datastore/ip_pool.rs  |   2 +-
 .../src/db/datastore/ipv4_nat_entry.rs        |   2 +-
 nexus/db-queries/src/db/datastore/mod.rs      | 434 +++++-----
 .../src/db/datastore/physical_disk.rs         |   3 +-
 nexus/db-queries/src/db/datastore/rack.rs     |   3 +-
 nexus/db-queries/src/db/datastore/sled.rs     | 799 ++++++++++++++++--
 .../src/db/datastore/switch_port.rs           |   3 +-
 .../db-queries/src/db/datastore/test_utils.rs | 343 ++++++++
 nexus/db-queries/src/db/datastore/volume.rs   |   2 +-
 nexus/db-queries/src/db/datastore/vpc.rs      |   2 +-
 nexus/db-queries/src/db/lookup.rs             |   3 +-
 nexus/db-queries/src/db/pool_connection.rs    |   3 +-
 .../db-queries/src/db/queries/external_ip.rs  |   3 +-
 .../src/db/queries/network_interface.rs       |   3 +-
 .../src/db/queries/region_allocation.rs       |   9 +-
 nexus/db-queries/src/transaction_retry.rs     |   2 +-
 nexus/deployment/src/blueprint_builder.rs     |  38 +-
 nexus/deployment/src/planner.rs               |  80 +-
 .../src/app/background/blueprint_execution.rs |   3 +-
 .../app/background/inventory_collection.rs    |   2 +-
 nexus/src/app/deployment.rs                   |   3 +-
 nexus/src/app/sled.rs                         |  26 +-
 nexus/src/external_api/http_entrypoints.rs    |  24 +-
 nexus/tests/integration_tests/endpoints.rs    |  17 +-
 nexus/tests/integration_tests/schema.rs       |  48 +-
 nexus/tests/output/nexus_tags.txt             |   2 +-
 nexus/types/src/deployment.rs                 |  22 +-
 nexus/types/src/external_api/params.rs        |  14 +-
 nexus/types/src/external_api/views.rs         | 200 ++++-
 openapi/nexus.json                            | 110 ++-
 schema/crdb/37.0.0/up01.sql                   |  13 +
 schema/crdb/37.0.0/up02.sql                   |  22 +
 schema/crdb/37.0.0/up03.sql                   |   7 +
 schema/crdb/37.0.0/up04.sql                   |  14 +
 schema/crdb/37.0.1/up01.sql                   |   8 +
 schema/crdb/37.0.1/up02.sql                   |   1 +
 schema/crdb/dbinit.sql                        |  49 +-
 53 files changed, 2115 insertions(+), 468 deletions(-)
 create mode 100644 nexus/db-model/src/sled_policy.rs
 delete mode 100644 nexus/db-model/src/sled_provision_state.rs
 create mode 100644 nexus/db-model/src/sled_state.rs
 create mode 100644 nexus/db-queries/src/db/datastore/test_utils.rs
 create mode 100644 schema/crdb/37.0.0/up01.sql
 create mode 100644 schema/crdb/37.0.0/up02.sql
 create mode 100644 schema/crdb/37.0.0/up03.sql
 create mode 100644 schema/crdb/37.0.0/up04.sql
 create mode 100644 schema/crdb/37.0.1/up01.sql
 create mode 100644 schema/crdb/37.0.1/up02.sql

diff --git a/Cargo.lock b/Cargo.lock
index e15afdfbab..85b7e5a186 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4405,6 +4405,7 @@ dependencies = [
  "pem",
  "petgraph",
  "pq-sys",
+ "predicates",
  "pretty_assertions",
  "rand 0.8.5",
  "rcgen",
diff --git a/nexus/blueprint-execution/src/dns.rs b/nexus/blueprint-execution/src/dns.rs
index 6dd9266f32..54611e9f66 100644
--- a/nexus/blueprint-execution/src/dns.rs
+++ b/nexus/blueprint-execution/src/dns.rs
@@ -328,7 +328,8 @@ mod test {
     use nexus_types::deployment::Policy;
     use nexus_types::deployment::SledResources;
     use nexus_types::deployment::ZpoolName;
-    use nexus_types::external_api::views::SledProvisionState;
+    use nexus_types::external_api::views::SledPolicy;
+    use nexus_types::external_api::views::SledState;
     use nexus_types::internal_api::params::DnsConfigParams;
     use nexus_types::internal_api::params::DnsConfigZone;
     use nexus_types::internal_api::params::DnsRecord;
@@ -409,7 +410,8 @@ mod test {
             .zip(possible_sled_subnets)
             .map(|(sled_id, subnet)| {
                 let sled_resources = SledResources {
-                    provision_state: SledProvisionState::Provisionable,
+                    policy: SledPolicy::provisionable(),
+                    state: SledState::Active,
                     zpools: BTreeSet::from([ZpoolName::from_str(&format!(
                         "oxp_{}",
                         Uuid::new_v4()
diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs
index ecbb8365fe..07c9f5eec6 100644
--- a/nexus/db-model/src/lib.rs
+++ b/nexus/db-model/src/lib.rs
@@ -73,9 +73,10 @@ mod silo_user;
 mod silo_user_password_hash;
 mod sled;
 mod sled_instance;
-mod sled_provision_state;
+mod sled_policy;
 mod sled_resource;
 mod sled_resource_kind;
+mod sled_state;
 mod sled_underlay_subnet_allocation;
 mod snapshot;
 mod ssh_key;
@@ -161,9 +162,10 @@ pub use silo_user::*;
 pub use silo_user_password_hash::*;
 pub use sled::*;
 pub use sled_instance::*;
-pub use sled_provision_state::*;
+pub use sled_policy::to_db_sled_policy; // Do not expose DbSledPolicy
 pub use sled_resource::*;
 pub use sled_resource_kind::*;
+pub use sled_state::*;
 pub use sled_underlay_subnet_allocation::*;
 pub use snapshot::*;
 pub use ssh_key::*;
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index 54755486e5..d8344c2258 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(36, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(37, 0, 1);
 
 table! {
     disk (id) {
@@ -824,7 +824,8 @@ table! {
         ip -> Inet,
         port -> Int4,
         last_used_address -> Inet,
-        provision_state -> crate::SledProvisionStateEnum,
+        sled_policy -> crate::sled_policy::SledPolicyEnum,
+        sled_state -> crate::SledStateEnum,
     }
 }
 
diff --git a/nexus/db-model/src/sled.rs b/nexus/db-model/src/sled.rs
index 52968c27d5..47912f89cc 100644
--- a/nexus/db-model/src/sled.rs
+++ b/nexus/db-model/src/sled.rs
@@ -2,10 +2,11 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use super::{ByteCount, Generation, SqlU16, SqlU32};
+use super::{ByteCount, Generation, SledState, SqlU16, SqlU32};
 use crate::collection::DatastoreCollectionConfig;
+use crate::ipv6;
 use crate::schema::{physical_disk, service, sled, zpool};
-use crate::{ipv6, SledProvisionState};
+use crate::sled_policy::DbSledPolicy;
 use chrono::{DateTime, Utc};
 use db_macros::Asset;
 use nexus_types::{external_api::shared, external_api::views, identity::Asset};
@@ -60,7 +61,11 @@ pub struct Sled {
     /// The last IP address provided to a propolis instance on this sled
     pub last_used_address: ipv6::Ipv6Addr,
 
-    provision_state: SledProvisionState,
+    #[diesel(column_name = sled_policy)]
+    policy: DbSledPolicy,
+
+    #[diesel(column_name = sled_state)]
+    state: SledState,
 }
 
 impl Sled {
@@ -84,8 +89,15 @@ impl Sled {
         &self.serial_number
     }
 
-    pub fn provision_state(&self) -> SledProvisionState {
-        self.provision_state
+    /// The policy here is the `views::SledPolicy` because we expect external
+    /// users to always use that.
+    pub fn policy(&self) -> views::SledPolicy {
+        self.policy.into()
+    }
+
+    /// Returns the sled's state.
+    pub fn state(&self) -> SledState {
+        self.state
     }
 }
 
@@ -99,7 +111,8 @@ impl From<Sled> for views::Sled {
                 part: sled.part_number,
                 revision: sled.revision,
             },
-            provision_state: sled.provision_state.into(),
+            policy: sled.policy.into(),
+            state: sled.state.into(),
             usable_hardware_threads: sled.usable_hardware_threads.0,
             usable_physical_ram: *sled.usable_physical_ram,
         }
@@ -197,8 +210,10 @@ impl SledUpdate {
             serial_number: self.serial_number,
             part_number: self.part_number,
             revision: self.revision,
-            // By default, sleds start as provisionable.
-            provision_state: SledProvisionState::Provisionable,
+            // By default, sleds start in-service.
+            policy: DbSledPolicy::InService,
+            // Currently, new sleds start in the "active" state.
+            state: SledState::Active,
             usable_hardware_threads: self.usable_hardware_threads,
             usable_physical_ram: self.usable_physical_ram,
             reservoir_size: self.reservoir_size,
diff --git a/nexus/db-model/src/sled_policy.rs b/nexus/db-model/src/sled_policy.rs
new file mode 100644
index 0000000000..13ecf10296
--- /dev/null
+++ b/nexus/db-model/src/sled_policy.rs
@@ -0,0 +1,68 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Database representation of a sled's operator-defined policy.
+//!
+//! This is related to, but different from `SledState`: a sled's **policy** is
+//! its disposition as specified by the operator, while its **state** refers to
+//! what's currently on it, as determined by Nexus.
+//!
+//! For example, a sled might be in the `Active` state, but have a policy of
+//! `Expunged` -- this would mean that Nexus knows about resources currently
+//! provisioned on the sled, but the operator has said that it should be marked
+//! as gone.
+
+use super::impl_enum_type;
+use nexus_types::external_api::views::{SledPolicy, SledProvisionPolicy};
+use serde::{Deserialize, Serialize};
+
+impl_enum_type!(
+    #[derive(Clone, SqlType, Debug, QueryId)]
+    #[diesel(postgres_type(name = "sled_policy", schema = "public"))]
+    pub struct SledPolicyEnum;
+
+    /// This type is not actually public, because [`SledPolicy`] has a somewhat
+    /// different, friendlier shape while being equivalent -- external code
+    /// should always use [`SledPolicy`].
+    ///
+    /// However, it must be marked `pub` to avoid errors like `crate-private
+    /// type `DbSledPolicy` in public interface`. Marking this type `pub`,
+    /// without actually making it public, tricks rustc in a desirable way.
+    #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)]
+    #[diesel(sql_type = SledPolicyEnum)]
+    pub enum DbSledPolicy;
+
+    // Enum values
+    InService => b"in_service"
+    NoProvision => b"no_provision"
+    Expunged => b"expunged"
+);
+
+/// Converts a [`SledPolicy`] to a version that can be inserted into a
+/// database.
+pub fn to_db_sled_policy(policy: SledPolicy) -> DbSledPolicy {
+    match policy {
+        SledPolicy::InService {
+            provision_policy: SledProvisionPolicy::Provisionable,
+        } => DbSledPolicy::InService,
+        SledPolicy::InService {
+            provision_policy: SledProvisionPolicy::NonProvisionable,
+        } => DbSledPolicy::NoProvision,
+        SledPolicy::Expunged => DbSledPolicy::Expunged,
+    }
+}
+
+impl From<DbSledPolicy> for SledPolicy {
+    fn from(policy: DbSledPolicy) -> Self {
+        match policy {
+            DbSledPolicy::InService => SledPolicy::InService {
+                provision_policy: SledProvisionPolicy::Provisionable,
+            },
+            DbSledPolicy::NoProvision => SledPolicy::InService {
+                provision_policy: SledProvisionPolicy::NonProvisionable,
+            },
+            DbSledPolicy::Expunged => SledPolicy::Expunged,
+        }
+    }
+}
diff --git a/nexus/db-model/src/sled_provision_state.rs b/nexus/db-model/src/sled_provision_state.rs
deleted file mode 100644
index ada842a32f..0000000000
--- a/nexus/db-model/src/sled_provision_state.rs
+++ /dev/null
@@ -1,53 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-use super::impl_enum_type;
-use nexus_types::external_api::views;
-use serde::{Deserialize, Serialize};
-use thiserror::Error;
-
-impl_enum_type!(
-    #[derive(Clone, SqlType, Debug, QueryId)]
-    #[diesel(postgres_type(name = "sled_provision_state", schema = "public"))]
-    pub struct SledProvisionStateEnum;
-
-    #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)]
-    #[diesel(sql_type = SledProvisionStateEnum)]
-    pub enum SledProvisionState;
-
-    // Enum values
-    Provisionable => b"provisionable"
-    NonProvisionable => b"non_provisionable"
-);
-
-impl From<SledProvisionState> for views::SledProvisionState {
-    fn from(state: SledProvisionState) -> Self {
-        match state {
-            SledProvisionState::Provisionable => {
-                views::SledProvisionState::Provisionable
-            }
-            SledProvisionState::NonProvisionable => {
-                views::SledProvisionState::NonProvisionable
-            }
-        }
-    }
-}
-
-impl From<views::SledProvisionState> for SledProvisionState {
-    fn from(state: views::SledProvisionState) -> Self {
-        match state {
-            views::SledProvisionState::Provisionable => {
-                SledProvisionState::Provisionable
-            }
-            views::SledProvisionState::NonProvisionable => {
-                SledProvisionState::NonProvisionable
-            }
-        }
-    }
-}
-
-/// An unknown [`views::SledProvisionState`] was encountered.
-#[derive(Clone, Debug, Error)]
-#[error("Unknown SledProvisionState")]
-pub struct UnknownSledProvisionState;
diff --git a/nexus/db-model/src/sled_state.rs b/nexus/db-model/src/sled_state.rs
new file mode 100644
index 0000000000..31029f3f4f
--- /dev/null
+++ b/nexus/db-model/src/sled_state.rs
@@ -0,0 +1,59 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Database representation of a sled's state as understood by Nexus.
+//!
+//! This is related to, but different from `SledState`: a sled's **policy** is
+//! its disposition as specified by the operator, while its **state** refers to
+//! what's currently on it, as determined by Nexus.
+//!
+//! For example, a sled might be in the `Active` state, but have a policy of
+//! `Expunged` -- this would mean that Nexus knows about resources currently
+//! provisioned on the sled, but the operator has said that it should be marked
+//! as gone.
+
+use super::impl_enum_type;
+use nexus_types::external_api::views;
+use serde::{Deserialize, Serialize};
+use std::fmt;
+use strum::EnumIter;
+
+impl_enum_type!(
+    #[derive(Clone, SqlType, Debug, QueryId)]
+    #[diesel(postgres_type(name = "sled_state", schema = "public"))]
+    pub struct SledStateEnum;
+
+    #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq, Eq, EnumIter)]
+    #[diesel(sql_type = SledStateEnum)]
+    pub enum SledState;
+
+    // Enum values
+    Active => b"active"
+    Decommissioned => b"decommissioned"
+);
+
+impl fmt::Display for SledState {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Forward to the canonical implementation in nexus-types.
+        views::SledState::from(*self).fmt(f)
+    }
+}
+
+impl From<SledState> for views::SledState {
+    fn from(state: SledState) -> Self {
+        match state {
+            SledState::Active => views::SledState::Active,
+            SledState::Decommissioned => views::SledState::Decommissioned,
+        }
+    }
+}
+
+impl From<views::SledState> for SledState {
+    fn from(state: views::SledState) -> Self {
+        match state {
+            views::SledState::Active => SledState::Active,
+            views::SledState::Decommissioned => SledState::Decommissioned,
+        }
+    }
+}
diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml
index 9c9a30799e..539a913476 100644
--- a/nexus/db-queries/Cargo.toml
+++ b/nexus/db-queries/Cargo.toml
@@ -43,6 +43,7 @@ sled-agent-client.workspace = true
 slog.workspace = true
 static_assertions.workspace = true
 steno.workspace = true
+strum.workspace = true
 swrite.workspace = true
 thiserror.workspace = true
 tokio = { workspace = true, features = ["full"] }
@@ -76,10 +77,10 @@ omicron-test-utils.workspace = true
 openapiv3.workspace = true
 pem.workspace = true
 petgraph.workspace = true
+predicates.workspace = true
 pretty_assertions.workspace = true
 rcgen.workspace = true
 regex.workspace = true
 rustls.workspace = true
-strum.workspace = true
 subprocess.workspace = true
 term.workspace = true
diff --git a/nexus/db-queries/src/authz/context.rs b/nexus/db-queries/src/authz/context.rs
index 3510da2735..0d6f2a73ac 100644
--- a/nexus/db-queries/src/authz/context.rs
+++ b/nexus/db-queries/src/authz/context.rs
@@ -217,7 +217,8 @@ mod test {
         let logctx = dev::test_setup_log("test_unregistered_resource");
         let mut db = test_setup_database(&logctx.log).await;
         let (opctx, datastore) =
-            crate::db::datastore::datastore_test(&logctx, &db).await;
+            crate::db::datastore::test_utils::datastore_test(&logctx, &db)
+                .await;
 
         // Define a resource that we "forget" to register with Oso.
         use super::AuthorizedResource;
diff --git a/nexus/db-queries/src/authz/policy_test/mod.rs b/nexus/db-queries/src/authz/policy_test/mod.rs
index 00a9904499..b6961bcc30 100644
--- a/nexus/db-queries/src/authz/policy_test/mod.rs
+++ b/nexus/db-queries/src/authz/policy_test/mod.rs
@@ -62,7 +62,8 @@ use uuid::Uuid;
 async fn test_iam_roles_behavior() {
     let logctx = dev::test_setup_log("test_iam_roles");
     let mut db = test_setup_database(&logctx.log).await;
-    let (opctx, datastore) = db::datastore::datastore_test(&logctx, &db).await;
+    let (opctx, datastore) =
+        db::datastore::test_utils::datastore_test(&logctx, &db).await;
 
     // Before we can create the resources, users, and role assignments that we
     // need, we must grant the "test-privileged" user privileges to fetch and
@@ -328,7 +329,8 @@ async fn test_conferred_roles() {
     // To start, this test looks a lot like the test above.
     let logctx = dev::test_setup_log("test_conferred_roles");
     let mut db = test_setup_database(&logctx.log).await;
-    let (opctx, datastore) = db::datastore::datastore_test(&logctx, &db).await;
+    let (opctx, datastore) =
+        db::datastore::test_utils::datastore_test(&logctx, &db).await;
 
     // Before we can create the resources, users, and role assignments that we
     // need, we must grant the "test-privileged" user privileges to fetch and
diff --git a/nexus/db-queries/src/context.rs b/nexus/db-queries/src/context.rs
index aea4a60e66..dfd1fe4322 100644
--- a/nexus/db-queries/src/context.rs
+++ b/nexus/db-queries/src/context.rs
@@ -357,7 +357,8 @@ mod test {
         let logctx = dev::test_setup_log("test_background_context");
         let mut db = test_setup_database(&logctx.log).await;
         let (_, datastore) =
-            crate::db::datastore::datastore_test(&logctx, &db).await;
+            crate::db::datastore::test_utils::datastore_test(&logctx, &db)
+                .await;
         let opctx = OpContext::for_background(
             logctx.log.new(o!()),
             Arc::new(authz::Authz::new(&logctx.log)),
@@ -389,7 +390,8 @@ mod test {
         let logctx = dev::test_setup_log("test_background_context");
         let mut db = test_setup_database(&logctx.log).await;
         let (_, datastore) =
-            crate::db::datastore::datastore_test(&logctx, &db).await;
+            crate::db::datastore::test_utils::datastore_test(&logctx, &db)
+                .await;
         let opctx = OpContext::for_tests(logctx.log.new(o!()), datastore);
 
         // Like in test_background_context(), this is essentially a test of the
@@ -410,7 +412,8 @@ mod test {
         let logctx = dev::test_setup_log("test_child_context");
         let mut db = test_setup_database(&logctx.log).await;
         let (_, datastore) =
-            crate::db::datastore::datastore_test(&logctx, &db).await;
+            crate::db::datastore::test_utils::datastore_test(&logctx, &db)
+                .await;
         let opctx = OpContext::for_background(
             logctx.log.new(o!()),
             Arc::new(authz::Authz::new(&logctx.log)),
diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs
index d9df143022..00a75a21da 100644
--- a/nexus/db-queries/src/db/datastore/deployment.rs
+++ b/nexus/db-queries/src/db/datastore/deployment.rs
@@ -1054,14 +1054,15 @@ impl RunQueryDsl<DbConnection> for InsertTargetQuery {}
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::db::datastore::datastore_test;
+    use crate::db::datastore::test_utils::datastore_test;
     use nexus_deployment::blueprint_builder::BlueprintBuilder;
     use nexus_deployment::blueprint_builder::Ensure;
     use nexus_inventory::now_db_precision;
     use nexus_test_utils::db::test_setup_database;
     use nexus_types::deployment::Policy;
     use nexus_types::deployment::SledResources;
-    use nexus_types::external_api::views::SledProvisionState;
+    use nexus_types::external_api::views::SledPolicy;
+    use nexus_types::external_api::views::SledState;
     use nexus_types::inventory::Collection;
     use omicron_common::address::Ipv6Subnet;
     use omicron_common::api::external::Generation;
@@ -1126,7 +1127,8 @@ mod tests {
             .collect();
         let ip = ip.unwrap_or_else(|| thread_rng().gen::<u128>().into());
         SledResources {
-            provision_state: SledProvisionState::Provisionable,
+            policy: SledPolicy::provisionable(),
+            state: SledState::Active,
             zpools,
             subnet: Ipv6Subnet::new(ip),
         }
diff --git a/nexus/db-queries/src/db/datastore/disk.rs b/nexus/db-queries/src/db/datastore/disk.rs
index 390376e627..2916573322 100644
--- a/nexus/db-queries/src/db/datastore/disk.rs
+++ b/nexus/db-queries/src/db/datastore/disk.rs
@@ -817,7 +817,7 @@ impl DataStore {
 mod tests {
     use super::*;
 
-    use crate::db::datastore::datastore_test;
+    use crate::db::datastore::test_utils::datastore_test;
     use nexus_test_utils::db::test_setup_database;
     use nexus_types::external_api::params;
     use omicron_common::api::external;
diff --git a/nexus/db-queries/src/db/datastore/dns.rs b/nexus/db-queries/src/db/datastore/dns.rs
index 180764d38c..b12df1875f 100644
--- a/nexus/db-queries/src/db/datastore/dns.rs
+++ b/nexus/db-queries/src/db/datastore/dns.rs
@@ -688,7 +688,7 @@ impl DnsVersionUpdateBuilder {
 
 #[cfg(test)]
 mod test {
-    use crate::db::datastore::datastore_test;
+    use crate::db::datastore::test_utils::datastore_test;
     use crate::db::datastore::DnsVersionUpdateBuilder;
     use crate::db::DataStore;
     use crate::db::TransactionError;
diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs
index 6b737f21ac..3f2f4bd127 100644
--- a/nexus/db-queries/src/db/datastore/inventory.rs
+++ b/nexus/db-queries/src/db/datastore/inventory.rs
@@ -1914,8 +1914,8 @@ impl DataStoreInventoryTest for DataStore {
 
 #[cfg(test)]
 mod test {
-    use crate::db::datastore::datastore_test;
     use crate::db::datastore::inventory::DataStoreInventoryTest;
+    use crate::db::datastore::test_utils::datastore_test;
     use crate::db::datastore::DataStoreConnection;
     use crate::db::schema;
     use anyhow::Context;
diff --git a/nexus/db-queries/src/db/datastore/ip_pool.rs b/nexus/db-queries/src/db/datastore/ip_pool.rs
index 4634fda9ee..4a37efd612 100644
--- a/nexus/db-queries/src/db/datastore/ip_pool.rs
+++ b/nexus/db-queries/src/db/datastore/ip_pool.rs
@@ -816,7 +816,7 @@ mod test {
     use std::num::NonZeroU32;
 
     use crate::authz;
-    use crate::db::datastore::datastore_test;
+    use crate::db::datastore::test_utils::datastore_test;
     use crate::db::model::{IpPool, IpPoolResource, IpPoolResourceType};
     use assert_matches::assert_matches;
     use nexus_test_utils::db::test_setup_database;
diff --git a/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs
index 27a6bad32f..670ca08960 100644
--- a/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs
+++ b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs
@@ -379,7 +379,7 @@ fn ipv4_nat_next_version() -> diesel::expression::SqlLiteral<BigInt> {
 mod test {
     use std::{net::Ipv4Addr, str::FromStr};
 
-    use crate::db::datastore::datastore_test;
+    use crate::db::datastore::test_utils::datastore_test;
     use chrono::Utc;
     use nexus_db_model::{Ipv4NatEntry, Ipv4NatValues, MacAddr, Vni};
     use nexus_test_utils::db::test_setup_database;
diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs
index 5f05aa1760..b5eff6cb85 100644
--- a/nexus/db-queries/src/db/datastore/mod.rs
+++ b/nexus/db-queries/src/db/datastore/mod.rs
@@ -44,6 +44,7 @@ use omicron_common::backoff::{
 use omicron_common::nexus_config::SchemaConfig;
 use slog::Logger;
 use std::net::Ipv6Addr;
+use std::num::NonZeroU32;
 use std::sync::Arc;
 use uuid::Uuid;
 
@@ -87,6 +88,8 @@ mod ssh_key;
 mod switch;
 mod switch_interface;
 mod switch_port;
+#[cfg(test)]
+pub(crate) mod test_utils;
 mod update;
 mod utilization;
 mod virtual_provisioning_collection;
@@ -105,7 +108,7 @@ pub use instance::InstanceAndActiveVmm;
 pub use inventory::DataStoreInventoryTest;
 pub use rack::RackInit;
 pub use silo::Discoverability;
-use std::num::NonZeroU32;
+pub use sled::SledUpsertOutput;
 pub use switch_port::SwitchPortSettingsCombinedResult;
 pub use virtual_provisioning_collection::StorageType;
 pub use volume::read_only_resources_associated_with_volume;
@@ -352,52 +355,16 @@ pub enum UpdatePrecondition<T> {
     Value(T),
 }
 
-/// Constructs a DataStore for use in test suites that has preloaded the
-/// built-in users, roles, and role assignments that are needed for basic
-/// operation
-#[cfg(test)]
-pub async fn datastore_test(
-    logctx: &dropshot::test_util::LogContext,
-    db: &omicron_test_utils::dev::db::CockroachInstance,
-) -> (OpContext, Arc<DataStore>) {
-    use crate::authn;
-
-    let cfg = db::Config { url: db.pg_config().clone() };
-    let pool = Arc::new(db::Pool::new(&logctx.log, &cfg));
-    let datastore =
-        Arc::new(DataStore::new(&logctx.log, pool, None).await.unwrap());
-
-    // Create an OpContext with the credentials of "db-init" just for the
-    // purpose of loading the built-in users, roles, and assignments.
-    let opctx = OpContext::for_background(
-        logctx.log.new(o!()),
-        Arc::new(authz::Authz::new(&logctx.log)),
-        authn::Context::internal_db_init(),
-        Arc::clone(&datastore),
-    );
-
-    // TODO: Can we just call "Populate" instead of doing this?
-    let rack_id = Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap();
-    datastore.load_builtin_users(&opctx).await.unwrap();
-    datastore.load_builtin_roles(&opctx).await.unwrap();
-    datastore.load_builtin_role_asgns(&opctx).await.unwrap();
-    datastore.load_builtin_silos(&opctx).await.unwrap();
-    datastore.load_builtin_projects(&opctx).await.unwrap();
-    datastore.load_builtin_vpcs(&opctx).await.unwrap();
-    datastore.load_silo_users(&opctx).await.unwrap();
-    datastore.load_silo_user_role_assignments(&opctx).await.unwrap();
-    datastore
-        .load_builtin_fleet_virtual_provisioning_collection(&opctx)
-        .await
-        .unwrap();
-    datastore.load_builtin_rack_data(&opctx, rack_id).await.unwrap();
-
-    // Create an OpContext with the credentials of "test-privileged" for general
-    // testing.
-    let opctx =
-        OpContext::for_tests(logctx.log.new(o!()), Arc::clone(&datastore));
-
-    (opctx, datastore)
+/// Whether state transitions should be validated. "No" is only accessible in
+/// test-only code.
+///
+/// Intended only for testing around illegal states.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+#[must_use]
+enum ValidateTransition {
+    Yes,
+    #[cfg(test)]
+    No,
 }
 
 #[cfg(test)]
@@ -406,6 +373,10 @@ mod test {
     use crate::authn;
     use crate::authn::SiloAuthnPolicy;
     use crate::authz;
+    use crate::db::datastore::test_utils::datastore_test;
+    use crate::db::datastore::test_utils::{
+        IneligibleSledKind, IneligibleSleds,
+    };
     use crate::db::explain::ExplainableAsync;
     use crate::db::fixed_data::silo::DEFAULT_SILO;
     use crate::db::fixed_data::silo::SILO_ID;
@@ -414,8 +385,8 @@ mod test {
     use crate::db::model::{
         BlockSize, ConsoleSession, Dataset, DatasetKind, ExternalIp,
         PhysicalDisk, PhysicalDiskKind, Project, Rack, Region, Service,
-        ServiceKind, SiloUser, SledBaseboard, SledProvisionState,
-        SledSystemHardware, SledUpdate, SshKey, VpcSubnet, Zpool,
+        ServiceKind, SiloUser, SledBaseboard, SledSystemHardware, SledUpdate,
+        SshKey, VpcSubnet, Zpool,
     };
     use crate::db::queries::vpc_subnet::FilterConflictingVpcSubnetRangesQuery;
     use chrono::{Duration, Utc};
@@ -435,6 +406,7 @@ mod test {
     use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddrV6};
     use std::num::NonZeroU32;
     use std::sync::Arc;
+    use strum::EnumCount;
     use uuid::Uuid;
 
     // Creates a "fake" Sled Baseboard.
@@ -648,39 +620,10 @@ mod test {
             sled_system_hardware_for_test(),
             rack_id,
         );
-        datastore.sled_upsert(sled_update).await.unwrap();
+        datastore.sled_upsert(sled_update).await.unwrap().unwrap();
         sled_id
     }
 
-    // Marks a sled as non-provisionable.
-    async fn mark_sled_non_provisionable(
-        datastore: &DataStore,
-        opctx: &OpContext,
-        sled_id: Uuid,
-    ) {
-        let (authz_sled, sled) = LookupPath::new(opctx, datastore)
-            .sled_id(sled_id)
-            .fetch_for(authz::Action::Modify)
-            .await
-            .unwrap();
-        println!("sled: {:?}", sled);
-        let old_state = datastore
-            .sled_set_provision_state(
-                &opctx,
-                &authz_sled,
-                SledProvisionState::NonProvisionable,
-            )
-            .await
-            .unwrap_or_else(|error| {
-                panic!(
-                    "error marking sled {sled_id} as non-provisionable: {error}"
-                )
-            });
-        // The old state should always be provisionable since that's where we
-        // start.
-        assert_eq!(old_state, SledProvisionState::Provisionable);
-    }
-
     fn test_zpool_size() -> ByteCount {
         ByteCount::from_gibibytes_u32(100)
     }
@@ -742,95 +685,184 @@ mod test {
         }
     }
 
-    struct TestDataset {
-        sled_id: Uuid,
-        dataset_id: Uuid,
+    #[derive(Debug)]
+    struct TestDatasets {
+        // eligible and ineligible aren't currently used, but are probably handy
+        // for the future.
+        #[allow(dead_code)]
+        eligible: SledToDatasetMap,
+        #[allow(dead_code)]
+        ineligible: SledToDatasetMap,
+
+        // A map from eligible dataset IDs to their corresponding sled IDs.
+        eligible_dataset_ids: HashMap<Uuid, Uuid>,
+        ineligible_dataset_ids: HashMap<Uuid, IneligibleSledKind>,
     }
 
-    async fn create_test_datasets_for_region_allocation(
-        opctx: &OpContext,
-        datastore: Arc<DataStore>,
-        number_of_sleds: usize,
-    ) -> Vec<TestDataset> {
-        // Create sleds...
-        let sled_ids: Vec<Uuid> = stream::iter(0..number_of_sleds)
-            .then(|_| create_test_sled(&datastore))
-            .collect()
+    // Map of sled IDs to dataset IDs.
+    type SledToDatasetMap = HashMap<Uuid, Vec<Uuid>>;
+
+    impl TestDatasets {
+        async fn create(
+            opctx: &OpContext,
+            datastore: Arc<DataStore>,
+            num_eligible_sleds: usize,
+        ) -> Self {
+            let eligible =
+                Self::create_impl(opctx, datastore.clone(), num_eligible_sleds)
+                    .await;
+
+            let eligible_dataset_ids = eligible
+                .iter()
+                .flat_map(|(sled_id, dataset_ids)| {
+                    dataset_ids
+                        .iter()
+                        .map(move |dataset_id| (*dataset_id, *sled_id))
+                })
+                .collect();
+
+            let ineligible = Self::create_impl(
+                opctx,
+                datastore.clone(),
+                IneligibleSledKind::COUNT,
+            )
             .await;
 
-        struct PhysicalDisk {
-            sled_id: Uuid,
-            disk_id: Uuid,
-        }
+            let mut ineligible_sled_ids = ineligible.keys();
 
-        // create 9 disks on each sled
-        let physical_disks: Vec<PhysicalDisk> = stream::iter(sled_ids)
-            .map(|sled_id| {
-                let sled_id_iter: Vec<Uuid> = (0..9).map(|_| sled_id).collect();
-                stream::iter(sled_id_iter).then(|sled_id| {
-                    let disk_id_future = create_test_physical_disk(
-                        &datastore,
-                        opctx,
-                        sled_id,
-                        PhysicalDiskKind::U2,
-                    );
-                    async move {
-                        let disk_id = disk_id_future.await;
-                        PhysicalDisk { sled_id, disk_id }
-                    }
-                })
-            })
-            .flatten()
-            .collect()
-            .await;
+            // Set up the ineligible sleds. (We're guaranteed that
+            // IneligibleSledKind::COUNT is the same as the number of next()
+            // calls below.)
+            let ineligible_sleds = IneligibleSleds {
+                non_provisionable: *ineligible_sled_ids.next().unwrap(),
+                expunged: *ineligible_sled_ids.next().unwrap(),
+                decommissioned: *ineligible_sled_ids.next().unwrap(),
+                illegal_decommissioned: *ineligible_sled_ids.next().unwrap(),
+            };
 
-        #[derive(Copy, Clone)]
-        struct Zpool {
-            sled_id: Uuid,
-            pool_id: Uuid,
-        }
+            eprintln!("Setting up ineligible sleds: {:?}", ineligible_sleds);
 
-        // 1 pool per disk
-        let zpools: Vec<Zpool> = stream::iter(physical_disks)
-            .then(|disk| {
-                let pool_id_future =
-                    create_test_zpool(&datastore, disk.sled_id, disk.disk_id);
-                async move {
-                    let pool_id = pool_id_future.await;
-                    Zpool { sled_id: disk.sled_id, pool_id }
+            ineligible_sleds
+                .setup(opctx, &datastore)
+                .await
+                .expect("error setting up ineligible sleds");
+
+            // Build a map of dataset IDs to their ineligible kind.
+            let mut ineligible_dataset_ids = HashMap::new();
+            for (kind, sled_id) in ineligible_sleds.iter() {
+                for dataset_id in ineligible.get(&sled_id).unwrap() {
+                    ineligible_dataset_ids.insert(*dataset_id, kind);
                 }
-            })
-            .collect()
-            .await;
+            }
 
-        let bogus_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0);
+            Self {
+                eligible,
+                eligible_dataset_ids,
+                ineligible,
+                ineligible_dataset_ids,
+            }
+        }
 
-        let datasets: Vec<TestDataset> = stream::iter(zpools)
-            .map(|zpool| {
-                // 3 datasets per zpool, to test that pools are distinct
-                let zpool_iter: Vec<Zpool> = (0..3).map(|_| zpool).collect();
-                stream::iter(zpool_iter).then(|zpool| {
-                    let id = Uuid::new_v4();
-                    let dataset = Dataset::new(
-                        id,
-                        zpool.pool_id,
-                        bogus_addr,
-                        DatasetKind::Crucible,
-                    );
+        // Returns a map of sled ID to dataset IDs.
+        async fn create_impl(
+            opctx: &OpContext,
+            datastore: Arc<DataStore>,
+            number_of_sleds: usize,
+        ) -> SledToDatasetMap {
+            // Create sleds...
+            let sled_ids: Vec<Uuid> = stream::iter(0..number_of_sleds)
+                .then(|_| create_test_sled(&datastore))
+                .collect()
+                .await;
 
-                    let datastore = datastore.clone();
-                    async move {
-                        datastore.dataset_upsert(dataset).await.unwrap();
+            struct PhysicalDisk {
+                sled_id: Uuid,
+                disk_id: Uuid,
+            }
 
-                        TestDataset { sled_id: zpool.sled_id, dataset_id: id }
+            // create 9 disks on each sled
+            let physical_disks: Vec<PhysicalDisk> = stream::iter(sled_ids)
+                .map(|sled_id| {
+                    let sled_id_iter: Vec<Uuid> =
+                        (0..9).map(|_| sled_id).collect();
+                    stream::iter(sled_id_iter).then(|sled_id| {
+                        let disk_id_future = create_test_physical_disk(
+                            &datastore,
+                            opctx,
+                            sled_id,
+                            PhysicalDiskKind::U2,
+                        );
+                        async move {
+                            let disk_id = disk_id_future.await;
+                            PhysicalDisk { sled_id, disk_id }
+                        }
+                    })
+                })
+                .flatten()
+                .collect()
+                .await;
+
+            #[derive(Copy, Clone)]
+            struct Zpool {
+                sled_id: Uuid,
+                pool_id: Uuid,
+            }
+
+            // 1 pool per disk
+            let zpools: Vec<Zpool> = stream::iter(physical_disks)
+                .then(|disk| {
+                    let pool_id_future = create_test_zpool(
+                        &datastore,
+                        disk.sled_id,
+                        disk.disk_id,
+                    );
+                    async move {
+                        let pool_id = pool_id_future.await;
+                        Zpool { sled_id: disk.sled_id, pool_id }
                     }
                 })
-            })
-            .flatten()
-            .collect()
-            .await;
+                .collect()
+                .await;
+
+            let bogus_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0);
+
+            let datasets = stream::iter(zpools)
+                .map(|zpool| {
+                    // 3 datasets per zpool, to test that pools are distinct
+                    let zpool_iter: Vec<Zpool> =
+                        (0..3).map(|_| zpool).collect();
+                    stream::iter(zpool_iter).then(|zpool| {
+                        let dataset_id = Uuid::new_v4();
+                        let dataset = Dataset::new(
+                            dataset_id,
+                            zpool.pool_id,
+                            bogus_addr,
+                            DatasetKind::Crucible,
+                        );
+
+                        let datastore = datastore.clone();
+                        async move {
+                            datastore.dataset_upsert(dataset).await.unwrap();
+
+                            (zpool.sled_id, dataset_id)
+                        }
+                    })
+                })
+                .flatten()
+                .fold(
+                    SledToDatasetMap::new(),
+                    |mut map, (sled_id, dataset_id)| {
+                        // Build a map of sled ID to dataset IDs.
+                        map.entry(sled_id)
+                            .or_insert_with(Vec::new)
+                            .push(dataset_id);
+                        async move { map }
+                    },
+                )
+                .await;
 
-        datasets
+            datasets
+        }
     }
 
     #[tokio::test]
@@ -841,21 +873,12 @@ mod test {
         let logctx = dev::test_setup_log("test_region_allocation_strat_random");
         let mut db = test_setup_database(&logctx.log).await;
         let (opctx, datastore) = datastore_test(&logctx, &db).await;
-        let test_datasets = create_test_datasets_for_region_allocation(
+        let test_datasets = TestDatasets::create(
             &opctx,
             datastore.clone(),
-            // Even though we're going to mark one sled as non-provisionable to
-            // test that logic, we aren't forcing the datasets to be on
-            // distinct sleds, so REGION_REDUNDANCY_THRESHOLD is enough.
-            REGION_REDUNDANCY_THRESHOLD,
-        )
-        .await;
-
-        let non_provisionable_dataset_id = test_datasets[0].dataset_id;
-        mark_sled_non_provisionable(
-            &datastore,
-            &opctx,
-            test_datasets[0].sled_id,
+            // We aren't forcing the datasets to be on distinct sleds, so we
+            // just need one eligible sled.
+            1,
         )
         .await;
 
@@ -891,8 +914,16 @@ mod test {
                 // Must be 3 unique datasets
                 assert!(disk_datasets.insert(dataset.id()));
 
-                // Dataset must not be non-provisionable.
-                assert_ne!(dataset.id(), non_provisionable_dataset_id);
+                // Dataset must not be eligible for provisioning.
+                if let Some(kind) =
+                    test_datasets.ineligible_dataset_ids.get(&dataset.id())
+                {
+                    panic!(
+                        "Dataset {} was ineligible for provisioning: {:?}",
+                        dataset.id(),
+                        kind
+                    );
+                }
 
                 // Must be 3 unique zpools
                 assert!(disk_zpools.insert(dataset.pool_id));
@@ -923,31 +954,16 @@ mod test {
         let (opctx, datastore) = datastore_test(&logctx, &db).await;
 
         // Create a rack with enough sleds for a successful allocation when we
-        // require 3 distinct provisionable sleds.
-        let test_datasets = create_test_datasets_for_region_allocation(
+        // require 3 distinct eligible sleds.
+        let test_datasets = TestDatasets::create(
             &opctx,
             datastore.clone(),
-            // We're going to mark one sled as non-provisionable to test that
-            // logic, and we *are* forcing the datasets to be on distinct
-            // sleds: hence threshold + 1.
-            REGION_REDUNDANCY_THRESHOLD + 1,
-        )
-        .await;
-
-        let non_provisionable_dataset_id = test_datasets[0].dataset_id;
-        mark_sled_non_provisionable(
-            &datastore,
-            &opctx,
-            test_datasets[0].sled_id,
+            // We're forcing the datasets to be on distinct sleds, hence the
+            // full REGION_REDUNDANCY_THRESHOLD.
+            REGION_REDUNDANCY_THRESHOLD,
         )
         .await;
 
-        // We need to check that our datasets end up on 3 distinct sleds, but the query doesn't return the sled ID, so we need to reverse map from dataset ID to sled ID
-        let sled_id_map: HashMap<Uuid, Uuid> = test_datasets
-            .into_iter()
-            .map(|test_dataset| (test_dataset.dataset_id, test_dataset.sled_id))
-            .collect();
-
         // Allocate regions from the datasets for this disk. Do it a few times
         // for good measure.
         for alloc_seed in 0..10 {
@@ -980,14 +996,25 @@ mod test {
                 // Must be 3 unique datasets
                 assert!(disk_datasets.insert(dataset.id()));
 
-                // Dataset must not be non-provisionable.
-                assert_ne!(dataset.id(), non_provisionable_dataset_id);
+                // Dataset must not be eligible for provisioning.
+                if let Some(kind) =
+                    test_datasets.ineligible_dataset_ids.get(&dataset.id())
+                {
+                    panic!(
+                        "Dataset {} was ineligible for provisioning: {:?}",
+                        dataset.id(),
+                        kind
+                    );
+                }
 
                 // Must be 3 unique zpools
                 assert!(disk_zpools.insert(dataset.pool_id));
 
                 // Must be 3 unique sleds
-                let sled_id = sled_id_map.get(&dataset.id()).unwrap();
+                let sled_id = test_datasets
+                    .eligible_dataset_ids
+                    .get(&dataset.id())
+                    .unwrap();
                 assert!(disk_sleds.insert(*sled_id));
 
                 assert_eq!(volume_id, region.volume_id());
@@ -1016,21 +1043,12 @@ mod test {
 
         // Create a rack without enough sleds for a successful allocation when
         // we require 3 distinct provisionable sleds.
-        let test_datasets = create_test_datasets_for_region_allocation(
+        TestDatasets::create(
             &opctx,
             datastore.clone(),
-            // Here, we need to have REGION_REDUNDANCY_THRESHOLD - 1
-            // provisionable sleds to test this failure condition. We're going
-            // to mark one sled as non-provisionable to test that logic, so we
-            // need to add 1 to that number.
-            REGION_REDUNDANCY_THRESHOLD,
-        )
-        .await;
-
-        mark_sled_non_provisionable(
-            &datastore,
-            &opctx,
-            test_datasets[0].sled_id,
+            // Here, we need to have REGION_REDUNDANCY_THRESHOLD - 1 eligible
+            // sleds to test this failure condition.
+            REGION_REDUNDANCY_THRESHOLD - 1,
         )
         .await;
 
@@ -1075,7 +1093,7 @@ mod test {
             dev::test_setup_log("test_region_allocation_is_idempotent");
         let mut db = test_setup_database(&logctx.log).await;
         let (opctx, datastore) = datastore_test(&logctx, &db).await;
-        create_test_datasets_for_region_allocation(
+        TestDatasets::create(
             &opctx,
             datastore.clone(),
             REGION_REDUNDANCY_THRESHOLD,
@@ -1220,7 +1238,7 @@ mod test {
         let mut db = test_setup_database(&logctx.log).await;
         let (opctx, datastore) = datastore_test(&logctx, &db).await;
 
-        create_test_datasets_for_region_allocation(
+        TestDatasets::create(
             &opctx,
             datastore.clone(),
             REGION_REDUNDANCY_THRESHOLD,
@@ -1321,7 +1339,7 @@ mod test {
             sled_system_hardware_for_test(),
             rack_id,
         );
-        datastore.sled_upsert(sled1).await.unwrap();
+        datastore.sled_upsert(sled1).await.unwrap().unwrap();
 
         let addr2 = "[fd00:1df::1]:12345".parse().unwrap();
         let sled2_id = "66285c18-0c79-43e0-e54f-95271f271314".parse().unwrap();
@@ -1332,7 +1350,7 @@ mod test {
             sled_system_hardware_for_test(),
             rack_id,
         );
-        datastore.sled_upsert(sled2).await.unwrap();
+        datastore.sled_upsert(sled2).await.unwrap().unwrap();
 
         let ip = datastore.next_ipv6_address(&opctx, sled1_id).await.unwrap();
         let expected_ip = Ipv6Addr::new(0xfd00, 0x1de, 0, 0, 0, 0, 1, 0);
diff --git a/nexus/db-queries/src/db/datastore/physical_disk.rs b/nexus/db-queries/src/db/datastore/physical_disk.rs
index ecb583ee29..d4e94745aa 100644
--- a/nexus/db-queries/src/db/datastore/physical_disk.rs
+++ b/nexus/db-queries/src/db/datastore/physical_disk.rs
@@ -137,10 +137,10 @@ impl DataStore {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::db::datastore::datastore_test;
     use crate::db::datastore::test::{
         sled_baseboard_for_test, sled_system_hardware_for_test,
     };
+    use crate::db::datastore::test_utils::datastore_test;
     use crate::db::model::{PhysicalDiskKind, Sled, SledUpdate};
     use dropshot::PaginationOrder;
     use nexus_test_utils::db::test_setup_database;
@@ -163,6 +163,7 @@ mod test {
         db.sled_upsert(sled_update)
             .await
             .expect("Could not upsert sled during test prep")
+            .unwrap()
     }
 
     fn list_disk_params() -> DataPageParams<'static, Uuid> {
diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs
index 99ed71a073..46a3e0af2d 100644
--- a/nexus/db-queries/src/db/datastore/rack.rs
+++ b/nexus/db-queries/src/db/datastore/rack.rs
@@ -853,10 +853,10 @@ impl DataStore {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::db::datastore::datastore_test;
     use crate::db::datastore::test::{
         sled_baseboard_for_test, sled_system_hardware_for_test,
     };
+    use crate::db::datastore::test_utils::datastore_test;
     use crate::db::datastore::Discoverability;
     use crate::db::lookup::LookupPath;
     use crate::db::model::ExternalIp;
@@ -1069,6 +1069,7 @@ mod test {
         db.sled_upsert(sled_update)
             .await
             .expect("Could not upsert sled during test prep")
+            .unwrap()
     }
 
     // Hacky macro helper to:
diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs
index eb50061272..8809f4d60d 100644
--- a/nexus/db-queries/src/db/datastore/sled.rs
+++ b/nexus/db-queries/src/db/datastore/sled.rs
@@ -9,18 +9,23 @@ use super::SQL_BATCH_SIZE;
 use crate::authz;
 use crate::context::OpContext;
 use crate::db;
+use crate::db::datastore::ValidateTransition;
 use crate::db::error::public_error_from_diesel;
 use crate::db::error::ErrorHandler;
+use crate::db::model::to_db_sled_policy;
 use crate::db::model::Sled;
 use crate::db::model::SledResource;
+use crate::db::model::SledState;
 use crate::db::model::SledUpdate;
 use crate::db::pagination::paginated;
 use crate::db::pagination::Paginator;
-use crate::db::update_and_check::UpdateAndCheck;
+use crate::db::update_and_check::{UpdateAndCheck, UpdateStatus};
 use crate::transaction_retry::OptionalError;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use chrono::Utc;
 use diesel::prelude::*;
+use nexus_types::external_api::views::SledPolicy;
+use nexus_types::external_api::views::SledProvisionPolicy;
 use nexus_types::identity::Asset;
 use omicron_common::api::external;
 use omicron_common::api::external::CreateResult;
@@ -28,16 +33,29 @@ use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::DeleteResult;
 use omicron_common::api::external::ListResultVec;
 use omicron_common::api::external::ResourceType;
+use std::fmt;
+use strum::IntoEnumIterator;
+use thiserror::Error;
 use uuid::Uuid;
 
 impl DataStore {
     /// Stores a new sled in the database.
+    ///
+    /// Produces `SledUpsertOutput::Decommissioned` if the sled is
+    /// decommissioned. This is not an error, because `sled_upsert`'s only
+    /// caller (sled-agent) is not expected to receive this error.
     pub async fn sled_upsert(
         &self,
         sled_update: SledUpdate,
-    ) -> CreateResult<Sled> {
+    ) -> CreateResult<SledUpsertOutput> {
         use db::schema::sled::dsl;
-        diesel::insert_into(dsl::sled)
+        // required for conditional upsert
+        use diesel::query_dsl::methods::FilterDsl;
+
+        // TODO: figure out what to do with time_deleted. We want to replace it
+        // with a time_decommissioned, most probably.
+
+        let query = diesel::insert_into(dsl::sled)
             .values(sled_update.clone().into_insertable())
             .on_conflict(dsl::id)
             .do_update()
@@ -52,9 +70,13 @@ impl DataStore {
                 dsl::usable_physical_ram.eq(sled_update.usable_physical_ram),
                 dsl::reservoir_size.eq(sled_update.reservoir_size),
             ))
-            .returning(Sled::as_returning())
+            .filter(dsl::sled_state.ne(SledState::Decommissioned))
+            .returning(Sled::as_returning());
+
+        let sled: Option<Sled> = query
             .get_result_async(&*self.pool_connection_unauthorized().await?)
             .await
+            .optional()
             .map_err(|e| {
                 public_error_from_diesel(
                     e,
@@ -63,7 +85,19 @@ impl DataStore {
                         &sled_update.id().to_string(),
                     ),
                 )
-            })
+            })?;
+
+        // The only situation in which a sled is not returned is if the
+        // `.filter(dsl::sled_state.ne(SledState::Decommissioned))` is not
+        // satisfied.
+        //
+        // If we want to return a sled even if it's decommissioned here, we may
+        // have to do something more complex. See
+        // https://stackoverflow.com/q/34708509.
+        match sled {
+            Some(sled) => Ok(SledUpsertOutput::Updated(sled)),
+            None => Ok(SledUpsertOutput::Decommissioned),
+        }
     }
 
     pub async fn sled_list(
@@ -194,10 +228,11 @@ impl DataStore {
                                     .and(sled_has_space_in_reservoir),
                             )
                             .filter(sled_dsl::time_deleted.is_null())
-                            // Filter out sleds that are not provisionable.
-                            .filter(sled_dsl::provision_state.eq(
-                                db::model::SledProvisionState::Provisionable,
+                            // Ensure that the sled is in-service and active.
+                            .filter(sled_dsl::sled_policy.eq(
+                                to_db_sled_policy(SledPolicy::provisionable()),
                             ))
+                            .filter(sled_dsl::sled_state.eq(SledState::Active))
                             .select(sled_dsl::id)
                             .into_boxed();
 
@@ -269,15 +304,61 @@ impl DataStore {
         Ok(())
     }
 
-    /// Sets the provision state for this sled.
+    /// Sets the provision policy for this sled.
+    ///
+    /// Errors if the sled is not in service.
+    ///
+    /// Returns the previous policy.
+    pub async fn sled_set_provision_policy(
+        &self,
+        opctx: &OpContext,
+        authz_sled: &authz::Sled,
+        policy: SledProvisionPolicy,
+    ) -> Result<SledProvisionPolicy, external::Error> {
+        match self
+            .sled_set_policy_impl(
+                opctx,
+                authz_sled,
+                SledPolicy::InService { provision_policy: policy },
+                ValidateTransition::Yes,
+            )
+            .await
+        {
+            Ok(old_policy) => Ok(old_policy
+                .provision_policy()
+                .expect("only valid policy was in-service")),
+            Err(error) => Err(error.into_external_error()),
+        }
+    }
+
+    /// Marks a sled as expunged, as directed by the operator.
     ///
-    /// Returns the previous state.
-    pub async fn sled_set_provision_state(
+    /// This is an irreversible process! It should only be called after
+    /// sufficient warning to the operator.
+    ///
+    /// This is idempotent, and it returns the old policy of the sled.
+    pub async fn sled_set_policy_to_expunged(
         &self,
         opctx: &OpContext,
         authz_sled: &authz::Sled,
-        state: db::model::SledProvisionState,
-    ) -> Result<db::model::SledProvisionState, external::Error> {
+    ) -> Result<SledPolicy, external::Error> {
+        self.sled_set_policy_impl(
+            opctx,
+            authz_sled,
+            SledPolicy::Expunged,
+            ValidateTransition::Yes,
+        )
+        .await
+        .map_err(|error| error.into_external_error())
+    }
+
+    pub(super) async fn sled_set_policy_impl(
+        &self,
+        opctx: &OpContext,
+        authz_sled: &authz::Sled,
+        new_policy: SledPolicy,
+        check: ValidateTransition,
+    ) -> Result<SledPolicy, TransitionError> {
         use db::schema::sled::dsl;
 
         opctx.authorize(authz::Action::Modify, authz_sled).await?;
@@ -285,38 +366,350 @@ impl DataStore {
         let sled_id = authz_sled.id();
         let query = diesel::update(dsl::sled)
             .filter(dsl::time_deleted.is_null())
-            .filter(dsl::id.eq(sled_id))
-            .filter(dsl::provision_state.ne(state))
+            .filter(dsl::id.eq(sled_id));
+
+        let t = SledTransition::Policy(new_policy);
+        let valid_old_policies = t.valid_old_policies();
+        let valid_old_states = t.valid_old_states();
+
+        let query = match check {
+            ValidateTransition::Yes => query
+                .filter(dsl::sled_policy.eq_any(
+                    valid_old_policies.into_iter().map(to_db_sled_policy),
+                ))
+                .filter(
+                    dsl::sled_state.eq_any(valid_old_states.iter().copied()),
+                )
+                .into_boxed(),
+            #[cfg(test)]
+            ValidateTransition::No => query.into_boxed(),
+        };
+
+        let query = query
             .set((
-                dsl::provision_state.eq(state),
+                dsl::sled_policy.eq(to_db_sled_policy(new_policy)),
                 dsl::time_modified.eq(Utc::now()),
             ))
             .check_if_exists::<Sled>(sled_id);
+
         let result = query
             .execute_and_check(&*self.pool_connection_authorized(opctx).await?)
             .await
             .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
 
-        Ok(result.found.provision_state())
+        match (check, result.status) {
+            (ValidateTransition::Yes, UpdateStatus::Updated) => {
+                Ok(result.found.policy())
+            }
+            (ValidateTransition::Yes, UpdateStatus::NotUpdatedButExists) => {
+                // Two reasons this can happen:
+                // 1. An idempotent update: this is treated as a success.
+                // 2. Invalid state transition: a failure.
+                //
+                // To differentiate between the two, check that the new policy
+                // is the same as the old policy, and that the old state is
+                // valid.
+                if result.found.policy() == new_policy
+                    && valid_old_states.contains(&result.found.state())
+                {
+                    Ok(result.found.policy())
+                } else {
+                    Err(TransitionError::InvalidTransition {
+                        current: result.found,
+                        transition: SledTransition::Policy(new_policy),
+                    })
+                }
+            }
+            #[cfg(test)]
+            (ValidateTransition::No, _) => Ok(result.found.policy()),
+        }
+    }
+
+    /// Marks the state of the sled as decommissioned, as believed by Nexus.
+    ///
+    /// This is an irreversible process! It should only be called after all
+    /// resources previously on the sled have been migrated over.
+    ///
+    /// This is idempotent, and it returns the old state of the sled.
+    ///
+    /// # Errors
+    ///
+    /// This method returns an error if the sled policy is not a state that is
+    /// valid to decommission from (i.e. if, for the current sled policy,
+    /// [`SledPolicy::is_decommissionable`] returns `false`).
+    pub async fn sled_set_state_to_decommissioned(
+        &self,
+        opctx: &OpContext,
+        authz_sled: &authz::Sled,
+    ) -> Result<SledState, external::Error> {
+        self.sled_set_state_impl(
+            opctx,
+            authz_sled,
+            SledState::Decommissioned,
+            ValidateTransition::Yes,
+        )
+        .await
+        .map_err(|error| error.into_external_error())
+    }
+
+    pub(super) async fn sled_set_state_impl(
+        &self,
+        opctx: &OpContext,
+        authz_sled: &authz::Sled,
+        new_state: SledState,
+        check: ValidateTransition,
+    ) -> Result<SledState, TransitionError> {
+        use db::schema::sled::dsl;
+
+        opctx.authorize(authz::Action::Modify, authz_sled).await?;
+
+        let sled_id = authz_sled.id();
+        let query = diesel::update(dsl::sled)
+            .filter(dsl::time_deleted.is_null())
+            .filter(dsl::id.eq(sled_id));
+
+        let t = SledTransition::State(new_state);
+        let valid_old_policies = t.valid_old_policies();
+        let valid_old_states = t.valid_old_states();
+
+        let query = match check {
+            ValidateTransition::Yes => query
+                .filter(dsl::sled_policy.eq_any(
+                    valid_old_policies.iter().copied().map(to_db_sled_policy),
+                ))
+                .filter(dsl::sled_state.eq_any(valid_old_states))
+                .into_boxed(),
+            #[cfg(test)]
+            ValidateTransition::No => query.into_boxed(),
+        };
+
+        let query = query
+            .set((
+                dsl::sled_state.eq(new_state),
+                dsl::time_modified.eq(Utc::now()),
+            ))
+            .check_if_exists::<Sled>(sled_id);
+
+        let result = query
+            .execute_and_check(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        match (check, result.status) {
+            (ValidateTransition::Yes, UpdateStatus::Updated) => {
+                Ok(result.found.state())
+            }
+            (ValidateTransition::Yes, UpdateStatus::NotUpdatedButExists) => {
+                // Two reasons this can happen:
+                // 1. An idempotent update: this is treated as a success.
+                // 2. Invalid state transition: a failure.
+                //
+                // To differentiate between the two, check that the new state
+                // is the same as the old state, and the found policy is valid.
+                if result.found.state() == new_state
+                    && valid_old_policies.contains(&result.found.policy())
+                {
+                    Ok(result.found.state())
+                } else {
+                    Err(TransitionError::InvalidTransition {
+                        current: result.found,
+                        transition: SledTransition::State(new_state),
+                    })
+                }
+            }
+            #[cfg(test)]
+            (ValidateTransition::No, _) => Ok(result.found.state()),
+        }
+    }
+}
+
+/// The result of [`DataStore::sled_upsert`].
+#[derive(Clone, Debug)]
+#[must_use]
+pub enum SledUpsertOutput {
+    /// The sled was updated.
+    Updated(Sled),
+    /// The sled was not updated because it is decommissioned.
+    Decommissioned,
+}
+
+impl SledUpsertOutput {
+    /// Returns the sled if it was updated, or panics if it was not.
+    pub fn unwrap(self) -> Sled {
+        match self {
+            SledUpsertOutput::Updated(sled) => sled,
+            SledUpsertOutput::Decommissioned => {
+                panic!("sled was decommissioned, not updated")
+            }
+        }
+    }
+}
+
+// ---
+// State transition validators
+// ---
+
+// The functions in this section return the old policies or states that are
+// valid for a new policy or state, except idempotent transitions.
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(super) enum SledTransition {
+    Policy(SledPolicy),
+    State(SledState),
+}
+
+impl SledTransition {
+    /// Returns the list of valid old policies, other than the provided one
+    /// (which is always considered valid).
+    ///
+    /// For a more descriptive listing of valid transitions, see
+    /// [`test_sled_transitions`].
+    fn valid_old_policies(&self) -> Vec<SledPolicy> {
+        use SledPolicy::*;
+        use SledProvisionPolicy::*;
+        use SledState::*;
+
+        match self {
+            SledTransition::Policy(new_policy) => match new_policy {
+                InService { provision_policy: Provisionable } => {
+                    vec![InService { provision_policy: NonProvisionable }]
+                }
+                InService { provision_policy: NonProvisionable } => {
+                    vec![InService { provision_policy: Provisionable }]
+                }
+                Expunged => SledProvisionPolicy::iter()
+                    .map(|provision_policy| InService { provision_policy })
+                    .collect(),
+            },
+            SledTransition::State(state) => {
+                match state {
+                    Active => {
+                        // Any policy is valid for the active state.
+                        SledPolicy::iter().collect()
+                    }
+                    Decommissioned => {
+                        SledPolicy::all_decommissionable().to_vec()
+                    }
+                }
+            }
+        }
+    }
+
+    /// Returns the list of valid old states, other than the provided one
+    /// (which is always considered valid).
+    ///
+    /// For a more descriptive listing of valid transitions, see
+    /// [`test_sled_transitions`].
+    fn valid_old_states(&self) -> Vec<SledState> {
+        use SledState::*;
+
+        match self {
+            SledTransition::Policy(_) => {
+                // Policies can only be transitioned in the active state. (In
+                // the future, this will include other non-decommissioned
+                // states.)
+                vec![Active]
+            }
+            SledTransition::State(state) => match state {
+                Active => vec![],
+                Decommissioned => vec![Active],
+            },
+        }
+    }
+}
+
+impl fmt::Display for SledTransition {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            SledTransition::Policy(policy) => {
+                write!(f, "policy \"{}\"", policy)
+            }
+            SledTransition::State(state) => write!(f, "state \"{}\"", state),
+        }
+    }
+}
+
+impl IntoEnumIterator for SledTransition {
+    type Iterator = std::vec::IntoIter<Self>;
+
+    fn iter() -> Self::Iterator {
+        let v: Vec<_> = SledPolicy::iter()
+            .map(SledTransition::Policy)
+            .chain(SledState::iter().map(SledTransition::State))
+            .collect();
+        v.into_iter()
+    }
+}
+
+/// An error that occurred while setting a policy or state.
+#[derive(Debug, Error)]
+#[must_use]
+pub(super) enum TransitionError {
+    /// The state transition check failed.
+    ///
+    /// The sled is returned.
+    #[error(
+        "sled id {} has current policy \"{}\" and state \"{}\" \
+        and the transition to {} is not permitted",
+        .current.id(),
+        .current.policy(),
+        .current.state(),
+        .transition,
+    )]
+    InvalidTransition {
+        /// The current sled as fetched from the database.
+        current: Sled,
+
+        /// The new policy or state that was attempted.
+        transition: SledTransition,
+    },
+
+    /// Some other kind of error occurred.
+    #[error("database error")]
+    External(#[from] external::Error),
+}
+
+impl TransitionError {
+    fn into_external_error(self) -> external::Error {
+        match self {
+            TransitionError::InvalidTransition { .. } => {
+                external::Error::conflict(self.to_string())
+            }
+            TransitionError::External(e) => e.clone(),
+        }
+    }
+
+    #[cfg(test)]
+    pub(super) fn ensure_invalid_transition(self) -> anyhow::Result<()> {
+        match self {
+            TransitionError::InvalidTransition { .. } => Ok(()),
+            TransitionError::External(e) => Err(anyhow::anyhow!(e)
+                .context("expected invalid transition, got other error")),
+        }
     }
 }
 
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::db::datastore::datastore_test;
     use crate::db::datastore::test::{
         sled_baseboard_for_test, sled_system_hardware_for_test,
     };
+    use crate::db::datastore::test_utils::{
+        datastore_test, sled_set_policy, sled_set_state, Expected,
+        IneligibleSleds,
+    };
     use crate::db::lookup::LookupPath;
     use crate::db::model::ByteCount;
     use crate::db::model::SqlU32;
+    use anyhow::{Context, Result};
+    use itertools::Itertools;
     use nexus_test_utils::db::test_setup_database;
     use nexus_types::identity::Asset;
     use omicron_common::api::external;
     use omicron_test_utils::dev;
+    use predicates::{prelude::*, BoxPredicate};
     use std::net::{Ipv6Addr, SocketAddrV6};
-    use std::num::NonZeroU32;
 
     fn rack_id() -> Uuid {
         Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap()
@@ -324,13 +717,13 @@ mod test {
 
     #[tokio::test]
     async fn upsert_sled_updates_hardware() {
-        let logctx = dev::test_setup_log("upsert_sled");
+        let logctx = dev::test_setup_log("upsert_sled_updates_hardware");
         let mut db = test_setup_database(&logctx.log).await;
         let (_opctx, datastore) = datastore_test(&logctx, &db).await;
 
         let mut sled_update = test_new_sled_update();
         let observed_sled =
-            datastore.sled_upsert(sled_update.clone()).await.unwrap();
+            datastore.sled_upsert(sled_update.clone()).await.unwrap().unwrap();
         assert_eq!(
             observed_sled.usable_hardware_threads,
             sled_update.usable_hardware_threads
@@ -362,7 +755,8 @@ mod test {
         let observed_sled = datastore
             .sled_upsert(sled_update.clone())
             .await
-            .expect("Could not upsert sled during test prep");
+            .expect("Could not upsert sled during test prep")
+            .unwrap();
         assert_eq!(
             observed_sled.usable_hardware_threads,
             sled_update.usable_hardware_threads
@@ -377,37 +771,128 @@ mod test {
         logctx.cleanup_successful();
     }
 
-    /// Test that new reservations aren't created on non-provisionable sleds.
     #[tokio::test]
-    async fn sled_reservation_create_non_provisionable() {
+    async fn upsert_sled_doesnt_update_decommissioned() {
         let logctx =
-            dev::test_setup_log("sled_reservation_create_non_provisionable");
+            dev::test_setup_log("upsert_sled_doesnt_update_decommissioned");
         let mut db = test_setup_database(&logctx.log).await;
         let (opctx, datastore) = datastore_test(&logctx, &db).await;
 
-        let sled_update = test_new_sled_update();
-        let non_provisionable_sled =
-            datastore.sled_upsert(sled_update.clone()).await.unwrap();
+        let mut sled_update = test_new_sled_update();
+        let observed_sled =
+            datastore.sled_upsert(sled_update.clone()).await.unwrap().unwrap();
+        assert_eq!(
+            observed_sled.usable_hardware_threads,
+            sled_update.usable_hardware_threads
+        );
+        assert_eq!(
+            observed_sled.usable_physical_ram,
+            sled_update.usable_physical_ram
+        );
+        assert_eq!(observed_sled.reservoir_size, sled_update.reservoir_size);
 
-        let (authz_sled, _) = LookupPath::new(&opctx, &datastore)
-            .sled_id(non_provisionable_sled.id())
-            .fetch_for(authz::Action::Modify)
-            .await
-            .unwrap();
+        // Set the sled to decommissioned (this is not a legal transition, but
+        // we don't care about sled policy in sled_upsert, just the state.)
+        sled_set_state(
+            &opctx,
+            &datastore,
+            observed_sled.id(),
+            SledState::Decommissioned,
+            ValidateTransition::No,
+            Expected::Ok(SledState::Active),
+        )
+        .await
+        .unwrap();
 
-        let old_state = datastore
-            .sled_set_provision_state(
-                &opctx,
-                &authz_sled,
-                db::model::SledProvisionState::NonProvisionable,
+        // Modify the sizes of hardware
+        sled_update.usable_hardware_threads =
+            SqlU32::new(sled_update.usable_hardware_threads.0 + 1);
+        const MIB: u64 = 1024 * 1024;
+        sled_update.usable_physical_ram = ByteCount::from(
+            external::ByteCount::try_from(
+                sled_update.usable_physical_ram.0.to_bytes() + MIB,
             )
+            .unwrap(),
+        );
+        sled_update.reservoir_size = ByteCount::from(
+            external::ByteCount::try_from(
+                sled_update.reservoir_size.0.to_bytes() + MIB,
+            )
+            .unwrap(),
+        );
+
+        // Upserting the sled should produce the `Decommisioned` variant.
+        let sled = datastore
+            .sled_upsert(sled_update.clone())
+            .await
+            .expect("updating a decommissioned sled should succeed");
+        assert!(
+            matches!(sled, SledUpsertOutput::Decommissioned),
+            "sled should be decommissioned"
+        );
+
+        // The sled should not have been updated.
+        let (_, observed_sled_2) = LookupPath::new(&opctx, &datastore)
+            .sled_id(observed_sled.id())
+            .fetch_for(authz::Action::Modify)
             .await
             .unwrap();
         assert_eq!(
-            old_state,
-            db::model::SledProvisionState::Provisionable,
-            "a newly created sled starts as provisionable"
+            observed_sled_2.usable_hardware_threads,
+            observed_sled.usable_hardware_threads,
+            "usable_hardware_threads should not have changed"
         );
+        assert_eq!(
+            observed_sled_2.usable_physical_ram,
+            observed_sled.usable_physical_ram,
+            "usable_physical_ram should not have changed"
+        );
+        assert_eq!(
+            observed_sled_2.reservoir_size, observed_sled.reservoir_size,
+            "reservoir_size should not have changed"
+        );
+
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
+
+    /// Test that new reservations aren't created on non-provisionable sleds.
+    #[tokio::test]
+    async fn sled_reservation_create_non_provisionable() {
+        let logctx =
+            dev::test_setup_log("sled_reservation_create_non_provisionable");
+        let mut db = test_setup_database(&logctx.log).await;
+        let (opctx, datastore) = datastore_test(&logctx, &db).await;
+
+        // Define some sleds that resources cannot be provisioned on.
+        let non_provisionable_sled = datastore
+            .sled_upsert(test_new_sled_update())
+            .await
+            .unwrap()
+            .unwrap();
+        let expunged_sled = datastore
+            .sled_upsert(test_new_sled_update())
+            .await
+            .unwrap()
+            .unwrap();
+        let decommissioned_sled = datastore
+            .sled_upsert(test_new_sled_update())
+            .await
+            .unwrap()
+            .unwrap();
+        let illegal_decommissioned_sled = datastore
+            .sled_upsert(test_new_sled_update())
+            .await
+            .unwrap()
+            .unwrap();
+
+        let ineligible_sleds = IneligibleSleds {
+            non_provisionable: non_provisionable_sled.id(),
+            expunged: expunged_sled.id(),
+            decommissioned: decommissioned_sled.id(),
+            illegal_decommissioned: illegal_decommissioned_sled.id(),
+        };
+        ineligible_sleds.setup(&opctx, &datastore).await.unwrap();
 
         // This should be an error since there are no provisionable sleds.
         let resources = db::model::Resources::new(
@@ -432,13 +917,7 @@ mod test {
         // Now add a provisionable sled and try again.
         let sled_update = test_new_sled_update();
         let provisionable_sled =
-            datastore.sled_upsert(sled_update.clone()).await.unwrap();
-
-        let sleds = datastore
-            .sled_list(&opctx, &first_page(NonZeroU32::new(10).unwrap()))
-            .await
-            .unwrap();
-        println!("sleds: {:?}", sleds);
+            datastore.sled_upsert(sled_update.clone()).await.unwrap().unwrap();
 
         // Try a few times to ensure that resources never get allocated to the
         // non-provisionable sled.
@@ -470,6 +949,212 @@ mod test {
         logctx.cleanup_successful();
     }
 
+    #[tokio::test]
+    async fn test_sled_transitions() {
+        // Test valid and invalid state and policy transitions.
+        let logctx = dev::test_setup_log("test_sled_transitions");
+        let mut db = test_setup_database(&logctx.log).await;
+
+        let (opctx, datastore) = datastore_test(&logctx, &db).await;
+
+        // This test generates all possible sets of transitions. Below, we list
+        // the before and after predicates for valid transitions.
+        //
+        // While it's possible to derive the list of valid transitions from the
+        // [`Transition::valid_old_policies`] and
+        // [`Transition::valid_old_states`] methods, we list them here
+        // explicitly since tests are really about writing things down twice.
+        let valid_transitions = [
+            (
+                // In-service and active sleds can be marked as expunged.
+                Before::new(
+                    predicate::in_iter(SledPolicy::all_in_service()),
+                    predicate::eq(SledState::Active),
+                ),
+                SledTransition::Policy(SledPolicy::Expunged),
+            ),
+            (
+                // The provision policy of in-service sleds can be changed, or
+                // kept the same (1 of 2).
+                Before::new(
+                    predicate::in_iter(SledPolicy::all_in_service()),
+                    predicate::eq(SledState::Active),
+                ),
+                SledTransition::Policy(SledPolicy::InService {
+                    provision_policy: SledProvisionPolicy::Provisionable,
+                }),
+            ),
+            (
+                // (2 of 2)
+                Before::new(
+                    predicate::in_iter(SledPolicy::all_in_service()),
+                    predicate::eq(SledState::Active),
+                ),
+                SledTransition::Policy(SledPolicy::InService {
+                    provision_policy: SledProvisionPolicy::NonProvisionable,
+                }),
+            ),
+            (
+                // Active sleds can be marked as active, regardless of their
+                // policy.
+                Before::new(
+                    predicate::always(),
+                    predicate::eq(SledState::Active),
+                ),
+                SledTransition::State(SledState::Active),
+            ),
+            (
+                // Expunged sleds can be marked as decommissioned.
+                Before::new(
+                    predicate::eq(SledPolicy::Expunged),
+                    predicate::eq(SledState::Active),
+                ),
+                SledTransition::State(SledState::Decommissioned),
+            ),
+            (
+                // Expunged sleds can always be marked as expunged again, as
+                // long as they aren't already decommissioned (we do not allow
+                // any transitions once a sled is decommissioned).
+                Before::new(
+                    predicate::eq(SledPolicy::Expunged),
+                    predicate::ne(SledState::Decommissioned),
+                ),
+                SledTransition::Policy(SledPolicy::Expunged),
+            ),
+            (
+                // Decommissioned sleds can always be marked as decommissioned
+                // again, as long as their policy is decommissionable.
+                Before::new(
+                    predicate::in_iter(SledPolicy::all_decommissionable()),
+                    predicate::eq(SledState::Decommissioned),
+                ),
+                SledTransition::State(SledState::Decommissioned),
+            ),
+        ];
+
+        // Generate all possible transitions.
+        let all_transitions = SledPolicy::iter()
+            .cartesian_product(SledState::iter())
+            .cartesian_product(SledTransition::iter())
+            .enumerate();
+
+        // Set up a sled to test against.
+        let sled = datastore
+            .sled_upsert(test_new_sled_update())
+            .await
+            .unwrap()
+            .unwrap();
+        let sled_id = sled.id();
+
+        for (i, ((policy, state), after)) in all_transitions {
+            test_sled_state_transitions_once(
+                &opctx,
+                &datastore,
+                sled_id,
+                policy,
+                state,
+                after,
+                &valid_transitions,
+            )
+            .await
+            .with_context(|| {
+                format!(
+                    "failed on transition {i} (policy: {policy}, \
+                        state: {state:?}, after: {after:?})",
+                )
+            })
+            .unwrap();
+        }
+
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
+
+    async fn test_sled_state_transitions_once(
+        opctx: &OpContext,
+        datastore: &DataStore,
+        sled_id: Uuid,
+        before_policy: SledPolicy,
+        before_state: SledState,
+        after: SledTransition,
+        valid_transitions: &[(Before, SledTransition)],
+    ) -> Result<()> {
+        // Is this a valid transition?
+        let is_valid = valid_transitions.iter().any(
+            |(Before(valid_policy, valid_state), valid_after)| {
+                valid_policy.eval(&before_policy)
+                    && valid_state.eval(&before_state)
+                    && valid_after == &after
+            },
+        );
+
+        // Set the sled to the initial policy and state, ignoring state
+        // transition errors (this is just to set up the initial state).
+        sled_set_policy(
+            opctx,
+            datastore,
+            sled_id,
+            before_policy,
+            ValidateTransition::No,
+            Expected::Ignore,
+        )
+        .await?;
+
+        sled_set_state(
+            opctx,
+            datastore,
+            sled_id,
+            before_state,
+            ValidateTransition::No,
+            Expected::Ignore,
+        )
+        .await?;
+
+        // Now perform the transition to the new policy or state.
+        match after {
+            SledTransition::Policy(new_policy) => {
+                let expected = if is_valid {
+                    Expected::Ok(before_policy)
+                } else {
+                    Expected::Invalid
+                };
+
+                sled_set_policy(
+                    opctx,
+                    datastore,
+                    sled_id,
+                    new_policy,
+                    ValidateTransition::Yes,
+                    expected,
+                )
+                .await?;
+            }
+            SledTransition::State(new_state) => {
+                let expected = if is_valid {
+                    Expected::Ok(before_state)
+                } else {
+                    Expected::Invalid
+                };
+
+                sled_set_state(
+                    opctx,
+                    datastore,
+                    sled_id,
+                    new_state,
+                    ValidateTransition::Yes,
+                    expected,
+                )
+                .await?;
+            }
+        }
+
+        Ok(())
+    }
+
+    // ---
+    // Helper methods
+    // ---
+
     fn test_new_sled_update() -> SledUpdate {
         let sled_id = Uuid::new_v4();
         let addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0);
@@ -482,13 +1167,17 @@ mod test {
         )
     }
 
-    /// Returns pagination parameters to fetch the first page of results for a
-    /// paginated endpoint
-    fn first_page<'a, T>(limit: NonZeroU32) -> DataPageParams<'a, T> {
-        DataPageParams {
-            marker: None,
-            direction: dropshot::PaginationOrder::Ascending,
-            limit,
+    /// Initial state for state transitions.
+    #[derive(Debug)]
+    struct Before(BoxPredicate<SledPolicy>, BoxPredicate<SledState>);
+
+    impl Before {
+        fn new<P, S>(policy: P, state: S) -> Self
+        where
+            P: Predicate<SledPolicy> + Send + Sync + 'static,
+            S: Predicate<SledState> + Send + Sync + 'static,
+        {
+            Before(policy.boxed(), state.boxed())
         }
     }
 
diff --git a/nexus/db-queries/src/db/datastore/switch_port.rs b/nexus/db-queries/src/db/datastore/switch_port.rs
index 4771768e43..842cd4bf11 100644
--- a/nexus/db-queries/src/db/datastore/switch_port.rs
+++ b/nexus/db-queries/src/db/datastore/switch_port.rs
@@ -1192,7 +1192,8 @@ impl DataStore {
 
 #[cfg(test)]
 mod test {
-    use crate::db::datastore::{datastore_test, UpdatePrecondition};
+    use crate::db::datastore::test_utils::datastore_test;
+    use crate::db::datastore::UpdatePrecondition;
     use nexus_test_utils::db::test_setup_database;
     use nexus_types::external_api::params::{
         BgpAnnounceSetCreate, BgpConfigCreate, BgpPeer, BgpPeerConfig,
diff --git a/nexus/db-queries/src/db/datastore/test_utils.rs b/nexus/db-queries/src/db/datastore/test_utils.rs
new file mode 100644
index 0000000000..6d26ad044b
--- /dev/null
+++ b/nexus/db-queries/src/db/datastore/test_utils.rs
@@ -0,0 +1,343 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Shared test-only code for the `datastore` module.
+
+use crate::authz;
+use crate::context::OpContext;
+use crate::db;
+use crate::db::datastore::ValidateTransition;
+use crate::db::lookup::LookupPath;
+use crate::db::DataStore;
+use anyhow::bail;
+use anyhow::ensure;
+use anyhow::Context;
+use anyhow::Result;
+use dropshot::test_util::LogContext;
+use nexus_db_model::SledState;
+use nexus_types::external_api::views::SledPolicy;
+use nexus_types::external_api::views::SledProvisionPolicy;
+use omicron_test_utils::dev::db::CockroachInstance;
+use std::sync::Arc;
+use strum::EnumCount;
+use uuid::Uuid;
+
+/// Constructs a DataStore for use in test suites that has preloaded the
+/// built-in users, roles, and role assignments that are needed for basic
+/// operation
+#[cfg(test)]
+pub async fn datastore_test(
+    logctx: &LogContext,
+    db: &CockroachInstance,
+) -> (OpContext, Arc<DataStore>) {
+    use crate::authn;
+
+    let cfg = db::Config { url: db.pg_config().clone() };
+    let pool = Arc::new(db::Pool::new(&logctx.log, &cfg));
+    let datastore =
+        Arc::new(DataStore::new(&logctx.log, pool, None).await.unwrap());
+
+    // Create an OpContext with the credentials of "db-init" just for the
+    // purpose of loading the built-in users, roles, and assignments.
+    let opctx = OpContext::for_background(
+        logctx.log.new(o!()),
+        Arc::new(authz::Authz::new(&logctx.log)),
+        authn::Context::internal_db_init(),
+        Arc::clone(&datastore),
+    );
+
+    // TODO: Can we just call "Populate" instead of doing this?
+    let rack_id = Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap();
+    datastore.load_builtin_users(&opctx).await.unwrap();
+    datastore.load_builtin_roles(&opctx).await.unwrap();
+    datastore.load_builtin_role_asgns(&opctx).await.unwrap();
+    datastore.load_builtin_silos(&opctx).await.unwrap();
+    datastore.load_builtin_projects(&opctx).await.unwrap();
+    datastore.load_builtin_vpcs(&opctx).await.unwrap();
+    datastore.load_silo_users(&opctx).await.unwrap();
+    datastore.load_silo_user_role_assignments(&opctx).await.unwrap();
+    datastore
+        .load_builtin_fleet_virtual_provisioning_collection(&opctx)
+        .await
+        .unwrap();
+    datastore.load_builtin_rack_data(&opctx, rack_id).await.unwrap();
+
+    // Create an OpContext with the credentials of "test-privileged" for general
+    // testing.
+    let opctx =
+        OpContext::for_tests(logctx.log.new(o!()), Arc::clone(&datastore));
+
+    (opctx, datastore)
+}
+
+/// Denotes a specific way in which a sled is ineligible.
+///
+/// This should match the list of sleds in `IneligibleSleds`.
+#[derive(
+    Clone, Copy, Debug, Eq, PartialEq, PartialOrd, Ord, Hash, EnumCount,
+)]
+pub(super) enum IneligibleSledKind {
+    NonProvisionable,
+    Expunged,
+    Decommissioned,
+    IllegalDecommissioned,
+}
+
+/// Specifies sleds to be marked as ineligible for provisioning.
+///
+/// This is less error-prone than several places duplicating this logic.
+#[derive(Debug)]
+pub(super) struct IneligibleSleds {
+    pub(super) non_provisionable: Uuid,
+    pub(super) expunged: Uuid,
+    pub(super) decommissioned: Uuid,
+    pub(super) illegal_decommissioned: Uuid,
+}
+
+impl IneligibleSleds {
+    pub(super) fn iter(
+        &self,
+    ) -> impl Iterator<Item = (IneligibleSledKind, Uuid)> {
+        [
+            (IneligibleSledKind::NonProvisionable, self.non_provisionable),
+            (IneligibleSledKind::Expunged, self.expunged),
+            (IneligibleSledKind::Decommissioned, self.decommissioned),
+            (
+                IneligibleSledKind::IllegalDecommissioned,
+                self.illegal_decommissioned,
+            ),
+        ]
+        .into_iter()
+    }
+
+    /// Marks the provided sleds as ineligible for provisioning.
+    ///
+    /// Assumes that:
+    ///
+    /// * the sleds have just been set up and are in the default state.
+    /// * the UUIDs are all distinct.
+    pub(super) async fn setup(
+        &self,
+        opctx: &OpContext,
+        datastore: &DataStore,
+    ) -> Result<()> {
+        let non_provisionable_fut = async {
+            sled_set_policy(
+                &opctx,
+                &datastore,
+                self.non_provisionable,
+                SledPolicy::InService {
+                    provision_policy: SledProvisionPolicy::NonProvisionable,
+                },
+                ValidateTransition::Yes,
+                Expected::Ok(SledPolicy::provisionable()),
+            )
+            .await
+            .with_context(|| {
+                format!(
+                    "failed to set non-provisionable policy for sled {}",
+                    self.non_provisionable
+                )
+            })
+        };
+
+        let expunged_fut = async {
+            sled_set_policy(
+                &opctx,
+                &datastore,
+                self.expunged,
+                SledPolicy::Expunged,
+                ValidateTransition::Yes,
+                Expected::Ok(SledPolicy::provisionable()),
+            )
+            .await
+            .with_context(|| {
+                format!(
+                    "failed to set expunged policy for sled {}",
+                    self.non_provisionable
+                )
+            })
+        };
+
+        // Legally, we must set the policy to expunged before setting the state
+        // to decommissioned. (In the future, we'll want to test graceful
+        // removal as well.)
+        let decommissioned_fut = async {
+            sled_set_policy(
+                &opctx,
+                &datastore,
+                self.decommissioned,
+                SledPolicy::Expunged,
+                ValidateTransition::Yes,
+                Expected::Ok(SledPolicy::provisionable()),
+            )
+            .await
+            .with_context(|| {
+                format!(
+                    "failed to set expunged policy for sled {}, \
+                     as prerequisite for decommissioning",
+                    self.decommissioned
+                )
+            })?;
+
+            sled_set_state(
+                &opctx,
+                &datastore,
+                self.decommissioned,
+                SledState::Decommissioned,
+                ValidateTransition::Yes,
+                Expected::Ok(SledState::Active),
+            )
+            .await
+            .with_context(|| {
+                format!(
+                    "failed to set decommissioned state for sled {}",
+                    self.decommissioned
+                )
+            })
+        };
+
+        // This is _not_ a legal state, BUT we test it out to ensure that if
+        // the system somehow enters this state anyway, we don't try and
+        // provision resources on it.
+        let illegal_decommissioned_fut = async {
+            sled_set_state(
+                &opctx,
+                &datastore,
+                self.illegal_decommissioned,
+                SledState::Decommissioned,
+                ValidateTransition::No,
+                Expected::Ok(SledState::Active),
+            )
+            .await
+            .with_context(|| {
+                format!(
+                    "failed to illegally set decommissioned state for sled {}",
+                    self.illegal_decommissioned
+                )
+            })
+        };
+
+        // We're okay cancelling the rest of the futures if one of them fails,
+        // since the overall test is going to fail anyway. Hence try_join
+        // rather than join_then_try.
+        futures::try_join!(
+            non_provisionable_fut,
+            expunged_fut,
+            decommissioned_fut,
+            illegal_decommissioned_fut
+        )?;
+
+        Ok(())
+    }
+}
+
+pub(super) async fn sled_set_policy(
+    opctx: &OpContext,
+    datastore: &DataStore,
+    sled_id: Uuid,
+    new_policy: SledPolicy,
+    check: ValidateTransition,
+    expected_old_policy: Expected<SledPolicy>,
+) -> Result<()> {
+    let (authz_sled, _) = LookupPath::new(&opctx, &datastore)
+        .sled_id(sled_id)
+        .fetch_for(authz::Action::Modify)
+        .await
+        .unwrap();
+
+    let res = datastore
+        .sled_set_policy_impl(opctx, &authz_sled, new_policy, check)
+        .await;
+    match expected_old_policy {
+        Expected::Ok(expected) => {
+            let actual = res.context(
+                "failed transition that was expected to be successful",
+            )?;
+            ensure!(
+                actual == expected,
+                "actual old policy ({actual}) is not \
+                 the same as expected ({expected})"
+            );
+        }
+        Expected::Invalid => match res {
+            Ok(old_policy) => {
+                bail!(
+                    "expected an invalid state transition error, \
+                     but transition was accepted with old policy: \
+                     {old_policy}"
+                )
+            }
+            Err(error) => {
+                error.ensure_invalid_transition()?;
+            }
+        },
+        Expected::Ignore => {
+            // The return value is ignored.
+        }
+    }
+
+    Ok(())
+}
+
+pub(super) async fn sled_set_state(
+    opctx: &OpContext,
+    datastore: &DataStore,
+    sled_id: Uuid,
+    new_state: SledState,
+    check: ValidateTransition,
+    expected_old_state: Expected<SledState>,
+) -> Result<()> {
+    let (authz_sled, _) = LookupPath::new(&opctx, &datastore)
+        .sled_id(sled_id)
+        .fetch_for(authz::Action::Modify)
+        .await
+        .unwrap();
+
+    let res = datastore
+        .sled_set_state_impl(&opctx, &authz_sled, new_state, check)
+        .await;
+    match expected_old_state {
+        Expected::Ok(expected) => {
+            let actual = res.context(
+                "failed transition that was expected to be successful",
+            )?;
+            ensure!(
+                actual == expected,
+                "actual old state ({actual:?}) \
+                 is not the same as expected ({expected:?})"
+            );
+        }
+        Expected::Invalid => match res {
+            Ok(old_state) => {
+                bail!(
+                    "expected an invalid state transition error, \
+                    but transition was accepted with old state: \
+                    {old_state:?}"
+                )
+            }
+            Err(error) => {
+                error.ensure_invalid_transition()?;
+            }
+        },
+        Expected::Ignore => {
+            // The return value is ignored.
+        }
+    }
+
+    Ok(())
+}
+
+/// For a transition, describes the expected value of the old state.
+pub(super) enum Expected<T> {
+    /// The transition is expected to successful, with the provided old
+    /// value.
+    Ok(T),
+
+    /// The transition is expected to be invalid.
+    Invalid,
+
+    /// The return value is ignored.
+    Ignore,
+}
diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs
index d0b093ff45..374ef2cf73 100644
--- a/nexus/db-queries/src/db/datastore/volume.rs
+++ b/nexus/db-queries/src/db/datastore/volume.rs
@@ -1064,7 +1064,7 @@ pub fn read_only_resources_associated_with_volume(
 mod tests {
     use super::*;
 
-    use crate::db::datastore::datastore_test;
+    use crate::db::datastore::test_utils::datastore_test;
     use nexus_test_utils::db::test_setup_database;
     use omicron_test_utils::dev;
 
diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs
index 4f0245e283..4626301f76 100644
--- a/nexus/db-queries/src/db/datastore/vpc.rs
+++ b/nexus/db-queries/src/db/datastore/vpc.rs
@@ -1171,7 +1171,7 @@ impl DataStore {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::db::datastore::datastore_test;
+    use crate::db::datastore::test_utils::datastore_test;
     use crate::db::model::Project;
     use crate::db::queries::vpc::MAX_VNI_SEARCH_RANGE_SIZE;
     use nexus_test_utils::db::test_setup_database;
diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs
index 18ea369685..050c1dcfe9 100644
--- a/nexus/db-queries/src/db/lookup.rs
+++ b/nexus/db-queries/src/db/lookup.rs
@@ -950,7 +950,8 @@ mod test {
         let logctx = dev::test_setup_log("test_lookup");
         let mut db = test_setup_database(&logctx.log).await;
         let (_, datastore) =
-            crate::db::datastore::datastore_test(&logctx, &db).await;
+            crate::db::datastore::test_utils::datastore_test(&logctx, &db)
+                .await;
         let opctx =
             OpContext::for_tests(logctx.log.new(o!()), Arc::clone(&datastore));
         let project_name: Name = Name("my-project".parse().unwrap());
diff --git a/nexus/db-queries/src/db/pool_connection.rs b/nexus/db-queries/src/db/pool_connection.rs
index 66fb125a7c..f419ba6852 100644
--- a/nexus/db-queries/src/db/pool_connection.rs
+++ b/nexus/db-queries/src/db/pool_connection.rs
@@ -59,9 +59,10 @@ static CUSTOM_TYPE_KEYS: &'static [&'static str] = &[
     "router_route_kind",
     "saga_state",
     "service_kind",
-    "sled_provision_state",
+    "sled_policy",
     "sled_resource_kind",
     "sled_role",
+    "sled_state",
     "snapshot_state",
     "sp_type",
     "switch_interface_kind",
diff --git a/nexus/db-queries/src/db/queries/external_ip.rs b/nexus/db-queries/src/db/queries/external_ip.rs
index 7456e6e6bb..739c0b0809 100644
--- a/nexus/db-queries/src/db/queries/external_ip.rs
+++ b/nexus/db-queries/src/db/queries/external_ip.rs
@@ -906,7 +906,8 @@ mod tests {
             let logctx = dev::test_setup_log(test_name);
             let log = logctx.log.new(o!());
             let db = test_setup_database(&log).await;
-            crate::db::datastore::datastore_test(&logctx, &db).await;
+            crate::db::datastore::test_utils::datastore_test(&logctx, &db)
+                .await;
             let cfg = crate::db::Config { url: db.pg_config().clone() };
             let pool = Arc::new(crate::db::Pool::new(&logctx.log, &cfg));
             let db_datastore = Arc::new(
diff --git a/nexus/db-queries/src/db/queries/network_interface.rs b/nexus/db-queries/src/db/queries/network_interface.rs
index a22c80b232..d96b26c9b3 100644
--- a/nexus/db-queries/src/db/queries/network_interface.rs
+++ b/nexus/db-queries/src/db/queries/network_interface.rs
@@ -1953,7 +1953,8 @@ mod tests {
             let log = logctx.log.new(o!());
             let db = test_setup_database(&log).await;
             let (opctx, db_datastore) =
-                crate::db::datastore::datastore_test(&logctx, &db).await;
+                crate::db::datastore::test_utils::datastore_test(&logctx, &db)
+                    .await;
 
             let authz_silo = opctx.authn.silo_required().unwrap();
 
diff --git a/nexus/db-queries/src/db/queries/region_allocation.rs b/nexus/db-queries/src/db/queries/region_allocation.rs
index 3c37bf6b2e..43e1750812 100644
--- a/nexus/db-queries/src/db/queries/region_allocation.rs
+++ b/nexus/db-queries/src/db/queries/region_allocation.rs
@@ -27,6 +27,9 @@ use nexus_db_model::queries::region_allocation::{
     proposed_dataset_changes, shuffled_candidate_datasets, updated_datasets,
 };
 use nexus_db_model::schema;
+use nexus_db_model::to_db_sled_policy;
+use nexus_db_model::SledState;
+use nexus_types::external_api::views::SledPolicy;
 use omicron_common::api::external;
 use omicron_common::nexus_config::RegionAllocationStrategy;
 
@@ -321,14 +324,16 @@ impl CandidateZpools {
             .on(zpool_dsl::id.eq(old_zpool_usage::dsl::pool_id))
             .inner_join(with_sled);
 
-        let sled_is_provisionable = sled_dsl::provision_state
-            .eq(crate::db::model::SledProvisionState::Provisionable);
+        let sled_is_provisionable = sled_dsl::sled_policy
+            .eq(to_db_sled_policy(SledPolicy::provisionable()));
+        let sled_is_active = sled_dsl::sled_state.eq(SledState::Active);
 
         let base_query = old_zpool_usage
             .query_source()
             .inner_join(with_zpool)
             .filter(it_will_fit)
             .filter(sled_is_provisionable)
+            .filter(sled_is_active)
             .select((old_zpool_usage::dsl::pool_id,));
 
         let query = if distinct_sleds {
diff --git a/nexus/db-queries/src/transaction_retry.rs b/nexus/db-queries/src/transaction_retry.rs
index 6b5098158b..74a94a0b8f 100644
--- a/nexus/db-queries/src/transaction_retry.rs
+++ b/nexus/db-queries/src/transaction_retry.rs
@@ -270,7 +270,7 @@ impl<E: std::fmt::Debug> OptionalError<E> {
 mod test {
     use super::*;
 
-    use crate::db::datastore::datastore_test;
+    use crate::db::datastore::test_utils::datastore_test;
     use nexus_test_utils::db::test_setup_database;
     use omicron_test_utils::dev;
     use oximeter::types::FieldValue;
diff --git a/nexus/deployment/src/blueprint_builder.rs b/nexus/deployment/src/blueprint_builder.rs
index 86a9b8da6e..2263a99ce0 100644
--- a/nexus/deployment/src/blueprint_builder.rs
+++ b/nexus/deployment/src/blueprint_builder.rs
@@ -724,7 +724,8 @@ impl<'a> BlueprintZones<'a> {
 #[cfg(test)]
 pub mod test {
     use super::*;
-    use nexus_types::external_api::views::SledProvisionState;
+    use nexus_types::external_api::views::SledPolicy;
+    use nexus_types::external_api::views::SledState;
     use omicron_common::address::IpRange;
     use omicron_common::address::Ipv4Range;
     use omicron_common::address::Ipv6Subnet;
@@ -736,14 +737,26 @@ pub mod test {
     };
     use std::str::FromStr;
 
-    /// Returns a collection and policy describing a pretty simple system
-    pub fn example() -> (Collection, Policy) {
+    pub const DEFAULT_N_SLEDS: usize = 3;
+
+    /// Returns a collection and policy describing a pretty simple system.
+    ///
+    /// `n_sleds` is the number of sleds supported. Currently, this value can
+    /// be anywhere between 0 and 5. (More can be added in the future if
+    /// necessary.)
+    pub fn example(n_sleds: usize) -> (Collection, Policy) {
         let mut builder = nexus_inventory::CollectionBuilder::new("test-suite");
 
+        if n_sleds > 5 {
+            panic!("example() only supports up to 5 sleds, but got {n_sleds}");
+        }
+
         let sled_ids = [
             "72443b6c-b8bb-4ffa-ab3a-aeaa428ed79b",
             "a5f3db3a-61aa-4f90-ad3e-02833c253bf5",
             "0d168386-2551-44e8-98dd-ae7a7570f8a0",
+            "aaaaa1a1-0c3f-4928-aba7-6ec5c1db05f7",
+            "85e88acb-7b86-45ff-9c88-734e1da71c3d",
         ];
         let mut policy = Policy {
             sleds: BTreeMap::new(),
@@ -771,7 +784,7 @@ pub mod test {
             })
         };
 
-        for sled_id_str in sled_ids.iter() {
+        for sled_id_str in sled_ids.iter().take(n_sleds) {
             let sled_id: Uuid = sled_id_str.parse().unwrap();
             let sled_ip = policy_add_sled(&mut policy, sled_id);
             let serial_number = format!("s{}", policy.sleds.len());
@@ -900,7 +913,8 @@ pub mod test {
         policy.sleds.insert(
             sled_id,
             SledResources {
-                provision_state: SledProvisionState::Provisionable,
+                policy: SledPolicy::provisionable(),
+                state: SledState::Active,
                 zpools,
                 subnet,
             },
@@ -931,7 +945,7 @@ pub mod test {
     fn test_initial() {
         // Test creating a blueprint from a collection and verifying that it
         // describes no changes.
-        let (collection, policy) = example();
+        let (collection, policy) = example(DEFAULT_N_SLEDS);
         let blueprint_initial =
             BlueprintBuilder::build_initial_from_collection(
                 &collection,
@@ -977,7 +991,7 @@ pub mod test {
 
     #[test]
     fn test_basic() {
-        let (collection, mut policy) = example();
+        let (collection, mut policy) = example(DEFAULT_N_SLEDS);
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
             &collection,
             Generation::new(),
@@ -1096,7 +1110,7 @@ pub mod test {
 
     #[test]
     fn test_add_nexus_with_no_existing_nexus_zones() {
-        let (mut collection, policy) = example();
+        let (mut collection, policy) = example(DEFAULT_N_SLEDS);
 
         // We don't care about the internal DNS version here.
         let internal_dns_version = Generation::new();
@@ -1147,7 +1161,7 @@ pub mod test {
 
     #[test]
     fn test_add_nexus_error_cases() {
-        let (mut collection, policy) = example();
+        let (mut collection, policy) = example(DEFAULT_N_SLEDS);
 
         // We don't care about the internal DNS version here.
         let internal_dns_version = Generation::new();
@@ -1259,7 +1273,7 @@ pub mod test {
 
     #[test]
     fn test_invalid_parent_blueprint_two_zones_with_same_external_ip() {
-        let (mut collection, policy) = example();
+        let (mut collection, policy) = example(DEFAULT_N_SLEDS);
 
         // We should fail if the parent blueprint claims to contain two
         // zones with the same external IP. Skim through the zones, copy the
@@ -1310,7 +1324,7 @@ pub mod test {
 
     #[test]
     fn test_invalid_parent_blueprint_two_nexus_zones_with_same_nic_ip() {
-        let (mut collection, policy) = example();
+        let (mut collection, policy) = example(DEFAULT_N_SLEDS);
 
         // We should fail if the parent blueprint claims to contain two
         // Nexus zones with the same NIC IP. Skim through the zones, copy
@@ -1359,7 +1373,7 @@ pub mod test {
 
     #[test]
     fn test_invalid_parent_blueprint_two_zones_with_same_vnic_mac() {
-        let (mut collection, policy) = example();
+        let (mut collection, policy) = example(DEFAULT_N_SLEDS);
 
         // We should fail if the parent blueprint claims to contain two
         // zones with the same service vNIC MAC address. Skim through the
diff --git a/nexus/deployment/src/planner.rs b/nexus/deployment/src/planner.rs
index 7973157068..0773fec2bf 100644
--- a/nexus/deployment/src/planner.rs
+++ b/nexus/deployment/src/planner.rs
@@ -12,7 +12,7 @@ use crate::blueprint_builder::EnsureMultiple;
 use crate::blueprint_builder::Error;
 use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::Policy;
-use nexus_types::external_api::views::SledProvisionState;
+use nexus_types::external_api::views::SledState;
 use nexus_types::inventory::Collection;
 use omicron_common::api::external::Generation;
 use slog::{info, warn, Logger};
@@ -84,6 +84,17 @@ impl<'a> Planner<'a> {
         let mut sleds_ineligible_for_services = BTreeSet::new();
 
         for (sled_id, sled_info) in &self.policy.sleds {
+            // Decommissioned sleds don't get any services. (This is an
+            // explicit match so that when more states are added, this fails to
+            // compile.)
+            match sled_info.state {
+                SledState::Decommissioned => {
+                    sleds_ineligible_for_services.insert(*sled_id);
+                    continue;
+                }
+                SledState::Active => {}
+            }
+
             // Check for an NTP zone.  Every sled should have one.  If it's not
             // there, all we can do is provision that one zone.  We have to wait
             // for that to succeed and synchronize the clock before we can
@@ -175,10 +186,8 @@ impl<'a> Planner<'a> {
         // assumption that there is something amiss with them.
         sleds_ineligible_for_services.extend(
             self.policy.sleds.iter().filter_map(|(sled_id, sled_info)| {
-                match sled_info.provision_state {
-                    SledProvisionState::Provisionable => None,
-                    SledProvisionState::NonProvisionable => Some(*sled_id),
-                }
+                (!sled_info.is_eligible_for_discretionary_services())
+                    .then_some(*sled_id)
             }),
         );
 
@@ -311,9 +320,12 @@ mod test {
     use crate::blueprint_builder::test::example;
     use crate::blueprint_builder::test::policy_add_sled;
     use crate::blueprint_builder::test::verify_blueprint;
+    use crate::blueprint_builder::test::DEFAULT_N_SLEDS;
     use crate::blueprint_builder::BlueprintBuilder;
     use nexus_inventory::now_db_precision;
-    use nexus_types::external_api::views::SledProvisionState;
+    use nexus_types::external_api::views::SledPolicy;
+    use nexus_types::external_api::views::SledProvisionPolicy;
+    use nexus_types::external_api::views::SledState;
     use nexus_types::inventory::OmicronZoneType;
     use nexus_types::inventory::OmicronZonesFound;
     use omicron_common::api::external::Generation;
@@ -328,7 +340,7 @@ mod test {
         let internal_dns_version = Generation::new();
 
         // Use our example inventory collection.
-        let (mut collection, mut policy) = example();
+        let (mut collection, mut policy) = example(DEFAULT_N_SLEDS);
 
         // Build the initial blueprint.  We don't bother verifying it here
         // because there's a separate test for that.
@@ -509,7 +521,7 @@ mod test {
         // Use our example inventory collection as a starting point, but strip
         // it down to just one sled.
         let (sled_id, collection, mut policy) = {
-            let (mut collection, mut policy) = example();
+            let (mut collection, mut policy) = example(DEFAULT_N_SLEDS);
 
             // Pick one sled ID to keep and remove the rest.
             let keep_sled_id =
@@ -593,7 +605,7 @@ mod test {
         );
 
         // Use our example inventory collection as a starting point.
-        let (collection, mut policy) = example();
+        let (collection, mut policy) = example(DEFAULT_N_SLEDS);
 
         // Build the initial blueprint.
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
@@ -674,7 +686,12 @@ mod test {
         );
 
         // Use our example inventory collection as a starting point.
-        let (collection, mut policy) = example();
+        //
+        // Request two extra sleds here so we test non-provisionable, expunged,
+        // and decommissioned sleds. (When we add more kinds of
+        // non-provisionable states in the future, we'll have to add more
+        // sleds.)
+        let (collection, mut policy) = example(5);
 
         // Build the initial blueprint.
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
@@ -685,8 +702,8 @@ mod test {
         )
         .expect("failed to create initial blueprint");
 
-        // This blueprint should only have 3 Nexus zones: one on each sled.
-        assert_eq!(blueprint1.omicron_zones.len(), 3);
+        // This blueprint should only have 5 Nexus zones: one on each sled.
+        assert_eq!(blueprint1.omicron_zones.len(), 5);
         for sled_config in blueprint1.omicron_zones.values() {
             assert_eq!(
                 sled_config
@@ -698,16 +715,39 @@ mod test {
             );
         }
 
-        // Arbitrarily choose one of the sleds and mark it non-provisionable.
+        // Arbitrarily choose some of the sleds and mark them non-provisionable
+        // in various ways.
+        let mut sleds_iter = policy.sleds.iter_mut();
+
         let nonprovisionable_sled_id = {
-            let (sled_id, resources) =
-                policy.sleds.iter_mut().next().expect("no sleds");
-            resources.provision_state = SledProvisionState::NonProvisionable;
+            let (sled_id, resources) = sleds_iter.next().expect("no sleds");
+            resources.policy = SledPolicy::InService {
+                provision_policy: SledProvisionPolicy::NonProvisionable,
+            };
+            *sled_id
+        };
+        let expunged_sled_id = {
+            let (sled_id, resources) = sleds_iter.next().expect("no sleds");
+            resources.policy = SledPolicy::Expunged;
+            *sled_id
+        };
+        let decommissioned_sled_id = {
+            let (sled_id, resources) = sleds_iter.next().expect("no sleds");
+            resources.state = SledState::Decommissioned;
             *sled_id
         };
 
-        // Now run the planner with a high number of target Nexus zones.
-        policy.target_nexus_zone_count = 14;
+        // Now run the planner with a high number of target Nexus zones. The
+        // number (16) is chosen such that:
+        //
+        // * we start with 5 sleds
+        // * we need to add 11 Nexus zones
+        // * there are two sleds eligible for provisioning
+        // * => 5 or 6 new Nexus zones per sled
+        //
+        // When the planner gets smarter about removing zones from expunged
+        // and/or removed sleds, we'll have to adjust this number.
+        policy.target_nexus_zone_count = 16;
         let blueprint2 = Planner::new_based_on(
             logctx.log.clone(),
             &blueprint1,
@@ -727,13 +767,15 @@ mod test {
         let sleds = diff.sleds_changed().collect::<Vec<_>>();
 
         // Only 2 of the 3 sleds should get additional Nexus zones. We expect a
-        // total of 11 new Nexus zones, which should be spread evenly across the
+        // total of 12 new Nexus zones, which should be spread evenly across the
         // two sleds (one gets 6 and the other gets 5), while the
         // non-provisionable sled should be unchanged.
         assert_eq!(sleds.len(), 2);
         let mut total_new_nexus_zones = 0;
         for (sled_id, sled_changes) in sleds {
             assert!(sled_id != nonprovisionable_sled_id);
+            assert!(sled_id != expunged_sled_id);
+            assert!(sled_id != decommissioned_sled_id);
             assert_eq!(sled_changes.zones_removed().count(), 0);
             assert_eq!(sled_changes.zones_changed().count(), 0);
             let zones = sled_changes.zones_added().collect::<Vec<_>>();
diff --git a/nexus/src/app/background/blueprint_execution.rs b/nexus/src/app/background/blueprint_execution.rs
index 32797facbf..373f023288 100644
--- a/nexus/src/app/background/blueprint_execution.rs
+++ b/nexus/src/app/background/blueprint_execution.rs
@@ -186,7 +186,8 @@ mod test {
             datastore
                 .sled_upsert(update)
                 .await
-                .expect("Failed to insert sled to db");
+                .expect("Failed to insert sled to db")
+                .unwrap();
         }
 
         let (blueprint_tx, blueprint_rx) = watch::channel(None);
diff --git a/nexus/src/app/background/inventory_collection.rs b/nexus/src/app/background/inventory_collection.rs
index 044e5a2234..c0d64d554a 100644
--- a/nexus/src/app/background/inventory_collection.rs
+++ b/nexus/src/app/background/inventory_collection.rs
@@ -338,7 +338,7 @@ mod test {
                 },
                 rack_id,
             );
-            sleds.push(datastore.sled_upsert(sled).await.unwrap());
+            sleds.push(datastore.sled_upsert(sled).await.unwrap().unwrap());
         }
 
         // The same enumerator should immediately find all the new sleds.
diff --git a/nexus/src/app/deployment.rs b/nexus/src/app/deployment.rs
index 61ce803d13..adf6119c5c 100644
--- a/nexus/src/app/deployment.rs
+++ b/nexus/src/app/deployment.rs
@@ -160,7 +160,8 @@ impl super::Nexus {
                     .remove(&sled_id)
                     .unwrap_or_else(BTreeSet::new);
                 let sled_info = SledResources {
-                    provision_state: sled_row.provision_state().into(),
+                    policy: sled_row.policy(),
+                    state: sled_row.state().into(),
                     subnet,
                     zpools,
                 };
diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs
index ec3f11dc6f..88955d78e9 100644
--- a/nexus/src/app/sled.rs
+++ b/nexus/src/app/sled.rs
@@ -11,9 +11,11 @@ use crate::internal_api::params::{
 use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db;
+use nexus_db_queries::db::datastore::SledUpsertOutput;
 use nexus_db_queries::db::lookup;
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_db_queries::db::model::DatasetKind;
+use nexus_types::external_api::views::SledProvisionPolicy;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::ListResultVec;
@@ -68,7 +70,19 @@ impl super::Nexus {
             },
             self.rack_id,
         );
-        self.db_datastore.sled_upsert(sled).await?;
+        match self.db_datastore.sled_upsert(sled).await? {
+            SledUpsertOutput::Updated(_) => {}
+            SledUpsertOutput::Decommissioned => {
+                // We currently don't bubble up errors for decommissioned sleds
+                // -- if it ever happens, a decommissioned sled-agent doesn't
+                // know about that.
+                warn!(
+                    self.log,
+                    "decommissioned sled-agent reached out for upserts";
+                    "sled_uuid" => id.to_string()
+                );
+            }
+        }
 
         Ok(())
     }
@@ -146,17 +160,17 @@ impl super::Nexus {
             .await
     }
 
-    /// Returns the old state.
-    pub(crate) async fn sled_set_provision_state(
+    /// Returns the old provision policy.
+    pub(crate) async fn sled_set_provision_policy(
         &self,
         opctx: &OpContext,
         sled_lookup: &lookup::Sled<'_>,
-        state: db::model::SledProvisionState,
-    ) -> Result<db::model::SledProvisionState, Error> {
+        new_policy: SledProvisionPolicy,
+    ) -> Result<SledProvisionPolicy, Error> {
         let (authz_sled,) =
             sled_lookup.lookup_for(authz::Action::Modify).await?;
         self.db_datastore
-            .sled_set_provision_state(opctx, &authz_sled, state)
+            .sled_set_provision_policy(opctx, &authz_sled, new_policy)
             .await
     }
 
diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs
index fd18cb2dab..b007cc6217 100644
--- a/nexus/src/external_api/http_entrypoints.rs
+++ b/nexus/src/external_api/http_entrypoints.rs
@@ -225,7 +225,7 @@ pub(crate) fn external_api() -> NexusApiDescription {
         api.register(rack_view)?;
         api.register(sled_list)?;
         api.register(sled_view)?;
-        api.register(sled_set_provision_state)?;
+        api.register(sled_set_provision_policy)?;
         api.register(sled_instance_list)?;
         api.register(sled_physical_disk_list)?;
         api.register(physical_disk_list)?;
@@ -5166,38 +5166,34 @@ async fn sled_view(
     apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
 }
 
-/// Set sled provision state
+/// Set sled provision policy
 #[endpoint {
     method = PUT,
-    path = "/v1/system/hardware/sleds/{sled_id}/provision-state",
+    path = "/v1/system/hardware/sleds/{sled_id}/provision-policy",
     tags = ["system/hardware"],
 }]
-async fn sled_set_provision_state(
+async fn sled_set_provision_policy(
     rqctx: RequestContext<Arc<ServerContext>>,
     path_params: Path<params::SledPath>,
-    new_provision_state: TypedBody<params::SledProvisionStateParams>,
-) -> Result<HttpResponseOk<params::SledProvisionStateResponse>, HttpError> {
+    new_provision_state: TypedBody<params::SledProvisionPolicyParams>,
+) -> Result<HttpResponseOk<params::SledProvisionPolicyResponse>, HttpError> {
     let apictx = rqctx.context();
     let handler = async {
         let nexus = &apictx.nexus;
 
         let path = path_params.into_inner();
-        let provision_state = new_provision_state.into_inner().state;
+        let new_state = new_provision_state.into_inner().state;
 
         let opctx = crate::context::op_context_for_external_api(&rqctx).await?;
-        // Convert the external `SledProvisionState` into our internal data model.
-        let new_state = db::model::SledProvisionState::from(provision_state);
 
         let sled_lookup = nexus.sled_lookup(&opctx, &path.sled_id)?;
 
         let old_state = nexus
-            .sled_set_provision_state(&opctx, &sled_lookup, new_state)
+            .sled_set_provision_policy(&opctx, &sled_lookup, new_state)
             .await?;
 
-        let response = params::SledProvisionStateResponse {
-            old_state: old_state.into(),
-            new_state: new_state.into(),
-        };
+        let response =
+            params::SledProvisionPolicyResponse { old_state, new_state };
 
         Ok(HttpResponseOk(response))
     };
diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs
index cd04bb6018..81f2a02b31 100644
--- a/nexus/tests/integration_tests/endpoints.rs
+++ b/nexus/tests/integration_tests/endpoints.rs
@@ -22,6 +22,7 @@ use nexus_types::external_api::params;
 use nexus_types::external_api::shared;
 use nexus_types::external_api::shared::IpRange;
 use nexus_types::external_api::shared::Ipv4Range;
+use nexus_types::external_api::views::SledProvisionPolicy;
 use omicron_common::api::external::AddressLotKind;
 use omicron_common::api::external::ByteCount;
 use omicron_common::api::external::IdentityMetadataCreateParams;
@@ -45,14 +46,12 @@ pub const HARDWARE_UNINITIALIZED_SLEDS: &'static str =
     "/v1/system/hardware/sleds-uninitialized";
 pub static HARDWARE_SLED_URL: Lazy<String> =
     Lazy::new(|| format!("/v1/system/hardware/sleds/{}", SLED_AGENT_UUID));
-pub static HARDWARE_SLED_PROVISION_STATE_URL: Lazy<String> = Lazy::new(|| {
-    format!("/v1/system/hardware/sleds/{}/provision-state", SLED_AGENT_UUID)
+pub static HARDWARE_SLED_PROVISION_POLICY_URL: Lazy<String> = Lazy::new(|| {
+    format!("/v1/system/hardware/sleds/{}/provision-policy", SLED_AGENT_UUID)
 });
-pub static DEMO_SLED_PROVISION_STATE: Lazy<params::SledProvisionStateParams> =
-    Lazy::new(|| {
-        params::SledProvisionStateParams {
-            state: nexus_types::external_api::views::SledProvisionState::NonProvisionable,
-        }
+pub static DEMO_SLED_PROVISION_POLICY: Lazy<params::SledProvisionPolicyParams> =
+    Lazy::new(|| params::SledProvisionPolicyParams {
+        state: SledProvisionPolicy::NonProvisionable,
     });
 
 pub static HARDWARE_SWITCH_URL: Lazy<String> =
@@ -1911,11 +1910,11 @@ pub static VERIFY_ENDPOINTS: Lazy<Vec<VerifyEndpoint>> = Lazy::new(|| {
         },
 
         VerifyEndpoint {
-            url: &HARDWARE_SLED_PROVISION_STATE_URL,
+            url: &HARDWARE_SLED_PROVISION_POLICY_URL,
             visibility: Visibility::Protected,
             unprivileged_access: UnprivilegedAccess::None,
             allowed_methods: vec![AllowedMethod::Put(
-                serde_json::to_value(&*DEMO_SLED_PROVISION_STATE).unwrap()
+                serde_json::to_value(&*DEMO_SLED_PROVISION_POLICY).unwrap()
             )],
         },
 
diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs
index 1c23f1b842..380ec1c975 100644
--- a/nexus/tests/integration_tests/schema.rs
+++ b/nexus/tests/integration_tests/schema.rs
@@ -1048,7 +1048,8 @@ fn after_23_0_0(client: &Client) -> BoxFuture<'_, ()> {
 fn before_24_0_0(client: &Client) -> BoxFuture<'_, ()> {
     // IP addresses were pulled off dogfood sled 16
     Box::pin(async move {
-        // Create two sleds
+        // Create two sleds. (SLED2 is marked non_provisionable for
+        // after_37_0_1.)
         client
             .batch_execute(&format!(
                 "INSERT INTO sled
@@ -1062,7 +1063,7 @@ fn before_24_0_0(client: &Client) -> BoxFuture<'_, ()> {
             'fd00:1122:3344:104::1ac', 'provisionable'),
           ('{SLED2}', now(), now(), NULL, 1, '{RACK1}', false, 'zzzz', 'xxxx',
              '2', 64, 12345678, 77,'fd00:1122:3344:107::1', 12345,
-            'fd00:1122:3344:107::d4', 'provisionable');
+            'fd00:1122:3344:107::d4', 'non_provisionable');
         "
             ))
             .await
@@ -1095,6 +1096,45 @@ fn after_24_0_0(client: &Client) -> BoxFuture<'_, ()> {
     })
 }
 
+// This reuses the sleds created in before_24_0_0.
+fn after_37_0_1(client: &Client) -> BoxFuture<'_, ()> {
+    Box::pin(async {
+        // Confirm that the IP Addresses have the last 2 bytes changed to `0xFFFF`
+        let rows = client
+            .query("SELECT sled_policy, sled_state FROM sled ORDER BY id", &[])
+            .await
+            .expect("Failed to select sled policy and state");
+        let policy_and_state = process_rows(&rows);
+
+        assert_eq!(
+            policy_and_state[0].values,
+            vec![
+                ColumnValue::new(
+                    "sled_policy",
+                    SqlEnum::from(("sled_policy", "in_service"))
+                ),
+                ColumnValue::new(
+                    "sled_state",
+                    SqlEnum::from(("sled_state", "active"))
+                ),
+            ]
+        );
+        assert_eq!(
+            policy_and_state[1].values,
+            vec![
+                ColumnValue::new(
+                    "sled_policy",
+                    SqlEnum::from(("sled_policy", "no_provision"))
+                ),
+                ColumnValue::new(
+                    "sled_state",
+                    SqlEnum::from(("sled_state", "active"))
+                ),
+            ]
+        );
+    })
+}
+
 // Lazily initializes all migration checks. The combination of Rust function
 // pointers and async makes defining a static table fairly painful, so we're
 // using lazy initialization instead.
@@ -1112,6 +1152,10 @@ fn get_migration_checks() -> BTreeMap<SemverVersion, DataMigrationFns> {
         SemverVersion(semver::Version::parse("24.0.0").unwrap()),
         DataMigrationFns { before: Some(before_24_0_0), after: after_24_0_0 },
     );
+    map.insert(
+        SemverVersion(semver::Version::parse("37.0.1").unwrap()),
+        DataMigrationFns { before: None, after: after_37_0_1 },
+    );
 
     map
 }
diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt
index 7ed73fd30a..adb36a24af 100644
--- a/nexus/tests/output/nexus_tags.txt
+++ b/nexus/tests/output/nexus_tags.txt
@@ -133,7 +133,7 @@ sled_instance_list                       GET      /v1/system/hardware/sleds/{sle
 sled_list                                GET      /v1/system/hardware/sleds
 sled_list_uninitialized                  GET      /v1/system/hardware/sleds-uninitialized
 sled_physical_disk_list                  GET      /v1/system/hardware/sleds/{sled_id}/disks
-sled_set_provision_state                 PUT      /v1/system/hardware/sleds/{sled_id}/provision-state
+sled_set_provision_policy                PUT      /v1/system/hardware/sleds/{sled_id}/provision-policy
 sled_view                                GET      /v1/system/hardware/sleds/{sled_id}
 switch_list                              GET      /v1/system/hardware/switches
 switch_view                              GET      /v1/system/hardware/switches/{switch_id}
diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs
index 2e683878be..0f601f3db5 100644
--- a/nexus/types/src/deployment.rs
+++ b/nexus/types/src/deployment.rs
@@ -11,7 +11,8 @@
 //! nexus/deployment does not currently know about nexus/db-model and it's
 //! convenient to separate these concerns.)
 
-use crate::external_api::views::SledProvisionState;
+use crate::external_api::views::SledPolicy;
+use crate::external_api::views::SledState;
 use crate::inventory::Collection;
 pub use crate::inventory::NetworkInterface;
 pub use crate::inventory::NetworkInterfaceKind;
@@ -64,8 +65,11 @@ pub struct Policy {
 /// Describes the resources available on each sled for the planner
 #[derive(Debug, Clone)]
 pub struct SledResources {
-    /// provision state of this sled
-    pub provision_state: SledProvisionState,
+    /// current sled policy
+    pub policy: SledPolicy,
+
+    /// current sled state
+    pub state: SledState,
 
     /// zpools on this sled
     ///
@@ -80,6 +84,17 @@ pub struct SledResources {
     pub subnet: Ipv6Subnet<SLED_PREFIX>,
 }
 
+impl SledResources {
+    /// Returns true if the sled can have services provisioned on it that
+    /// aren't required to be on every sled.
+    ///
+    /// For example, NTP must exist on every sled, but Nexus does not have to.
+    pub fn is_eligible_for_discretionary_services(&self) -> bool {
+        self.policy.is_provisionable()
+            && self.state.is_eligible_for_discretionary_services()
+    }
+}
+
 /// Describes a complete set of software and configuration for the system
 // Blueprints are a fundamental part of how the system modifies itself.  Each
 // blueprint completely describes all of the software and configuration
@@ -266,6 +281,7 @@ pub struct OmicronZonesDiff<'a> {
 }
 
 /// Describes a sled that appeared on both sides of a diff (possibly changed)
+#[derive(Debug)]
 pub struct DiffSledCommon<'a> {
     /// id of the sled
     pub sled_id: Uuid,
diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs
index 6cb878084d..07eeb9b679 100644
--- a/nexus/types/src/external_api/params.rs
+++ b/nexus/types/src/external_api/params.rs
@@ -96,21 +96,21 @@ pub struct SledSelector {
     pub sled: Uuid,
 }
 
-/// Parameters for `sled_set_provision_state`.
+/// Parameters for `sled_set_provision_policy`.
 #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-pub struct SledProvisionStateParams {
+pub struct SledProvisionPolicyParams {
     /// The provision state.
-    pub state: super::views::SledProvisionState,
+    pub state: super::views::SledProvisionPolicy,
 }
 
-/// Response to `sled_set_provision_state`.
+/// Response to `sled_set_provision_policy`.
 #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
-pub struct SledProvisionStateResponse {
+pub struct SledProvisionPolicyResponse {
     /// The old provision state.
-    pub old_state: super::views::SledProvisionState,
+    pub old_state: super::views::SledProvisionPolicy,
 
     /// The new provision state.
-    pub new_state: super::views::SledProvisionState,
+    pub new_state: super::views::SledProvisionPolicy,
 }
 
 pub struct SwitchSelector {
diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs
index 84648f109f..a3e87b162e 100644
--- a/nexus/types/src/external_api/views.rs
+++ b/nexus/types/src/external_api/views.rs
@@ -19,7 +19,9 @@ use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
+use std::fmt;
 use std::net::IpAddr;
+use strum::{EnumIter, IntoEnumIterator};
 use uuid::Uuid;
 
 use super::params::PhysicalDiskKind;
@@ -418,32 +420,214 @@ pub struct Sled {
     pub baseboard: Baseboard,
     /// The rack to which this Sled is currently attached
     pub rack_id: Uuid,
-    /// The provision state of the sled.
-    pub provision_state: SledProvisionState,
+    /// The operator-defined policy of a sled.
+    pub policy: SledPolicy,
+    /// The current state Nexus believes the sled to be in.
+    pub state: SledState,
     /// The number of hardware threads which can execute on this sled
     pub usable_hardware_threads: u32,
     /// Amount of RAM which may be used by the Sled's OS
     pub usable_physical_ram: ByteCount,
 }
 
-/// The provision state of a sled.
+/// The operator-defined provision policy of a sled.
 ///
 /// This controls whether new resources are going to be provisioned on this
 /// sled.
 #[derive(
-    Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq,
+    Copy,
+    Clone,
+    Debug,
+    Deserialize,
+    Serialize,
+    JsonSchema,
+    PartialEq,
+    Eq,
+    EnumIter,
 )]
 #[serde(rename_all = "snake_case")]
-pub enum SledProvisionState {
+pub enum SledProvisionPolicy {
     /// New resources will be provisioned on this sled.
     Provisionable,
 
-    /// New resources will not be provisioned on this sled. However, existing
-    /// resources will continue to be on this sled unless manually migrated
-    /// off.
+    /// New resources will not be provisioned on this sled. However, if the
+    /// sled is currently in service, existing resources will continue to be on
+    /// this sled unless manually migrated off.
     NonProvisionable,
 }
 
+impl SledProvisionPolicy {
+    /// Returns the opposite of the current provision state.
+    pub const fn invert(self) -> Self {
+        match self {
+            Self::Provisionable => Self::NonProvisionable,
+            Self::NonProvisionable => Self::Provisionable,
+        }
+    }
+}
+
+/// The operator-defined policy of a sled.
+#[derive(
+    Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq,
+)]
+#[serde(rename_all = "snake_case", tag = "kind")]
+pub enum SledPolicy {
+    /// The operator has indicated that the sled is in-service.
+    InService {
+        /// Determines whether new resources can be provisioned onto the sled.
+        provision_policy: SledProvisionPolicy,
+    },
+
+    /// The operator has indicated that the sled has been permanently removed
+    /// from service.
+    ///
+    /// This is a terminal state: once a particular sled ID is expunged, it
+    /// will never return to service. (The actual hardware may be reused, but
+    /// it will be treated as a brand-new sled.)
+    ///
+    /// An expunged sled is always non-provisionable.
+    Expunged,
+    // NOTE: if you add a new value here, be sure to add it to
+    // the `IntoEnumIterator` impl below!
+}
+
+// Can't automatically derive strum::EnumIter because that doesn't provide a
+// way to iterate over nested enums.
+impl IntoEnumIterator for SledPolicy {
+    type Iterator = std::array::IntoIter<Self, 3>;
+
+    fn iter() -> Self::Iterator {
+        [
+            Self::InService {
+                provision_policy: SledProvisionPolicy::Provisionable,
+            },
+            Self::InService {
+                provision_policy: SledProvisionPolicy::NonProvisionable,
+            },
+            Self::Expunged,
+        ]
+        .into_iter()
+    }
+}
+
+impl SledPolicy {
+    /// Creates a new `SledPolicy` that is in-service and provisionable.
+    pub fn provisionable() -> Self {
+        Self::InService { provision_policy: SledProvisionPolicy::Provisionable }
+    }
+
+    /// Returns the list of all in-service policies.
+    pub fn all_in_service() -> &'static [Self] {
+        &[
+            Self::InService {
+                provision_policy: SledProvisionPolicy::Provisionable,
+            },
+            Self::InService {
+                provision_policy: SledProvisionPolicy::NonProvisionable,
+            },
+        ]
+    }
+
+    /// Returns true if the sled can have services provisioned on it.
+    pub fn is_provisionable(&self) -> bool {
+        match self {
+            Self::InService {
+                provision_policy: SledProvisionPolicy::Provisionable,
+            } => true,
+            Self::InService {
+                provision_policy: SledProvisionPolicy::NonProvisionable,
+            }
+            | Self::Expunged => false,
+        }
+    }
+
+    /// Returns the provision policy, if the sled is in service.
+    pub fn provision_policy(&self) -> Option<SledProvisionPolicy> {
+        match self {
+            Self::InService { provision_policy } => Some(*provision_policy),
+            Self::Expunged => None,
+        }
+    }
+
+    /// Returns true if the sled can be decommissioned in this state.
+    pub fn is_decommissionable(&self) -> bool {
+        // This should be kept in sync with decommissionable_states below.
+        match self {
+            Self::InService { .. } => false,
+            Self::Expunged => true,
+        }
+    }
+
+    /// Returns all the possible policies a sled can have for it to be
+    /// decommissioned.
+    pub fn all_decommissionable() -> &'static [Self] {
+        &[Self::Expunged]
+    }
+}
+
+impl fmt::Display for SledPolicy {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            SledPolicy::InService {
+                provision_policy: SledProvisionPolicy::Provisionable,
+            } => write!(f, "in service"),
+            SledPolicy::InService {
+                provision_policy: SledProvisionPolicy::NonProvisionable,
+            } => write!(f, "in service (not provisionable)"),
+            SledPolicy::Expunged => write!(f, "expunged"),
+        }
+    }
+}
+
+/// The current state of the sled, as determined by Nexus.
+#[derive(
+    Copy,
+    Clone,
+    Debug,
+    Deserialize,
+    Serialize,
+    JsonSchema,
+    PartialEq,
+    Eq,
+    EnumIter,
+)]
+#[serde(rename_all = "snake_case")]
+pub enum SledState {
+    /// The sled is currently active, and has resources allocated on it.
+    Active,
+
+    /// The sled has been permanently removed from service.
+    ///
+    /// This is a terminal state: once a particular sled ID is decommissioned,
+    /// it will never return to service. (The actual hardware may be reused,
+    /// but it will be treated as a brand-new sled.)
+    Decommissioned,
+}
+
+impl SledState {
+    /// Returns true if the sled state makes it eligible for services that
+    /// aren't required to be on every sled.
+    ///
+    /// For example, NTP must exist on every sled, but Nexus does not have to.
+    pub fn is_eligible_for_discretionary_services(&self) -> bool {
+        // (Explicit match, so that this fails to compile if a new state is
+        // added.)
+        match self {
+            SledState::Active => true,
+            SledState::Decommissioned => false,
+        }
+    }
+}
+
+impl fmt::Display for SledState {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            SledState::Active => write!(f, "active"),
+            SledState::Decommissioned => write!(f, "decommissioned"),
+        }
+    }
+}
+
 /// An operator's view of an instance running on a given sled
 #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
 pub struct SledInstance {
diff --git a/openapi/nexus.json b/openapi/nexus.json
index f42841dcf6..cc8edec51d 100644
--- a/openapi/nexus.json
+++ b/openapi/nexus.json
@@ -4221,13 +4221,13 @@
         }
       }
     },
-    "/v1/system/hardware/sleds/{sled_id}/provision-state": {
+    "/v1/system/hardware/sleds/{sled_id}/provision-policy": {
       "put": {
         "tags": [
           "system/hardware"
         ],
-        "summary": "Set sled provision state",
-        "operationId": "sled_set_provision_state",
+        "summary": "Set sled provision policy",
+        "operationId": "sled_set_provision_policy",
         "parameters": [
           {
             "in": "path",
@@ -4244,7 +4244,7 @@
           "content": {
             "application/json": {
               "schema": {
-                "$ref": "#/components/schemas/SledProvisionStateParams"
+                "$ref": "#/components/schemas/SledProvisionPolicyParams"
               }
             }
           },
@@ -4256,7 +4256,7 @@
             "content": {
               "application/json": {
                 "schema": {
-                  "$ref": "#/components/schemas/SledProvisionStateResponse"
+                  "$ref": "#/components/schemas/SledProvisionPolicyResponse"
                 }
               }
             }
@@ -14839,11 +14839,11 @@
             "type": "string",
             "format": "uuid"
           },
-          "provision_state": {
-            "description": "The provision state of the sled.",
+          "policy": {
+            "description": "The operator-defined policy of a sled.",
             "allOf": [
               {
-                "$ref": "#/components/schemas/SledProvisionState"
+                "$ref": "#/components/schemas/SledPolicy"
               }
             ]
           },
@@ -14852,6 +14852,14 @@
             "type": "string",
             "format": "uuid"
           },
+          "state": {
+            "description": "The current state Nexus believes the sled to be in.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/SledState"
+              }
+            ]
+          },
           "time_created": {
             "description": "timestamp when this resource was created",
             "type": "string",
@@ -14880,8 +14888,9 @@
         "required": [
           "baseboard",
           "id",
-          "provision_state",
+          "policy",
           "rack_id",
+          "state",
           "time_created",
           "time_modified",
           "usable_hardware_threads",
@@ -14971,8 +14980,52 @@
           "items"
         ]
       },
-      "SledProvisionState": {
-        "description": "The provision state of a sled.\n\nThis controls whether new resources are going to be provisioned on this sled.",
+      "SledPolicy": {
+        "description": "The operator-defined policy of a sled.",
+        "oneOf": [
+          {
+            "description": "The operator has indicated that the sled is in-service.",
+            "type": "object",
+            "properties": {
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "in_service"
+                ]
+              },
+              "provision_policy": {
+                "description": "Determines whether new resources can be provisioned onto the sled.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/SledProvisionPolicy"
+                  }
+                ]
+              }
+            },
+            "required": [
+              "kind",
+              "provision_policy"
+            ]
+          },
+          {
+            "description": "The operator has indicated that the sled has been permanently removed from service.\n\nThis is a terminal state: once a particular sled ID is expunged, it will never return to service. (The actual hardware may be reused, but it will be treated as a brand-new sled.)\n\nAn expunged sled is always non-provisionable.",
+            "type": "object",
+            "properties": {
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "expunged"
+                ]
+              }
+            },
+            "required": [
+              "kind"
+            ]
+          }
+        ]
+      },
+      "SledProvisionPolicy": {
+        "description": "The operator-defined provision policy of a sled.\n\nThis controls whether new resources are going to be provisioned on this sled.",
         "oneOf": [
           {
             "description": "New resources will be provisioned on this sled.",
@@ -14982,7 +15035,7 @@
             ]
           },
           {
-            "description": "New resources will not be provisioned on this sled. However, existing resources will continue to be on this sled unless manually migrated off.",
+            "description": "New resources will not be provisioned on this sled. However, if the sled is currently in service, existing resources will continue to be on this sled unless manually migrated off.",
             "type": "string",
             "enum": [
               "non_provisionable"
@@ -14990,15 +15043,15 @@
           }
         ]
       },
-      "SledProvisionStateParams": {
-        "description": "Parameters for `sled_set_provision_state`.",
+      "SledProvisionPolicyParams": {
+        "description": "Parameters for `sled_set_provision_policy`.",
         "type": "object",
         "properties": {
           "state": {
             "description": "The provision state.",
             "allOf": [
               {
-                "$ref": "#/components/schemas/SledProvisionState"
+                "$ref": "#/components/schemas/SledProvisionPolicy"
               }
             ]
           }
@@ -15007,15 +15060,15 @@
           "state"
         ]
       },
-      "SledProvisionStateResponse": {
-        "description": "Response to `sled_set_provision_state`.",
+      "SledProvisionPolicyResponse": {
+        "description": "Response to `sled_set_provision_policy`.",
         "type": "object",
         "properties": {
           "new_state": {
             "description": "The new provision state.",
             "allOf": [
               {
-                "$ref": "#/components/schemas/SledProvisionState"
+                "$ref": "#/components/schemas/SledProvisionPolicy"
               }
             ]
           },
@@ -15023,7 +15076,7 @@
             "description": "The old provision state.",
             "allOf": [
               {
-                "$ref": "#/components/schemas/SledProvisionState"
+                "$ref": "#/components/schemas/SledProvisionPolicy"
               }
             ]
           }
@@ -15054,6 +15107,25 @@
           "items"
         ]
       },
+      "SledState": {
+        "description": "The current state of the sled, as determined by Nexus.",
+        "oneOf": [
+          {
+            "description": "The sled is currently active, and has resources allocated on it.",
+            "type": "string",
+            "enum": [
+              "active"
+            ]
+          },
+          {
+            "description": "The sled has been permanently removed from service.\n\nThis is a terminal state: once a particular sled ID is decommissioned, it will never return to service. (The actual hardware may be reused, but it will be treated as a brand-new sled.)",
+            "type": "string",
+            "enum": [
+              "decommissioned"
+            ]
+          }
+        ]
+      },
       "Snapshot": {
         "description": "View of a Snapshot",
         "type": "object",
diff --git a/schema/crdb/37.0.0/up01.sql b/schema/crdb/37.0.0/up01.sql
new file mode 100644
index 0000000000..bb66ff283e
--- /dev/null
+++ b/schema/crdb/37.0.0/up01.sql
@@ -0,0 +1,13 @@
+-- The disposition for a particular sled. This is updated solely by the
+-- operator, and not by Nexus.
+CREATE TYPE IF NOT EXISTS omicron.public.sled_policy AS ENUM (
+    -- The sled is in service, and new resources can be provisioned onto it.
+    'in_service',
+    -- The sled is in service, but the operator has indicated that new
+    -- resources should not be provisioned onto it.
+    'no_provision',
+    -- The operator has marked that the sled has, or will be, removed from the
+    -- rack, and it should be assumed that any resources currently on it are
+    -- now permanently missing.
+    'expunged'
+);
diff --git a/schema/crdb/37.0.0/up02.sql b/schema/crdb/37.0.0/up02.sql
new file mode 100644
index 0000000000..01cba8f7fe
--- /dev/null
+++ b/schema/crdb/37.0.0/up02.sql
@@ -0,0 +1,22 @@
+-- The actual state of the sled. This is updated exclusively by Nexus.
+--
+-- Nexus's goal is to match the sled's state with the operator-indicated
+-- policy. For example, if the sled_policy is "expunged" and the sled_state is
+-- "active", Nexus will assume that the sled is gone. Based on that, Nexus will
+-- reallocate resources currently on the expunged sled to other sleds, etc.
+-- Once the expunged sled no longer has any resources attached to it, Nexus
+-- will mark it as decommissioned.
+CREATE TYPE IF NOT EXISTS omicron.public.sled_state AS ENUM (
+    -- The sled has resources of any kind allocated on it, or, is available for
+    -- new resources.
+    --
+    -- The sled can be in this state and have a different sled policy, e.g.
+    -- "expunged".
+    'active',
+
+    -- The sled no longer has resources allocated on it, now or in the future.
+    --
+    -- This is a terminal state. This state is only valid if the sled policy is
+    -- 'expunged'.
+    'decommissioned'
+);
diff --git a/schema/crdb/37.0.0/up03.sql b/schema/crdb/37.0.0/up03.sql
new file mode 100644
index 0000000000..7e51cf9546
--- /dev/null
+++ b/schema/crdb/37.0.0/up03.sql
@@ -0,0 +1,7 @@
+-- Modify the existing sled table to add the columns as required.
+ALTER TABLE omicron.public.sled
+    -- Nullable for now -- we're going to set the data in sled_policy in the
+    -- next migration statement.
+    ADD COLUMN IF NOT EXISTS sled_policy omicron.public.sled_policy,
+    ADD COLUMN IF NOT EXISTS sled_state omicron.public.sled_state
+        NOT NULL DEFAULT 'active';
diff --git a/schema/crdb/37.0.0/up04.sql b/schema/crdb/37.0.0/up04.sql
new file mode 100644
index 0000000000..a85367ec10
--- /dev/null
+++ b/schema/crdb/37.0.0/up04.sql
@@ -0,0 +1,14 @@
+-- Mass-update the sled_policy column to match the sled_provision_state column.
+
+-- This is a full table scan, but is unavoidable.
+SET
+  LOCAL disallow_full_table_scans = OFF;
+
+UPDATE omicron.public.sled
+    SET sled_policy = 
+        (CASE provision_state
+            WHEN 'provisionable' THEN 'in_service'
+            WHEN 'non_provisionable' THEN 'no_provision'
+            -- No need to specify the ELSE case because the enum has been
+            -- exhaustively matched (sled_provision_state already bans NULL).
+        END);
diff --git a/schema/crdb/37.0.1/up01.sql b/schema/crdb/37.0.1/up01.sql
new file mode 100644
index 0000000000..c23e3e5a11
--- /dev/null
+++ b/schema/crdb/37.0.1/up01.sql
@@ -0,0 +1,8 @@
+-- This is a follow-up to the previous migration, done separately to ensure
+-- that the updated values for sled_policy are committed before the
+-- provision_state column is dropped.
+
+ALTER TABLE omicron.public.sled
+    DROP COLUMN IF EXISTS provision_state,
+    ALTER COLUMN sled_policy SET NOT NULL,
+    ALTER COLUMN sled_state DROP DEFAULT;
diff --git a/schema/crdb/37.0.1/up02.sql b/schema/crdb/37.0.1/up02.sql
new file mode 100644
index 0000000000..342f794c82
--- /dev/null
+++ b/schema/crdb/37.0.1/up02.sql
@@ -0,0 +1 @@
+DROP TYPE IF EXISTS omicron.public.sled_provision_state;
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 87a22d1adc..837be42c35 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -73,11 +73,41 @@ CREATE TABLE IF NOT EXISTS omicron.public.rack (
  * Sleds
  */
 
-CREATE TYPE IF NOT EXISTS omicron.public.sled_provision_state AS ENUM (
-    -- New resources can be provisioned onto the sled
-    'provisionable',
-    -- New resources must not be provisioned onto the sled
-    'non_provisionable'
+-- The disposition for a particular sled. This is updated solely by the
+-- operator, and not by Nexus.
+CREATE TYPE IF NOT EXISTS omicron.public.sled_policy AS ENUM (
+    -- The sled is in service, and new resources can be provisioned onto it.
+    'in_service',
+    -- The sled is in service, but the operator has indicated that new
+    -- resources should not be provisioned onto it.
+    'no_provision',
+    -- The operator has marked that the sled has, or will be, removed from the
+    -- rack, and it should be assumed that any resources currently on it are
+    -- now permanently missing.
+    'expunged'
+);
+
+-- The actual state of the sled. This is updated exclusively by Nexus.
+--
+-- Nexus's goal is to match the sled's state with the operator-indicated
+-- policy. For example, if the sled_policy is "expunged" and the sled_state is
+-- "active", Nexus will assume that the sled is gone. Based on that, Nexus will
+-- reallocate resources currently on the expunged sled to other sleds, etc.
+-- Once the expunged sled no longer has any resources attached to it, Nexus
+-- will mark it as decommissioned.
+CREATE TYPE IF NOT EXISTS omicron.public.sled_state AS ENUM (
+    -- The sled has resources of any kind allocated on it, or, is available for
+    -- new resources.
+    --
+    -- The sled can be in this state and have a different sled policy, e.g.
+    -- "expunged".
+    'active',
+
+    -- The sled no longer has resources allocated on it, now or in the future.
+    --
+    -- This is a terminal state. This state is only valid if the sled policy is
+    -- 'expunged'.
+    'decommissioned'
 );
 
 CREATE TABLE IF NOT EXISTS omicron.public.sled (
@@ -111,8 +141,11 @@ CREATE TABLE IF NOT EXISTS omicron.public.sled (
     /* The last address allocated to a propolis instance on this sled. */
     last_used_address INET NOT NULL,
 
-    /* The state of whether resources should be provisioned onto the sled */
-    provision_state omicron.public.sled_provision_state NOT NULL,
+    /* The policy for the sled, updated exclusively by the operator */
+    sled_policy omicron.public.sled_policy NOT NULL,
+
+    /* The actual state of the sled, updated exclusively by Nexus */
+    sled_state omicron.public.sled_state NOT NULL,
 
     -- This constraint should be upheld, even for deleted disks
     -- in the fleet.
@@ -3518,7 +3551,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '36.0.0', NULL)
+    ( TRUE, NOW(), NOW(), '37.0.1', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;

From c2ff7be9ce54fc4af61176c94ddc5f357ae33626 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Sat, 24 Feb 2024 05:08:24 +0000
Subject: [PATCH 039/157] chore(deps): update taiki-e/install-action digest to
 ffdab02 (#5137)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [taiki-e/install-action](https://togithub.com/taiki-e/install-action)
| action | digest | [`19e9b54` ->
`ffdab02`](https://togithub.com/taiki-e/install-action/compare/19e9b54...ffdab02)
|

---

### Configuration

📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone
America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone
America/Los_Angeles.

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update
again.

---

- [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check
this box

---

This PR has been generated by [Renovate
Bot](https://togithub.com/renovatebot/renovate).

<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNy4yMTIuMCIsInVwZGF0ZWRJblZlciI6IjM3LjIxMi4wIiwidGFyZ2V0QnJhbmNoIjoibWFpbiJ9-->

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index 1f55f2f255..a55c05ba7a 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@19e9b549a48620cc50fcf6e6e866b8fb4eca1b01 # v2
+        uses: taiki-e/install-action@ffdab026038b43b56c3c9540cdadb98181c6155c # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From b6d39a0aab1c950f40e26ec7f5e021aed9c22448 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Sun, 25 Feb 2024 05:10:08 +0000
Subject: [PATCH 040/157] chore(deps): update taiki-e/install-action digest to
 b7add58 (#5139)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [taiki-e/install-action](https://togithub.com/taiki-e/install-action)
| action | digest | [`ffdab02` ->
`b7add58`](https://togithub.com/taiki-e/install-action/compare/ffdab02...b7add58)
|

---

### Configuration

📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone
America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone
America/Los_Angeles.

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update
again.

---

- [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check
this box

---

This PR has been generated by [Renovate
Bot](https://togithub.com/renovatebot/renovate).

<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNy4yMTMuMCIsInVwZGF0ZWRJblZlciI6IjM3LjIxMy4wIiwidGFyZ2V0QnJhbmNoIjoibWFpbiJ9-->

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index a55c05ba7a..2463ac4e2e 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@ffdab026038b43b56c3c9540cdadb98181c6155c # v2
+        uses: taiki-e/install-action@b7add58e53e52e624966da65007ce24524f3dcf3 # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From 767c7fe2650040d7b8097a995bb3e9bde00a7512 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Mon, 26 Feb 2024 14:25:48 +0000
Subject: [PATCH 041/157] chore(deps): update taiki-e/install-action digest to
 4ce8785 (#5142)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [taiki-e/install-action](https://togithub.com/taiki-e/install-action)
| action | digest | [`b7add58` ->
`4ce8785`](https://togithub.com/taiki-e/install-action/compare/b7add58...4ce8785)
|

---

### Configuration

📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone
America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone
America/Los_Angeles.

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update
again.

---

- [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check
this box

---

This PR has been generated by [Renovate
Bot](https://togithub.com/renovatebot/renovate).

<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNy4yMTQuMCIsInVwZGF0ZWRJblZlciI6IjM3LjIxNC4wIiwidGFyZ2V0QnJhbmNoIjoibWFpbiJ9-->

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index 2463ac4e2e..10c1c04003 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@b7add58e53e52e624966da65007ce24524f3dcf3 # v2
+        uses: taiki-e/install-action@4ce8785db2a8a56c9ede16f705c2c49c5c61669c # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From 7fae994cbe954d576731638d800e2c0caf17a37e Mon Sep 17 00:00:00 2001
From: David Crespo <david-crespo@users.noreply.github.com>
Date: Mon, 26 Feb 2024 11:24:20 -0600
Subject: [PATCH 042/157] Bump web console (floating IPs) (#5126)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### User-facing changes

* [80f11673](https://github.com/oxidecomputer/console/commit/80f11673)
oxidecomputer/console#1957
* [f31d5331](https://github.com/oxidecomputer/console/commit/f31d5331)
oxidecomputer/console#1944
* [b454cefd](https://github.com/oxidecomputer/console/commit/b454cefd)
oxidecomputer/console#1955
* [1936e0d8](https://github.com/oxidecomputer/console/commit/1936e0d8)
oxidecomputer/console#1948

---

### All changes

https://github.com/oxidecomputer/console/compare/e5a1f804...80f11673

* [80f11673](https://github.com/oxidecomputer/console/commit/80f11673)
oxidecomputer/console#1957
* [5d989a70](https://github.com/oxidecomputer/console/commit/5d989a70)
oxidecomputer/console#1959
* [d4ec1927](https://github.com/oxidecomputer/console/commit/d4ec1927)
turn off license-eye comments, I hate them. CI failure is sufficient
* [f8d2f36e](https://github.com/oxidecomputer/console/commit/f8d2f36e)
oxidecomputer/console#1960
* [e0b676dc](https://github.com/oxidecomputer/console/commit/e0b676dc)
upgrade husky commands for v9
https://github.com/typicode/husky/releases/tag/v9.0.1
* [f31d5331](https://github.com/oxidecomputer/console/commit/f31d5331)
oxidecomputer/console#1944
* [b4552eea](https://github.com/oxidecomputer/console/commit/b4552eea)
Revert "Revert all app changes since v6 except two small fixes
(oxidecomputer/console#1958)"
* [1f8ebf28](https://github.com/oxidecomputer/console/commit/1f8ebf28)
oxidecomputer/console#1958
* [b454cefd](https://github.com/oxidecomputer/console/commit/b454cefd)
oxidecomputer/console#1955
* [1ce32d32](https://github.com/oxidecomputer/console/commit/1ce32d32)
oxidecomputer/console#1956
* [794ae11d](https://github.com/oxidecomputer/console/commit/794ae11d)
oxidecomputer/console#1952
* [903b8f6a](https://github.com/oxidecomputer/console/commit/903b8f6a)
tweak api-diff: fix type error on first arg, use dax's new built-in pipe
* [a4e15cdd](https://github.com/oxidecomputer/console/commit/a4e15cdd)
oxidecomputer/console#1950
* [32037d40](https://github.com/oxidecomputer/console/commit/32037d40)
oxidecomputer/console#1949
* [f02678b0](https://github.com/oxidecomputer/console/commit/f02678b0)
vite 5.1
* [1936e0d8](https://github.com/oxidecomputer/console/commit/1936e0d8)
oxidecomputer/console#1948
* [cfda1636](https://github.com/oxidecomputer/console/commit/cfda1636)
forgot to bump mockServiceWorker.js (no actual changes)
* [4792105e](https://github.com/oxidecomputer/console/commit/4792105e)
oxidecomputer/console#1943
* [a26db9ea](https://github.com/oxidecomputer/console/commit/a26db9ea)
upgrade date-fns thru major version, update calls accordingly
* [3dd635a6](https://github.com/oxidecomputer/console/commit/3dd635a6)
oxidecomputer/console#1933
* [6c8f7a9c](https://github.com/oxidecomputer/console/commit/6c8f7a9c)
upgrade filesize through a major version
* [8f641b97](https://github.com/oxidecomputer/console/commit/8f641b97)
remove blathering in readme about node 16, which is EOL
* [e9d157a1](https://github.com/oxidecomputer/console/commit/e9d157a1)
do ladle too—oh my god why does it have 3000 lines in the lockfile
* [e1cdcc13](https://github.com/oxidecomputer/console/commit/e1cdcc13)
oxidecomputer/console#1941
* [76877ffb](https://github.com/oxidecomputer/console/commit/76877ffb)
oxidecomputer/console#1938
---
 tools/console_version | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/console_version b/tools/console_version
index 7f80546553..1b2cc62547 100644
--- a/tools/console_version
+++ b/tools/console_version
@@ -1,2 +1,2 @@
-COMMIT="e5a1f804faa913de3be5b4cddac2011247a99774"
-SHA2="54ff1026062fc1a3f0de86aa558d051b8ad6248d458c1767b9e926f2606e75f5"
+COMMIT="80f116734fdf8ef4f3b5e49ed39e49339460407c"
+SHA2="458727fe747797860d02c9f75e62d308586c235d4708c549d5b70a40cce4d2d1"

From d7db26d9c6f09fec3fec6991e4a9cd7f90128465 Mon Sep 17 00:00:00 2001
From: David Pacheco <dap@oxidecomputer.com>
Date: Mon, 26 Feb 2024 11:54:31 -0800
Subject: [PATCH 043/157] trigger inventory collection after blueprint
 execution (#5130)

---
 .../src/app/background/blueprint_execution.rs | 11 +++-
 nexus/src/app/background/init.rs              | 56 +++++++++++--------
 2 files changed, 42 insertions(+), 25 deletions(-)

diff --git a/nexus/src/app/background/blueprint_execution.rs b/nexus/src/app/background/blueprint_execution.rs
index 373f023288..4ba60ab566 100644
--- a/nexus/src/app/background/blueprint_execution.rs
+++ b/nexus/src/app/background/blueprint_execution.rs
@@ -20,6 +20,7 @@ pub struct BlueprintExecutor {
     datastore: Arc<DataStore>,
     rx_blueprint: watch::Receiver<Option<Arc<(BlueprintTarget, Blueprint)>>>,
     nexus_label: String,
+    tx: watch::Sender<usize>,
 }
 
 impl BlueprintExecutor {
@@ -30,7 +31,12 @@ impl BlueprintExecutor {
         >,
         nexus_label: String,
     ) -> BlueprintExecutor {
-        BlueprintExecutor { datastore, rx_blueprint, nexus_label }
+        let (tx, _) = watch::channel(0);
+        BlueprintExecutor { datastore, rx_blueprint, nexus_label, tx }
+    }
+
+    pub fn watcher(&self) -> watch::Receiver<usize> {
+        self.tx.subscribe()
     }
 }
 
@@ -71,6 +77,9 @@ impl BackgroundTask for BlueprintExecutor {
             )
             .await;
 
+            // Trigger anybody waiting for this to finish.
+            self.tx.send_modify(|count| *count = *count + 1);
+
             // Return the result as a `serde_json::Value`
             match result {
                 Ok(()) => json!({}),
diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs
index 846051a068..9ba30bab64 100644
--- a/nexus/src/app/background/init.rs
+++ b/nexus/src/app/background/init.rs
@@ -170,30 +170,6 @@ impl BackgroundTasks {
             )
         };
 
-        // Background task: inventory collector
-        let task_inventory_collection = {
-            let collector = inventory_collection::InventoryCollector::new(
-                datastore.clone(),
-                resolver,
-                &nexus_id.to_string(),
-                config.inventory.nkeep,
-                config.inventory.disable,
-            );
-            let task = driver.register(
-                String::from("inventory_collection"),
-                String::from(
-                    "collects hardware and software inventory data from the \
-                    whole system",
-                ),
-                config.inventory.period_secs,
-                Box::new(collector),
-                opctx.child(BTreeMap::new()),
-                vec![],
-            );
-
-            task
-        };
-
         // Background task: phantom disk detection
         let task_phantom_disks = {
             let detector =
@@ -230,6 +206,7 @@ impl BackgroundTasks {
             rx_blueprint.clone(),
             nexus_id.to_string(),
         );
+        let rx_blueprint_exec = blueprint_executor.watcher();
         let task_blueprint_executor = driver.register(
             String::from("blueprint_executor"),
             String::from("Executes the target blueprint"),
@@ -239,6 +216,37 @@ impl BackgroundTasks {
             vec![Box::new(rx_blueprint)],
         );
 
+        // Background task: inventory collector
+        //
+        // This currently depends on the "output" of the blueprint executor in
+        // order to automatically trigger inventory collection whenever the
+        // blueprint executor runs.  In the limit, this could become a problem
+        // because the blueprint executor might also depend indirectly on the
+        // inventory collector.  In that case, we may need to do something more
+        // complicated.  But for now, this works.
+        let task_inventory_collection = {
+            let collector = inventory_collection::InventoryCollector::new(
+                datastore.clone(),
+                resolver,
+                &nexus_id.to_string(),
+                config.inventory.nkeep,
+                config.inventory.disable,
+            );
+            let task = driver.register(
+                String::from("inventory_collection"),
+                String::from(
+                    "collects hardware and software inventory data from the \
+                    whole system",
+                ),
+                config.inventory.period_secs,
+                Box::new(collector),
+                opctx.child(BTreeMap::new()),
+                vec![Box::new(rx_blueprint_exec)],
+            );
+
+            task
+        };
+
         let task_service_zone_nat_tracker = {
             driver.register(
                 "service_zone_nat_tracker".to_string(),

From d514878f417a94247791bd5564fbaafa9b4170a0 Mon Sep 17 00:00:00 2001
From: Charlie Park <charlie@oxidecomputer.com>
Date: Mon, 26 Feb 2024 14:32:16 -0800
Subject: [PATCH 044/157] Update Floating IP field name from address to ip
 (#5144)

Fixes #5046

Our API for Floating IPs currently has an `address` field, though in a
few other places we use `ip`. This PR just makes the API more consistent
across the API (and DB).
---
 nexus/db-queries/src/db/datastore/external_ip.rs | 2 +-
 nexus/test-utils/src/resource_helpers.rs         | 4 ++--
 nexus/tests/integration_tests/endpoints.rs       | 2 +-
 nexus/tests/integration_tests/external_ips.rs    | 8 ++++----
 nexus/types/src/external_api/params.rs           | 2 +-
 openapi/nexus.json                               | 8 ++++----
 6 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/nexus/db-queries/src/db/datastore/external_ip.rs b/nexus/db-queries/src/db/datastore/external_ip.rs
index 24439aa3a0..d15e1b7ca8 100644
--- a/nexus/db-queries/src/db/datastore/external_ip.rs
+++ b/nexus/db-queries/src/db/datastore/external_ip.rs
@@ -256,7 +256,7 @@ impl DataStore {
 
         let pool_id = pool.id();
 
-        let data = if let Some(ip) = params.address {
+        let data = if let Some(ip) = params.ip {
             IncompleteExternalIp::for_floating_explicit(
                 ip_id,
                 &Name(params.identity.name),
diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs
index 764332c5bc..25e58d093d 100644
--- a/nexus/test-utils/src/resource_helpers.rs
+++ b/nexus/test-utils/src/resource_helpers.rs
@@ -267,7 +267,7 @@ pub async fn create_floating_ip(
     client: &ClientTestContext,
     fip_name: &str,
     project: &str,
-    address: Option<IpAddr>,
+    ip: Option<IpAddr>,
     parent_pool_name: Option<&str>,
 ) -> FloatingIp {
     object_create(
@@ -278,7 +278,7 @@ pub async fn create_floating_ip(
                 name: fip_name.parse().unwrap(),
                 description: String::from("a floating ip"),
             },
-            address,
+            ip,
             pool: parent_pool_name.map(|v| NameOrId::Name(v.parse().unwrap())),
         },
     )
diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs
index 81f2a02b31..c5c69df232 100644
--- a/nexus/tests/integration_tests/endpoints.rs
+++ b/nexus/tests/integration_tests/endpoints.rs
@@ -768,7 +768,7 @@ pub static DEMO_FLOAT_IP_CREATE: Lazy<params::FloatingIpCreate> =
             name: DEMO_FLOAT_IP_NAME.clone(),
             description: String::from("a new IP pool"),
         },
-        address: Some(std::net::Ipv4Addr::new(10, 0, 0, 141).into()),
+        ip: Some(std::net::Ipv4Addr::new(10, 0, 0, 141).into()),
         pool: None,
     });
 
diff --git a/nexus/tests/integration_tests/external_ips.rs b/nexus/tests/integration_tests/external_ips.rs
index 57f813d505..ee59c6a034 100644
--- a/nexus/tests/integration_tests/external_ips.rs
+++ b/nexus/tests/integration_tests/external_ips.rs
@@ -189,7 +189,7 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) {
             name: fip_name.parse().unwrap(),
             description: String::from("a floating ip"),
         },
-        address: None,
+        ip: None,
         pool: Some(NameOrId::Name("other-pool".parse().unwrap())),
     };
     let url = format!("/v1/floating-ips?project={}", project.identity.name);
@@ -259,7 +259,7 @@ async fn test_floating_ip_create_fails_in_other_silo_pool(
             name: fip_name.parse().unwrap(),
             description: String::from("a floating ip"),
         },
-        address: None,
+        ip: None,
         pool: Some(NameOrId::Name("external-silo-pool".parse().unwrap())),
     };
 
@@ -316,7 +316,7 @@ async fn test_floating_ip_create_ip_in_use(
                 name: FIP_NAMES[1].parse().unwrap(),
                 description: "another fip".into(),
             },
-            address: Some(contested_ip),
+            ip: Some(contested_ip),
             pool: None,
         }))
         .expect_status(Some(StatusCode::BAD_REQUEST)),
@@ -364,7 +364,7 @@ async fn test_floating_ip_create_name_in_use(
                 name: contested_name.parse().unwrap(),
                 description: "another fip".into(),
             },
-            address: None,
+            ip: None,
             pool: None,
         }))
         .expect_status(Some(StatusCode::BAD_REQUEST)),
diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs
index 07eeb9b679..567b1ff4ad 100644
--- a/nexus/types/src/external_api/params.rs
+++ b/nexus/types/src/external_api/params.rs
@@ -883,7 +883,7 @@ pub struct FloatingIpCreate {
     /// An IP address to reserve for use as a floating IP. This field is
     /// optional: when not set, an address will be automatically chosen from
     /// `pool`. If set, then the IP must be available in the resolved `pool`.
-    pub address: Option<IpAddr>,
+    pub ip: Option<IpAddr>,
 
     /// The parent IP pool that a floating IP is pulled from. If unset, the
     /// default pool is selected.
diff --git a/openapi/nexus.json b/openapi/nexus.json
index cc8edec51d..b0aa84d67a 100644
--- a/openapi/nexus.json
+++ b/openapi/nexus.json
@@ -11864,15 +11864,15 @@
         "description": "Parameters for creating a new floating IP address for instances.",
         "type": "object",
         "properties": {
-          "address": {
+          "description": {
+            "type": "string"
+          },
+          "ip": {
             "nullable": true,
             "description": "An IP address to reserve for use as a floating IP. This field is optional: when not set, an address will be automatically chosen from `pool`. If set, then the IP must be available in the resolved `pool`.",
             "type": "string",
             "format": "ip"
           },
-          "description": {
-            "type": "string"
-          },
           "name": {
             "$ref": "#/components/schemas/Name"
           },

From 58f7129d09ba0bc99c213806aa6fa8c8965f2ea7 Mon Sep 17 00:00:00 2001
From: Charlie Park <charlie@oxidecomputer.com>
Date: Tue, 27 Feb 2024 11:24:01 -0800
Subject: [PATCH 045/157] Bump web console (#5148)

Updating console for #5144

https://github.com/oxidecomputer/console/compare/80f11673...e991ec5c
---
 tools/console_version | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/console_version b/tools/console_version
index 1b2cc62547..ee7f619b53 100644
--- a/tools/console_version
+++ b/tools/console_version
@@ -1,2 +1,2 @@
-COMMIT="80f116734fdf8ef4f3b5e49ed39e49339460407c"
-SHA2="458727fe747797860d02c9f75e62d308586c235d4708c549d5b70a40cce4d2d1"
+COMMIT="e991ec5cf148d83f135b6a692dca9b8df05acef0"
+SHA2="18d53e4485e63d9e2273a889e70785648052e8a231df89f3bcbc2ed01a0eeeb1"

From 2e4287b4656eb649db4dc8ad21915d5f673f52c6 Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Tue, 27 Feb 2024 16:27:37 -0500
Subject: [PATCH 046/157] Blueprint execution: Add dataset records for Crucible
 zones (#5143)

On each invocation, for every Crucible zone present in the blueprint
(all of which have already successfully been sent to sled-agent by this
point), we attempt to insert a `dataset` record for that zone, but only
if a record does not already exist. The datasets themselves are created
by sled-agent when we send the zone configs.

Fixes #5118.
---
 Cargo.lock                                   |   1 +
 nexus/blueprint-execution/Cargo.toml         |   1 +
 nexus/blueprint-execution/src/datasets.rs    | 315 +++++++++++++++++++
 nexus/blueprint-execution/src/lib.rs         |  13 +-
 nexus/db-model/src/dataset_kind.rs           |   2 +-
 nexus/db-queries/src/db/collection_insert.rs |  21 ++
 nexus/db-queries/src/db/datastore/dataset.rs | 265 +++++++++++++++-
 7 files changed, 614 insertions(+), 4 deletions(-)
 create mode 100644 nexus/blueprint-execution/src/datasets.rs

diff --git a/Cargo.lock b/Cargo.lock
index 85b7e5a186..27b5bbd206 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4271,6 +4271,7 @@ dependencies = [
  "dns-service-client",
  "futures",
  "httptest",
+ "illumos-utils",
  "internal-dns",
  "ipnet",
  "nexus-db-model",
diff --git a/nexus/blueprint-execution/Cargo.toml b/nexus/blueprint-execution/Cargo.toml
index 3284bda27e..164559b468 100644
--- a/nexus/blueprint-execution/Cargo.toml
+++ b/nexus/blueprint-execution/Cargo.toml
@@ -10,6 +10,7 @@ omicron-rpaths.workspace = true
 anyhow.workspace = true
 dns-service-client.workspace = true
 futures.workspace = true
+illumos-utils.workspace = true
 internal-dns.workspace = true
 nexus-db-model.workspace = true
 nexus-db-queries.workspace = true
diff --git a/nexus/blueprint-execution/src/datasets.rs b/nexus/blueprint-execution/src/datasets.rs
new file mode 100644
index 0000000000..97d8324fdb
--- /dev/null
+++ b/nexus/blueprint-execution/src/datasets.rs
@@ -0,0 +1,315 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Ensures dataset records required by a given blueprint
+
+use anyhow::Context;
+use illumos_utils::zpool::ZpoolName;
+use nexus_db_model::Dataset;
+use nexus_db_model::DatasetKind;
+use nexus_db_queries::context::OpContext;
+use nexus_db_queries::db::DataStore;
+use nexus_types::deployment::OmicronZoneConfig;
+use nexus_types::deployment::OmicronZoneType;
+use nexus_types::identity::Asset;
+use slog::info;
+use slog::warn;
+use slog_error_chain::InlineErrorChain;
+use std::collections::BTreeSet;
+use std::net::SocketAddrV6;
+
+/// For each crucible zone in `blueprint`, ensure that a corresponding dataset
+/// record exists in `datastore`
+///
+/// Does not modify any existing dataset records. Returns the number of datasets
+/// inserted.
+pub(crate) async fn ensure_crucible_dataset_records_exist(
+    opctx: &OpContext,
+    datastore: &DataStore,
+    all_omicron_zones: impl Iterator<Item = &OmicronZoneConfig>,
+) -> anyhow::Result<usize> {
+    // Before attempting to insert any datasets, first query for any existing
+    // dataset records so we can filter them out. This looks like a typical
+    // TOCTOU issue, but it is purely a performance optimization. We expect
+    // almost all executions of this function to do nothing: new crucible
+    // datasets are created very rarely relative to how frequently blueprint
+    // realization happens. We could remove this check and filter and instead
+    // run the below "insert if not exists" query on every crucible zone, and
+    // the behavior would still be correct. However, that would issue far more
+    // queries than necessary in the very common case of "we don't need to do
+    // anything at all".
+    let mut crucible_datasets = datastore
+        .dataset_list_all_batched(opctx, Some(DatasetKind::Crucible))
+        .await
+        .context("failed to list all datasets")?
+        .into_iter()
+        .map(|dataset| dataset.id())
+        .collect::<BTreeSet<_>>();
+
+    let mut num_inserted = 0;
+    let mut num_already_exist = 0;
+
+    for zone in all_omicron_zones {
+        let OmicronZoneType::Crucible { address, dataset } = &zone.zone_type
+        else {
+            continue;
+        };
+
+        let id = zone.id;
+
+        // If already present in the datastore, move on.
+        if crucible_datasets.remove(&id) {
+            num_already_exist += 1;
+            continue;
+        }
+
+        // Map progenitor client strings into the types we need. We never
+        // expect these to fail.
+        let addr: SocketAddrV6 = match address.parse() {
+            Ok(addr) => addr,
+            Err(err) => {
+                warn!(
+                    opctx.log, "failed to parse crucible zone address";
+                    "address" => address,
+                    "err" => InlineErrorChain::new(&err),
+                );
+                continue;
+            }
+        };
+        let zpool_name: ZpoolName = match dataset.pool_name.parse() {
+            Ok(name) => name,
+            Err(err) => {
+                warn!(
+                    opctx.log, "failed to parse crucible zone pool name";
+                    "pool_name" => &*dataset.pool_name,
+                    "err" => err,
+                );
+                continue;
+            }
+        };
+
+        let pool_id = zpool_name.id();
+        let dataset = Dataset::new(id, pool_id, addr, DatasetKind::Crucible);
+        let maybe_inserted = datastore
+            .dataset_insert_if_not_exists(dataset)
+            .await
+            .with_context(|| {
+                format!("failed to insert dataset record for dataset {id}")
+            })?;
+
+        // If we succeeded in inserting, log it; if `maybe_dataset` is `None`,
+        // we must have lost the TOCTOU race described above, and another Nexus
+        // must have inserted this dataset before we could.
+        if maybe_inserted.is_some() {
+            info!(
+                opctx.log,
+                "inserted new dataset for crucible zone";
+                "id" => %id,
+            );
+            num_inserted += 1;
+        } else {
+            num_already_exist += 1;
+        }
+    }
+
+    // We don't currently support removing datasets, so this would be
+    // surprising: the database contains dataset records that are no longer in
+    // our blueprint. We can't do anything about this, so just warn.
+    if !crucible_datasets.is_empty() {
+        warn!(
+            opctx.log,
+            "database contains {} unexpected crucible datasets",
+            crucible_datasets.len();
+            "dataset_ids" => ?crucible_datasets,
+        );
+    }
+
+    info!(
+        opctx.log,
+        "ensured all crucible zones have dataset records";
+        "num_inserted" => num_inserted,
+        "num_already_existed" => num_already_exist,
+    );
+
+    Ok(num_inserted)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use nexus_db_model::SledBaseboard;
+    use nexus_db_model::SledSystemHardware;
+    use nexus_db_model::SledUpdate;
+    use nexus_db_model::Zpool;
+    use nexus_test_utils_macros::nexus_test;
+    use sled_agent_client::types::OmicronZoneDataset;
+    use uuid::Uuid;
+
+    type ControlPlaneTestContext =
+        nexus_test_utils::ControlPlaneTestContext<omicron_nexus::Server>;
+
+    #[nexus_test]
+    async fn test_ensure_crucible_dataset_records_exist(
+        cptestctx: &ControlPlaneTestContext,
+    ) {
+        // Set up.
+        let nexus = &cptestctx.server.apictx().nexus;
+        let datastore = nexus.datastore();
+        let opctx = OpContext::for_tests(
+            cptestctx.logctx.log.clone(),
+            datastore.clone(),
+        );
+        let opctx = &opctx;
+
+        // Use the standard representative inventory collection.
+        let representative = nexus_inventory::examples::representative();
+        let collection = representative.builder.build();
+
+        // Record the sleds and zpools contained in this collection.
+        let rack_id = Uuid::new_v4();
+        for (&sled_id, config) in &collection.omicron_zones {
+            let sled = SledUpdate::new(
+                sled_id,
+                "[::1]:0".parse().unwrap(),
+                SledBaseboard {
+                    serial_number: format!("test-{sled_id}"),
+                    part_number: "test-sled".to_string(),
+                    revision: 0,
+                },
+                SledSystemHardware {
+                    is_scrimlet: false,
+                    usable_hardware_threads: 128,
+                    usable_physical_ram: (64 << 30).try_into().unwrap(),
+                    reservoir_size: (16 << 30).try_into().unwrap(),
+                },
+                rack_id,
+            );
+            datastore
+                .sled_upsert(sled)
+                .await
+                .expect("failed to upsert sled")
+                .unwrap();
+
+            for zone in &config.zones.zones {
+                let OmicronZoneType::Crucible { dataset, .. } = &zone.zone_type
+                else {
+                    continue;
+                };
+                let zpool_name: ZpoolName =
+                    dataset.pool_name.parse().expect("invalid zpool name");
+                let zpool = Zpool::new(
+                    zpool_name.id(),
+                    sled_id,
+                    Uuid::new_v4(), // physical_disk_id
+                    (1 << 30).try_into().unwrap(), // total_size
+                );
+                datastore
+                    .zpool_upsert(zpool)
+                    .await
+                    .expect("failed to upsert zpool");
+            }
+        }
+
+        // How many crucible zones are there?
+        let ncrucible_zones = collection
+            .all_omicron_zones()
+            .filter(|z| matches!(z.zone_type, OmicronZoneType::Crucible { .. }))
+            .count();
+
+        // Prior to ensuring datasets exist, there should be none.
+        assert_eq!(
+            datastore
+                .dataset_list_all_batched(opctx, Some(DatasetKind::Crucible))
+                .await
+                .unwrap()
+                .len(),
+            0
+        );
+        let ndatasets_inserted = ensure_crucible_dataset_records_exist(
+            opctx,
+            datastore,
+            collection.all_omicron_zones(),
+        )
+        .await
+        .expect("failed to ensure crucible datasets");
+
+        // We should have inserted a dataset for each crucible zone.
+        assert_eq!(ncrucible_zones, ndatasets_inserted);
+        assert_eq!(
+            datastore
+                .dataset_list_all_batched(opctx, Some(DatasetKind::Crucible))
+                .await
+                .unwrap()
+                .len(),
+            ncrucible_zones,
+        );
+
+        // Ensuring the same crucible datasets again should insert no new
+        // records.
+        let ndatasets_inserted = ensure_crucible_dataset_records_exist(
+            opctx,
+            datastore,
+            collection.all_omicron_zones(),
+        )
+        .await
+        .expect("failed to ensure crucible datasets");
+        assert_eq!(0, ndatasets_inserted);
+        assert_eq!(
+            datastore
+                .dataset_list_all_batched(opctx, Some(DatasetKind::Crucible))
+                .await
+                .unwrap()
+                .len(),
+            ncrucible_zones,
+        );
+
+        // Create another zpool on one of the sleds, so we can add a new
+        // crucible zone that uses it.
+        let new_zpool_id = Uuid::new_v4();
+        for &sled_id in collection.omicron_zones.keys().take(1) {
+            let zpool = Zpool::new(
+                new_zpool_id,
+                sled_id,
+                Uuid::new_v4(),                // physical_disk_id
+                (1 << 30).try_into().unwrap(), // total_size
+            );
+            datastore
+                .zpool_upsert(zpool)
+                .await
+                .expect("failed to upsert zpool");
+        }
+
+        // Call `ensure_crucible_dataset_records_exist` again, adding a new
+        // crucible zone. It should insert only this new zone.
+        let new_zone = OmicronZoneConfig {
+            id: Uuid::new_v4(),
+            underlay_address: "::1".parse().unwrap(),
+            zone_type: OmicronZoneType::Crucible {
+                address: "[::1]:0".to_string(),
+                dataset: OmicronZoneDataset {
+                    pool_name: ZpoolName::new_external(new_zpool_id)
+                        .to_string()
+                        .parse()
+                        .unwrap(),
+                },
+            },
+        };
+        let ndatasets_inserted = ensure_crucible_dataset_records_exist(
+            opctx,
+            datastore,
+            collection.all_omicron_zones().chain(std::iter::once(&new_zone)),
+        )
+        .await
+        .expect("failed to ensure crucible datasets");
+        assert_eq!(ndatasets_inserted, 1);
+        assert_eq!(
+            datastore
+                .dataset_list_all_batched(opctx, Some(DatasetKind::Crucible))
+                .await
+                .unwrap()
+                .len(),
+            ncrucible_zones + 1,
+        );
+    }
+}
diff --git a/nexus/blueprint-execution/src/lib.rs b/nexus/blueprint-execution/src/lib.rs
index d6f5f8fc31..531c4f57a8 100644
--- a/nexus/blueprint-execution/src/lib.rs
+++ b/nexus/blueprint-execution/src/lib.rs
@@ -19,6 +19,7 @@ use std::collections::BTreeMap;
 use std::net::SocketAddrV6;
 use uuid::Uuid;
 
+mod datasets;
 mod dns;
 mod omicron_zones;
 mod resource_allocation;
@@ -89,6 +90,14 @@ where
     omicron_zones::deploy_zones(&opctx, &sleds_by_id, &blueprint.omicron_zones)
         .await?;
 
+    datasets::ensure_crucible_dataset_records_exist(
+        &opctx,
+        datastore,
+        blueprint.all_omicron_zones().map(|(_sled_id, zone)| zone),
+    )
+    .await
+    .map_err(|err| vec![err])?;
+
     dns::deploy_dns(
         &opctx,
         datastore,
@@ -97,5 +106,7 @@ where
         &sleds_by_id,
     )
     .await
-    .map_err(|e| vec![anyhow!("{}", InlineErrorChain::new(&e))])
+    .map_err(|e| vec![anyhow!("{}", InlineErrorChain::new(&e))])?;
+
+    Ok(())
 }
diff --git a/nexus/db-model/src/dataset_kind.rs b/nexus/db-model/src/dataset_kind.rs
index 00317592e8..86495b9d61 100644
--- a/nexus/db-model/src/dataset_kind.rs
+++ b/nexus/db-model/src/dataset_kind.rs
@@ -11,7 +11,7 @@ impl_enum_type!(
     #[diesel(postgres_type(name = "dataset_kind", schema = "public"))]
     pub struct DatasetKindEnum;
 
-    #[derive(Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)]
+    #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)]
     #[diesel(sql_type = DatasetKindEnum)]
     pub enum DatasetKind;
 
diff --git a/nexus/db-queries/src/db/collection_insert.rs b/nexus/db-queries/src/db/collection_insert.rs
index b295f0574d..ef2a4a4d48 100644
--- a/nexus/db-queries/src/db/collection_insert.rs
+++ b/nexus/db-queries/src/db/collection_insert.rs
@@ -202,6 +202,27 @@ where
             .map_err(|e| Self::translate_async_error(e))
     }
 
+    /// Issues the CTE asynchronously and parses the result.
+    ///
+    /// The four outcomes are:
+    /// - Ok(Some(new row))
+    /// - Ok(None)
+    /// - Error(collection not found)
+    /// - Error(other diesel error)
+    pub async fn insert_and_get_optional_result_async(
+        self,
+        conn: &async_bb8_diesel::Connection<DbConnection>,
+    ) -> AsyncInsertIntoCollectionResult<Option<ResourceType>>
+    where
+        // We require this bound to ensure that "Self" is runnable as query.
+        Self: query_methods::LoadQuery<'static, DbConnection, ResourceType>,
+    {
+        self.get_result_async::<ResourceType>(conn)
+            .await
+            .optional()
+            .map_err(|e| Self::translate_async_error(e))
+    }
+
     /// Issues the CTE asynchronously and parses the result.
     ///
     /// The three outcomes are:
diff --git a/nexus/db-queries/src/db/datastore/dataset.rs b/nexus/db-queries/src/db/datastore/dataset.rs
index 0b26789e8f..3c1fd0afb1 100644
--- a/nexus/db-queries/src/db/datastore/dataset.rs
+++ b/nexus/db-queries/src/db/datastore/dataset.rs
@@ -5,6 +5,9 @@
 //! [`DataStore`] methods on [`Dataset`]s.
 
 use super::DataStore;
+use super::SQL_BATCH_SIZE;
+use crate::authz;
+use crate::context::OpContext;
 use crate::db;
 use crate::db::collection_insert::AsyncInsertError;
 use crate::db::collection_insert::DatastoreCollection;
@@ -13,12 +16,17 @@ use crate::db::error::ErrorHandler;
 use crate::db::identity::Asset;
 use crate::db::model::Dataset;
 use crate::db::model::Zpool;
+use crate::db::pagination::paginated;
+use crate::db::pagination::Paginator;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use chrono::Utc;
 use diesel::prelude::*;
 use diesel::upsert::excluded;
+use nexus_db_model::DatasetKind;
 use omicron_common::api::external::CreateResult;
+use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::Error;
+use omicron_common::api::external::ListResultVec;
 use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::LookupType;
 use omicron_common::api::external::ResourceType;
@@ -45,11 +53,12 @@ impl DataStore {
     ) -> CreateResult<Dataset> {
         use db::schema::dataset::dsl;
 
+        let dataset_id = dataset.id();
         let zpool_id = dataset.pool_id;
         Zpool::insert_resource(
             zpool_id,
             diesel::insert_into(dsl::dataset)
-                .values(dataset.clone())
+                .values(dataset)
                 .on_conflict(dsl::id)
                 .do_update()
                 .set((
@@ -73,9 +82,261 @@ impl DataStore {
                 e,
                 ErrorHandler::Conflict(
                     ResourceType::Dataset,
-                    &dataset.id().to_string(),
+                    &dataset_id.to_string(),
                 ),
             ),
         })
     }
+
+    /// Stores a new dataset in the database, but only if a dataset with the
+    /// given `id` does not already exist
+    ///
+    /// Does not update existing rows. If a dataset with the given ID already
+    /// exists, returns `Ok(None)`.
+    pub async fn dataset_insert_if_not_exists(
+        &self,
+        dataset: Dataset,
+    ) -> CreateResult<Option<Dataset>> {
+        use db::schema::dataset::dsl;
+
+        let zpool_id = dataset.pool_id;
+        Zpool::insert_resource(
+            zpool_id,
+            diesel::insert_into(dsl::dataset)
+                .values(dataset)
+                .on_conflict(dsl::id)
+                .do_nothing(),
+        )
+        .insert_and_get_optional_result_async(
+            &*self.pool_connection_unauthorized().await?,
+        )
+        .await
+        .map_err(|e| match e {
+            AsyncInsertError::CollectionNotFound => Error::ObjectNotFound {
+                type_name: ResourceType::Zpool,
+                lookup_type: LookupType::ById(zpool_id),
+            },
+            AsyncInsertError::DatabaseError(e) => {
+                public_error_from_diesel(e, ErrorHandler::Server)
+            }
+        })
+    }
+
+    /// List one page of datasets
+    ///
+    /// If `filter_kind` is `Some(value)`, only datasets with a `kind` matching
+    /// `value` will be returned. If `filter_kind` is `None`, all datasets will
+    /// be returned.
+    async fn dataset_list(
+        &self,
+        opctx: &OpContext,
+        filter_kind: Option<DatasetKind>,
+        pagparams: &DataPageParams<'_, Uuid>,
+    ) -> ListResultVec<Dataset> {
+        opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?;
+        use db::schema::dataset::dsl;
+
+        let mut query = paginated(dsl::dataset, dsl::id, pagparams)
+            .filter(dsl::time_deleted.is_null());
+
+        if let Some(kind) = filter_kind {
+            query = query.filter(dsl::kind.eq(kind));
+        }
+
+        query
+            .select(Dataset::as_select())
+            .load_async(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
+    }
+
+    /// List all datasets, making as many queries as needed to get them all
+    ///
+    /// If `filter_kind` is `Some(value)`, only datasets with a `kind` matching
+    /// `value` will be returned. If `filter_kind` is `None`, all datasets will
+    /// be returned.
+    ///
+    /// This should generally not be used in API handlers or other
+    /// latency-sensitive contexts, but it can make sense in saga actions or
+    /// background tasks.
+    pub async fn dataset_list_all_batched(
+        &self,
+        opctx: &OpContext,
+        filter_kind: Option<DatasetKind>,
+    ) -> ListResultVec<Dataset> {
+        opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?;
+        opctx.check_complex_operations_allowed()?;
+
+        let mut all_datasets = Vec::new();
+        let mut paginator = Paginator::new(SQL_BATCH_SIZE);
+        while let Some(p) = paginator.next() {
+            let batch = self
+                .dataset_list(opctx, filter_kind, &p.current_pagparams())
+                .await?;
+            paginator =
+                p.found_batch(&batch, &|d: &nexus_db_model::Dataset| d.id());
+            all_datasets.extend(batch);
+        }
+
+        Ok(all_datasets)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::db::datastore::test_utils::datastore_test;
+    use nexus_db_model::SledBaseboard;
+    use nexus_db_model::SledSystemHardware;
+    use nexus_db_model::SledUpdate;
+    use nexus_test_utils::db::test_setup_database;
+    use omicron_test_utils::dev;
+
+    #[tokio::test]
+    async fn test_insert_if_not_exists() {
+        let logctx = dev::test_setup_log("inventory_insert");
+        let mut db = test_setup_database(&logctx.log).await;
+        let (opctx, datastore) = datastore_test(&logctx, &db).await;
+        let opctx = &opctx;
+
+        // There should be no datasets initially.
+        assert_eq!(
+            datastore.dataset_list_all_batched(opctx, None).await.unwrap(),
+            []
+        );
+
+        // Create a fake sled that holds our fake zpool.
+        let sled_id = Uuid::new_v4();
+        let sled = SledUpdate::new(
+            sled_id,
+            "[::1]:0".parse().unwrap(),
+            SledBaseboard {
+                serial_number: "test-sn".to_string(),
+                part_number: "test-pn".to_string(),
+                revision: 0,
+            },
+            SledSystemHardware {
+                is_scrimlet: false,
+                usable_hardware_threads: 128,
+                usable_physical_ram: (64 << 30).try_into().unwrap(),
+                reservoir_size: (16 << 30).try_into().unwrap(),
+            },
+            Uuid::new_v4(),
+        );
+        datastore
+            .sled_upsert(sled)
+            .await
+            .expect("failed to upsert sled")
+            .unwrap();
+
+        // Create a fake zpool that backs our fake datasets.
+        let zpool_id = Uuid::new_v4();
+        let zpool = Zpool::new(
+            zpool_id,
+            sled_id,
+            Uuid::new_v4(),
+            (1 << 30).try_into().unwrap(),
+        );
+        datastore.zpool_upsert(zpool).await.expect("failed to upsert zpool");
+
+        // Inserting a new dataset should succeed.
+        let dataset1 = datastore
+            .dataset_insert_if_not_exists(Dataset::new(
+                Uuid::new_v4(),
+                zpool_id,
+                "[::1]:0".parse().unwrap(),
+                DatasetKind::Crucible,
+            ))
+            .await
+            .expect("failed to insert dataset")
+            .expect("insert found unexpected existing dataset");
+        let mut expected_datasets = vec![dataset1.clone()];
+        assert_eq!(
+            datastore.dataset_list_all_batched(opctx, None).await.unwrap(),
+            expected_datasets,
+        );
+        assert_eq!(
+            datastore
+                .dataset_list_all_batched(opctx, Some(DatasetKind::Crucible))
+                .await
+                .unwrap(),
+            expected_datasets,
+        );
+        assert_eq!(
+            datastore
+                .dataset_list_all_batched(opctx, Some(DatasetKind::Cockroach))
+                .await
+                .unwrap(),
+            [],
+        );
+
+        // Attempting to insert another dataset with the same ID should succeed
+        // without updating the existing record. We'll check this by passing a
+        // different socket address and kind.
+        let insert_again_result = datastore
+            .dataset_insert_if_not_exists(Dataset::new(
+                dataset1.id(),
+                zpool_id,
+                "[::1]:12345".parse().unwrap(),
+                DatasetKind::Cockroach,
+            ))
+            .await
+            .expect("failed to do-nothing insert dataset");
+        assert_eq!(insert_again_result, None);
+        assert_eq!(
+            datastore.dataset_list_all_batched(opctx, None).await.unwrap(),
+            expected_datasets,
+        );
+
+        // We can can also upsert a different dataset...
+        let dataset2 = datastore
+            .dataset_upsert(Dataset::new(
+                Uuid::new_v4(),
+                zpool_id,
+                "[::1]:0".parse().unwrap(),
+                DatasetKind::Cockroach,
+            ))
+            .await
+            .expect("failed to upsert dataset");
+        expected_datasets.push(dataset2.clone());
+        expected_datasets.sort_by_key(|d| d.id());
+        assert_eq!(
+            datastore.dataset_list_all_batched(opctx, None).await.unwrap(),
+            expected_datasets,
+        );
+        assert_eq!(
+            datastore
+                .dataset_list_all_batched(opctx, Some(DatasetKind::Crucible))
+                .await
+                .unwrap(),
+            [dataset1.clone()],
+        );
+        assert_eq!(
+            datastore
+                .dataset_list_all_batched(opctx, Some(DatasetKind::Cockroach))
+                .await
+                .unwrap(),
+            [dataset2.clone()],
+        );
+
+        // ... and trying to `insert_if_not_exists` should similarly return
+        // `None`.
+        let insert_again_result = datastore
+            .dataset_insert_if_not_exists(Dataset::new(
+                dataset1.id(),
+                zpool_id,
+                "[::1]:12345".parse().unwrap(),
+                DatasetKind::Cockroach,
+            ))
+            .await
+            .expect("failed to do-nothing insert dataset");
+        assert_eq!(insert_again_result, None);
+        assert_eq!(
+            datastore.dataset_list_all_batched(opctx, None).await.unwrap(),
+            expected_datasets,
+        );
+
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
 }

From 0154193c5cee57e29e6d8f8f2e36610e3ea686d8 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Tue, 27 Feb 2024 22:29:42 -0800
Subject: [PATCH 047/157] chore(deps): update rust crate dyn-clone to 1.0.17
 (#5141)

---
 Cargo.lock | 4 ++--
 Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 27b5bbd206..b057b84f6a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2042,9 +2042,9 @@ dependencies = [
 
 [[package]]
 name = "dyn-clone"
-version = "1.0.16"
+version = "1.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d"
+checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125"
 
 [[package]]
 name = "ecdsa"
diff --git a/Cargo.toml b/Cargo.toml
index db37547ea0..ec2c345edf 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -200,7 +200,7 @@ dns-server = { path = "dns-server" }
 dns-service-client = { path = "clients/dns-service-client" }
 dpd-client = { path = "clients/dpd-client" }
 dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] }
-dyn-clone = "1.0.16"
+dyn-clone = "1.0.17"
 either = "1.10.0"
 expectorate = "1.1.0"
 fatfs = "0.3.6"

From 0761adaef664c9c0f86726ca308ad91d36a0f53d Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Wed, 28 Feb 2024 13:30:27 +0000
Subject: [PATCH 048/157] chore(deps): update taiki-e/install-action digest to
 1d776b1 (#5146)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [taiki-e/install-action](https://togithub.com/taiki-e/install-action)
| action | digest | [`4ce8785` ->
`1d776b1`](https://togithub.com/taiki-e/install-action/compare/4ce8785...1d776b1)
|

---

### Configuration

📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone
America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone
America/Los_Angeles.

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update
again.

---

- [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check
this box

---

This PR has been generated by [Renovate
Bot](https://togithub.com/renovatebot/renovate).

<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNy4yMTQuMSIsInVwZGF0ZWRJblZlciI6IjM3LjIxOS4yIiwidGFyZ2V0QnJhbmNoIjoibWFpbiJ9-->

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index 10c1c04003..0f942e7d76 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@4ce8785db2a8a56c9ede16f705c2c49c5c61669c # v2
+        uses: taiki-e/install-action@1d776b18af134fca43744080130085d256938196 # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From ceeceee341967e776e349b0efffa55d30ff55c7e Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Thu, 29 Feb 2024 03:38:27 +0000
Subject: [PATCH 049/157] chore(deps): update rust to v1.76.0 (#5097)

Co-authored-by: Rain <rain@oxide.computer>
---
 Cargo.lock                      | 300 +++++++++++++-------------------
 Cargo.toml                      |   7 +-
 dev-tools/oxlog/src/lib.rs      |   6 +-
 dev-tools/xtask/src/main.rs     |  20 ++-
 nexus/src/app/background/mod.rs |   2 -
 nexus/src/cidata.rs             |   6 +-
 rust-toolchain.toml             |   2 +-
 sled-hardware/Cargo.toml        |   1 +
 sled-hardware/src/cleanup.rs    |  13 +-
 wicket/src/state/mod.rs         |   7 +-
 wicket/src/ui/widgets/mod.rs    |   2 +-
 workspace-hack/Cargo.toml       |  14 +-
 12 files changed, 169 insertions(+), 211 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index b057b84f6a..2c1e4acd50 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -174,7 +174,7 @@ dependencies = [
  "omicron-workspace-hack",
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -275,7 +275,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -297,7 +297,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -308,7 +308,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -361,7 +361,7 @@ dependencies = [
  "quote",
  "serde",
  "serde_tokenstream 0.2.0",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -463,20 +463,20 @@ dependencies = [
 [[package]]
 name = "bhyve_api"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d#c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d"
+source = "git+https://github.com/oxidecomputer/propolis?rev=488a70c644e332dae98447292a95e45074c8107c#488a70c644e332dae98447292a95e45074c8107c"
 dependencies = [
  "bhyve_api_sys",
  "libc",
- "strum 0.25.0",
+ "strum 0.26.1",
 ]
 
 [[package]]
 name = "bhyve_api_sys"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d#c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d"
+source = "git+https://github.com/oxidecomputer/propolis?rev=488a70c644e332dae98447292a95e45074c8107c#488a70c644e332dae98447292a95e45074c8107c"
 dependencies = [
  "libc",
- "strum 0.25.0",
+ "strum 0.26.1",
 ]
 
 [[package]]
@@ -498,7 +498,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.48",
+ "syn 2.0.51",
  "which",
 ]
 
@@ -655,7 +655,7 @@ dependencies = [
  "ipnetwork",
  "omicron-common",
  "omicron-workspace-hack",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
  "regress",
  "reqwest",
  "schemars",
@@ -1018,7 +1018,7 @@ dependencies = [
  "heck 0.4.1",
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -1315,7 +1315,7 @@ dependencies = [
  "chrono",
  "crucible-workspace-hack",
  "percent-encoding",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
  "reqwest",
  "schemars",
  "serde",
@@ -1331,7 +1331,7 @@ dependencies = [
  "chrono",
  "crucible-workspace-hack",
  "percent-encoding",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
  "reqwest",
  "schemars",
  "serde",
@@ -1452,7 +1452,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -1500,7 +1500,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim 0.10.0",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -1522,7 +1522,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
 dependencies = [
  "darling_core 0.20.3",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -1556,7 +1556,7 @@ dependencies = [
  "quote",
  "serde",
  "serde_tokenstream 0.2.0",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -1568,8 +1568,8 @@ dependencies = [
  "omicron-common",
  "omicron-workspace-hack",
  "omicron-zone-package",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
- "progenitor-client 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
+ "progenitor-client",
  "quote",
  "reqwest",
  "rustfmt-wrapper",
@@ -1608,7 +1608,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -1641,7 +1641,7 @@ checksum = "5fe87ce4529967e0ba1dcf8450bab64d97dfd5010a6256187ffe2e43e6f0e049"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -1661,7 +1661,7 @@ checksum = "62d671cc41a825ebabc75757b62d3d168c577f9149b2d49ece1dad1f72119d25"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -1748,7 +1748,7 @@ dependencies = [
  "diesel_table_macro_syntax",
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -1757,7 +1757,7 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5"
 dependencies = [
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -1891,7 +1891,7 @@ dependencies = [
  "chrono",
  "http 0.2.11",
  "omicron-workspace-hack",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
  "reqwest",
  "schemars",
  "serde",
@@ -1945,8 +1945,8 @@ dependencies = [
  "ipnetwork",
  "omicron-workspace-hack",
  "omicron-zone-package",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
- "progenitor-client 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
+ "progenitor-client",
  "quote",
  "rand 0.8.5",
  "regress",
@@ -2015,7 +2015,7 @@ dependencies = [
  "quote",
  "serde",
  "serde_tokenstream 0.2.0",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -2388,7 +2388,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -2505,7 +2505,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -2584,7 +2584,7 @@ dependencies = [
  "chrono",
  "gateway-messages",
  "omicron-workspace-hack",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
  "rand 0.8.5",
  "reqwest",
  "schemars",
@@ -3506,7 +3506,7 @@ version = "0.1.0"
 dependencies = [
  "installinator-common",
  "omicron-workspace-hack",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
  "regress",
  "reqwest",
  "schemars",
@@ -3586,7 +3586,7 @@ dependencies = [
  "omicron-common",
  "omicron-test-utils",
  "omicron-workspace-hack",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
  "reqwest",
  "serde",
  "serde_json",
@@ -3739,7 +3739,7 @@ version = "0.1.0"
 source = "git+https://github.com/oxidecomputer/opte?rev=1d29ef60a18179babfb44f0f7a3c2fe71034a2c1#1d29ef60a18179babfb44f0f7a3c2fe71034a2c1"
 dependencies = [
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -4083,8 +4083,8 @@ dependencies = [
  "omicron-common",
  "omicron-workspace-hack",
  "omicron-zone-package",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
- "progenitor-client 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
+ "progenitor-client",
  "quote",
  "reqwest",
  "rustfmt-wrapper",
@@ -4164,7 +4164,7 @@ dependencies = [
  "cfg-if",
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -4306,7 +4306,7 @@ dependencies = [
  "omicron-common",
  "omicron-passwords",
  "omicron-workspace-hack",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
  "regress",
  "reqwest",
  "schemars",
@@ -4498,7 +4498,7 @@ dependencies = [
  "omicron-workspace-hack",
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -4560,7 +4560,7 @@ version = "0.1.0"
 dependencies = [
  "omicron-workspace-hack",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -4714,7 +4714,7 @@ checksum = "9e6a0fd4f737c707bd9086cc16c925f294943eb62eb71499e9fd4cf71f8b9f4e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -4889,7 +4889,7 @@ dependencies = [
  "omicron-workspace-hack",
  "once_cell",
  "parse-display",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
  "proptest",
  "rand 0.8.5",
  "regress",
@@ -5064,7 +5064,7 @@ dependencies = [
  "petgraph",
  "pq-sys",
  "pretty_assertions",
- "progenitor-client 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor-client",
  "propolis-client",
  "rand 0.8.5",
  "rcgen",
@@ -5439,10 +5439,9 @@ dependencies = [
  "socket2 0.5.5",
  "spin 0.9.8",
  "string_cache",
- "strum 0.25.0",
  "subtle",
  "syn 1.0.109",
- "syn 2.0.48",
+ "syn 2.0.51",
  "time",
  "time-macros",
  "tokio",
@@ -5562,7 +5561,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -5673,7 +5672,7 @@ dependencies = [
  "http 0.2.11",
  "hyper 0.14.27",
  "omicron-workspace-hack",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
  "rand 0.8.5",
  "regress",
  "reqwest",
@@ -5729,7 +5728,7 @@ dependencies = [
  "futures",
  "omicron-common",
  "omicron-workspace-hack",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
  "reqwest",
  "serde",
  "slog",
@@ -5847,7 +5846,7 @@ dependencies = [
  "omicron-workspace-hack",
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -6012,7 +6011,7 @@ dependencies = [
  "regex",
  "regex-syntax 0.8.2",
  "structmeta 0.3.0",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -6159,7 +6158,7 @@ dependencies = [
  "pest_meta",
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -6229,7 +6228,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -6484,7 +6483,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5"
 dependencies = [
  "proc-macro2",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -6542,43 +6541,18 @@ dependencies = [
 [[package]]
 name = "progenitor"
 version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor?branch=main#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
-dependencies = [
- "progenitor-client 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
- "progenitor-impl 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
- "progenitor-macro 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
- "serde_json",
-]
-
-[[package]]
-name = "progenitor"
-version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
+source = "git+https://github.com/oxidecomputer/progenitor?branch=main#08bbafc251d3d6828ca5921d1658c3d12d64fe7c"
 dependencies = [
- "progenitor-client 0.5.0 (git+https://github.com/oxidecomputer/progenitor)",
- "progenitor-impl 0.5.0 (git+https://github.com/oxidecomputer/progenitor)",
- "progenitor-macro 0.5.0 (git+https://github.com/oxidecomputer/progenitor)",
+ "progenitor-client",
+ "progenitor-impl",
+ "progenitor-macro",
  "serde_json",
 ]
 
 [[package]]
 name = "progenitor-client"
 version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor?branch=main#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
-dependencies = [
- "bytes",
- "futures-core",
- "percent-encoding",
- "reqwest",
- "serde",
- "serde_json",
- "serde_urlencoded",
-]
-
-[[package]]
-name = "progenitor-client"
-version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
+source = "git+https://github.com/oxidecomputer/progenitor?branch=main#08bbafc251d3d6828ca5921d1658c3d12d64fe7c"
 dependencies = [
  "bytes",
  "futures-core",
@@ -6592,7 +6566,7 @@ dependencies = [
 [[package]]
 name = "progenitor-impl"
 version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor?branch=main#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
+source = "git+https://github.com/oxidecomputer/progenitor?branch=main#08bbafc251d3d6828ca5921d1658c3d12d64fe7c"
 dependencies = [
  "getopts",
  "heck 0.4.1",
@@ -6605,77 +6579,38 @@ dependencies = [
  "schemars",
  "serde",
  "serde_json",
- "syn 2.0.48",
+ "syn 2.0.51",
  "thiserror",
  "typify",
  "unicode-ident",
 ]
 
-[[package]]
-name = "progenitor-impl"
-version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
-dependencies = [
- "getopts",
- "heck 0.4.1",
- "http 0.2.11",
- "indexmap 2.2.3",
- "openapiv3",
- "proc-macro2",
- "quote",
- "regex",
- "schemars",
- "serde",
- "serde_json",
- "syn 2.0.48",
- "thiserror",
- "typify",
- "unicode-ident",
-]
-
-[[package]]
-name = "progenitor-macro"
-version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor?branch=main#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
-dependencies = [
- "openapiv3",
- "proc-macro2",
- "progenitor-impl 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
- "quote",
- "schemars",
- "serde",
- "serde_json",
- "serde_tokenstream 0.2.0",
- "serde_yaml",
- "syn 2.0.48",
-]
-
 [[package]]
 name = "progenitor-macro"
 version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor#bc0bb4b0fb40084f189eb1a8807b17fbd0ce0b64"
+source = "git+https://github.com/oxidecomputer/progenitor?branch=main#08bbafc251d3d6828ca5921d1658c3d12d64fe7c"
 dependencies = [
  "openapiv3",
  "proc-macro2",
- "progenitor-impl 0.5.0 (git+https://github.com/oxidecomputer/progenitor)",
+ "progenitor-impl",
  "quote",
  "schemars",
  "serde",
  "serde_json",
  "serde_tokenstream 0.2.0",
  "serde_yaml",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
 name = "propolis-client"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d#c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d"
+source = "git+https://github.com/oxidecomputer/propolis?rev=488a70c644e332dae98447292a95e45074c8107c#488a70c644e332dae98447292a95e45074c8107c"
 dependencies = [
  "async-trait",
  "base64",
  "futures",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor)",
+ "progenitor",
  "rand 0.8.5",
  "reqwest",
  "schemars",
@@ -6691,7 +6626,7 @@ dependencies = [
 [[package]]
 name = "propolis-mock-server"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d#c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d"
+source = "git+https://github.com/oxidecomputer/propolis?rev=488a70c644e332dae98447292a95e45074c8107c#488a70c644e332dae98447292a95e45074c8107c"
 dependencies = [
  "anyhow",
  "atty",
@@ -6700,7 +6635,7 @@ dependencies = [
  "dropshot",
  "futures",
  "hyper 0.14.27",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor)",
+ "progenitor",
  "propolis_types",
  "rand 0.8.5",
  "reqwest",
@@ -6721,7 +6656,7 @@ dependencies = [
 [[package]]
 name = "propolis_types"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d#c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d"
+source = "git+https://github.com/oxidecomputer/propolis?rev=488a70c644e332dae98447292a95e45074c8107c#488a70c644e332dae98447292a95e45074c8107c"
 dependencies = [
  "schemars",
  "serde",
@@ -7043,7 +6978,7 @@ checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -7120,9 +7055,9 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.11.22"
+version = "0.11.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b"
+checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251"
 dependencies = [
  "base64",
  "bytes",
@@ -7148,6 +7083,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_urlencoded",
+ "sync_wrapper",
  "system-configuration",
  "tokio",
  "tokio-native-tls",
@@ -7285,7 +7221,7 @@ dependencies = [
  "regex",
  "relative-path",
  "rustc_version 0.4.0",
- "syn 2.0.48",
+ "syn 2.0.51",
  "unicode-ident",
 ]
 
@@ -7714,7 +7650,7 @@ checksum = "7f81c2fde025af7e69b1d1420531c8a8811ca898919db177141a85313b1cb932"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -7808,9 +7744,9 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.196"
+version = "1.0.197"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32"
+checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
 dependencies = [
  "serde_derive",
 ]
@@ -7846,13 +7782,13 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.196"
+version = "1.0.197"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67"
+checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -7913,7 +7849,7 @@ checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -7945,7 +7881,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -7987,7 +7923,7 @@ dependencies = [
  "darling 0.20.3",
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -8166,7 +8102,7 @@ dependencies = [
  "ipnetwork",
  "omicron-common",
  "omicron-workspace-hack",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
  "regress",
  "reqwest",
  "schemars",
@@ -8196,6 +8132,7 @@ dependencies = [
  "schemars",
  "serde",
  "slog",
+ "slog-error-chain",
  "thiserror",
  "tofino",
  "tokio",
@@ -8303,7 +8240,7 @@ source = "git+https://github.com/oxidecomputer/slog-error-chain?branch=main#15f6
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -8558,7 +8495,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -8665,7 +8602,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "structmeta-derive 0.2.0",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -8677,7 +8614,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "structmeta-derive 0.3.0",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -8688,7 +8625,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -8699,7 +8636,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -8731,9 +8668,6 @@ name = "strum"
 version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
-dependencies = [
- "strum_macros 0.25.2",
-]
 
 [[package]]
 name = "strum"
@@ -8767,7 +8701,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -8780,7 +8714,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -8827,15 +8761,21 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.48"
+version = "2.0.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
+checksum = "6ab617d94515e94ae53b8406c628598680aa0c9587474ecbe58188f7b345d66c"
 dependencies = [
  "proc-macro2",
  "quote",
  "unicode-ident",
 ]
 
+[[package]]
+name = "sync_wrapper"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
+
 [[package]]
 name = "synstructure"
 version = "0.12.6"
@@ -9010,7 +8950,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "structmeta 0.2.0",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -9050,7 +8990,7 @@ checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -9237,7 +9177,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -9504,7 +9444,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -9731,7 +9671,7 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
 [[package]]
 name = "typify"
 version = "0.0.15"
-source = "git+https://github.com/oxidecomputer/typify#ce009d6f83b620cbd0e3acdd9b9ea071018471d8"
+source = "git+https://github.com/oxidecomputer/typify#131fe0ef3722d3034a61a8ab2994c29f9c978903"
 dependencies = [
  "typify-impl",
  "typify-macro",
@@ -9740,7 +9680,7 @@ dependencies = [
 [[package]]
 name = "typify-impl"
 version = "0.0.15"
-source = "git+https://github.com/oxidecomputer/typify#ce009d6f83b620cbd0e3acdd9b9ea071018471d8"
+source = "git+https://github.com/oxidecomputer/typify#131fe0ef3722d3034a61a8ab2994c29f9c978903"
 dependencies = [
  "heck 0.4.1",
  "log",
@@ -9749,7 +9689,7 @@ dependencies = [
  "regress",
  "schemars",
  "serde_json",
- "syn 2.0.48",
+ "syn 2.0.51",
  "thiserror",
  "unicode-ident",
 ]
@@ -9757,7 +9697,7 @@ dependencies = [
 [[package]]
 name = "typify-macro"
 version = "0.0.15"
-source = "git+https://github.com/oxidecomputer/typify#ce009d6f83b620cbd0e3acdd9b9ea071018471d8"
+source = "git+https://github.com/oxidecomputer/typify#131fe0ef3722d3034a61a8ab2994c29f9c978903"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -9765,7 +9705,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_tokenstream 0.2.0",
- "syn 2.0.48",
+ "syn 2.0.51",
  "typify-impl",
 ]
 
@@ -9999,7 +9939,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_tokenstream 0.2.0",
- "syn 2.0.48",
+ "syn 2.0.51",
  "usdt-impl 0.5.0",
 ]
 
@@ -10037,7 +9977,7 @@ dependencies = [
  "quote",
  "serde",
  "serde_json",
- "syn 2.0.48",
+ "syn 2.0.51",
  "thiserror",
  "thread-id",
  "version_check",
@@ -10067,7 +10007,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_tokenstream 0.2.0",
- "syn 2.0.48",
+ "syn 2.0.51",
  "usdt-impl 0.5.0",
 ]
 
@@ -10219,7 +10159,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
  "wasm-bindgen-shared",
 ]
 
@@ -10253,7 +10193,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -10266,9 +10206,9 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
 
 [[package]]
 name = "wasm-streams"
-version = "0.3.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7"
+checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129"
 dependencies = [
  "futures-util",
  "js-sys",
@@ -10480,7 +10420,7 @@ dependencies = [
  "installinator-common",
  "ipnetwork",
  "omicron-workspace-hack",
- "progenitor 0.5.0 (git+https://github.com/oxidecomputer/progenitor?branch=main)",
+ "progenitor",
  "regress",
  "reqwest",
  "schemars",
@@ -10804,7 +10744,7 @@ checksum = "56097d5b91d711293a42be9289403896b68654625021732067eac7a4ca388a1f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -10815,7 +10755,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
@@ -10835,7 +10775,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.48",
+ "syn 2.0.51",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index ec2c345edf..ad3d801876 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -314,9 +314,9 @@ prettyplease = { version = "0.2.16", features = ["verbatim"] }
 proc-macro2 = "1.0"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" }
 progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" }
-bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d" }
-propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d" }
-propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d" }
+bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "488a70c644e332dae98447292a95e45074c8107c" }
+propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "488a70c644e332dae98447292a95e45074c8107c" }
+propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "488a70c644e332dae98447292a95e45074c8107c" }
 proptest = "1.4.0"
 quote = "1.0"
 rand = "0.8.5"
@@ -609,6 +609,7 @@ opt-level = 3
 #
 #[patch."https://github.com/oxidecomputer/progenitor"]
 #progenitor = { path = "../progenitor/progenitor" }
+#progenitor-client = { path = "../progenitor/progenitor-client" }
 #[patch."https://github.com/oxidecomputer/typify"]
 #typify = { path = "../typify/typify" }
 
diff --git a/dev-tools/oxlog/src/lib.rs b/dev-tools/oxlog/src/lib.rs
index 589b113928..a4774fabbc 100644
--- a/dev-tools/oxlog/src/lib.rs
+++ b/dev-tools/oxlog/src/lib.rs
@@ -351,8 +351,7 @@ fn load_svc_logs(dir: Utf8PathBuf, logs: &mut BTreeMap<ServiceName, SvcLogs>) {
 
             let is_current = filename.ends_with(".log");
 
-            let svc_logs =
-                logs.entry(svc_name.to_string()).or_insert(SvcLogs::default());
+            let svc_logs = logs.entry(svc_name.to_string()).or_default();
 
             if is_current {
                 svc_logs.current = Some(logfile.clone());
@@ -374,8 +373,7 @@ fn load_extra_logs(
         return;
     };
 
-    let svc_logs =
-        logs.entry(svc_name.to_string()).or_insert(SvcLogs::default());
+    let svc_logs = logs.entry(svc_name.to_string()).or_default();
 
     for entry in entries {
         let Ok(entry) = entry else {
diff --git a/dev-tools/xtask/src/main.rs b/dev-tools/xtask/src/main.rs
index 4ab42736ca..42e6c10d64 100644
--- a/dev-tools/xtask/src/main.rs
+++ b/dev-tools/xtask/src/main.rs
@@ -26,23 +26,35 @@ enum Cmds {
     /// workspace
     CheckWorkspaceDeps,
     /// Run configured clippy checks
-    Clippy,
+    Clippy(ClippyArgs),
+}
+
+#[derive(Parser)]
+struct ClippyArgs {
+    /// Automatically apply lint suggestions.
+    #[clap(long)]
+    fix: bool,
 }
 
 fn main() -> Result<()> {
     let args = Args::parse();
     match args.cmd {
-        Cmds::Clippy => cmd_clippy(),
+        Cmds::Clippy(args) => cmd_clippy(args),
         Cmds::CheckWorkspaceDeps => cmd_check_workspace_deps(),
     }
 }
 
-fn cmd_clippy() -> Result<()> {
+fn cmd_clippy(args: ClippyArgs) -> Result<()> {
     let cargo =
         std::env::var("CARGO").unwrap_or_else(|_| String::from("cargo"));
     let mut command = Command::new(&cargo);
+    command.arg("clippy");
+
+    if args.fix {
+        command.arg("--fix");
+    }
+
     command
-        .arg("clippy")
         // Make sure we check everything.
         .arg("--all-targets")
         .arg("--")
diff --git a/nexus/src/app/background/mod.rs b/nexus/src/app/background/mod.rs
index 27cdddfe15..0065a41a9e 100644
--- a/nexus/src/app/background/mod.rs
+++ b/nexus/src/app/background/mod.rs
@@ -20,6 +20,4 @@ mod region_replacement;
 mod status;
 mod sync_service_zone_nat;
 
-pub use common::Driver;
-pub use common::TaskHandle;
 pub use init::BackgroundTasks;
diff --git a/nexus/src/cidata.rs b/nexus/src/cidata.rs
index 8f776501b6..08e4926af1 100644
--- a/nexus/src/cidata.rs
+++ b/nexus/src/cidata.rs
@@ -6,8 +6,6 @@ use serde::Serialize;
 use std::io::{self, Cursor, Write};
 use uuid::Uuid;
 
-pub use nexus_types::external_api::params::MAX_USER_DATA_BYTES;
-
 pub trait InstanceCiData {
     fn generate_cidata(&self, public_keys: &[String])
         -> Result<Vec<u8>, Error>;
@@ -93,6 +91,8 @@ fn build_vfat(meta_data: &[u8], user_data: &[u8]) -> io::Result<Vec<u8>> {
 
 #[cfg(test)]
 mod tests {
+    use nexus_types::external_api::params::MAX_USER_DATA_BYTES;
+
     /// the fatfs crate has some unfortunate panics if you ask it to do
     /// incredibly stupid things, like format an empty disk or create a
     /// filesystem with an invalid cluster size.
@@ -104,7 +104,7 @@ mod tests {
     /// little further.)
     #[test]
     fn build_vfat_works_with_arbitrarily_sized_input() {
-        let upper = crate::cidata::MAX_USER_DATA_BYTES + 4096;
+        let upper = MAX_USER_DATA_BYTES + 4096;
         // somewhat arbitrarily-chosen prime numbers near 1 KiB and 256 bytes
         for md_size in (0..upper).step_by(1019) {
             for ud_size in (0..upper).step_by(269) {
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index 2e7a87b58b..2e3f4c137b 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -4,5 +4,5 @@
 #
 # We choose a specific toolchain (rather than "stable") for repeatability.  The
 # intent is to keep this up-to-date with recently-released stable Rust.
-channel = "1.74.1"
+channel = "1.76.0"
 profile = "default"
diff --git a/sled-hardware/Cargo.toml b/sled-hardware/Cargo.toml
index 3d1259f46f..50e3cce072 100644
--- a/sled-hardware/Cargo.toml
+++ b/sled-hardware/Cargo.toml
@@ -19,6 +19,7 @@ rand.workspace = true
 schemars.workspace = true
 serde.workspace = true
 slog.workspace = true
+slog-error-chain.workspace = true
 thiserror.workspace = true
 tofino.workspace = true
 tokio.workspace = true
diff --git a/sled-hardware/src/cleanup.rs b/sled-hardware/src/cleanup.rs
index 1a7f8be2f7..f438098792 100644
--- a/sled-hardware/src/cleanup.rs
+++ b/sled-hardware/src/cleanup.rs
@@ -18,6 +18,7 @@ use illumos_utils::ExecutionError;
 use illumos_utils::{execute, PFEXEC};
 use slog::warn;
 use slog::Logger;
+use slog_error_chain::InlineErrorChain;
 use std::process::Command;
 
 pub fn delete_underlay_addresses(log: &Logger) -> Result<(), Error> {
@@ -45,7 +46,17 @@ fn delete_addresses_matching_prefixes(
     let addrobjs = output
         .stdout
         .lines()
-        .flatten()
+        .filter_map(|line| match line {
+            Ok(line) => Some(line),
+            Err(err) => {
+                warn!(
+                    log,
+                    "ipadm show-addr returned line that wasn't valid UTF-8";
+                    InlineErrorChain::new(&err),
+                );
+                None
+            }
+        })
         .collect::<std::collections::HashSet<_>>();
 
     for addrobj in addrobjs {
diff --git a/wicket/src/state/mod.rs b/wicket/src/state/mod.rs
index e20ba1c9c3..d287a153a9 100644
--- a/wicket/src/state/mod.rs
+++ b/wicket/src/state/mod.rs
@@ -12,15 +12,14 @@ mod update;
 
 pub use force_update::ForceUpdateState;
 pub use inventory::{
-    Component, ComponentId, Inventory, ParsableComponentId, PowerState, Sp,
-    ALL_COMPONENT_IDS,
+    Component, ComponentId, Inventory, ParsableComponentId, ALL_COMPONENT_IDS,
 };
 pub use rack::{KnightRiderMode, RackState};
-pub use status::{Liveness, ServiceStatus};
+pub use status::ServiceStatus;
 pub use update::{
     parse_event_report_map, update_component_title,
     CreateClearUpdateStateOptions, CreateStartUpdateOptions, RackUpdateState,
-    UpdateItemState, UpdateRunningState,
+    UpdateItemState,
 };
 
 use serde::{Deserialize, Serialize};
diff --git a/wicket/src/ui/widgets/mod.rs b/wicket/src/ui/widgets/mod.rs
index 9527b68e2f..fd5ed18e3a 100644
--- a/wicket/src/ui/widgets/mod.rs
+++ b/wicket/src/ui/widgets/mod.rs
@@ -16,6 +16,6 @@ pub use animated_logo::{Logo, LogoState, LOGO_HEIGHT, LOGO_WIDTH};
 pub use box_connector::{BoxConnector, BoxConnectorKind};
 pub use fade::Fade;
 pub use ignition::IgnitionPopup;
-pub use popup::{ButtonText, Popup, PopupBuilder, PopupScrollOffset};
+pub use popup::{ButtonText, PopupBuilder, PopupScrollOffset};
 pub use rack::Rack;
 pub use status_view::StatusView;
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index ebe683e51a..01b7e440ae 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -85,11 +85,11 @@ rand_chacha = { version = "0.3.1", default-features = false, features = ["std"]
 regex = { version = "1.10.3" }
 regex-automata = { version = "0.4.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
 regex-syntax = { version = "0.8.2" }
-reqwest = { version = "0.11.22", features = ["blocking", "json", "rustls-tls", "stream"] }
+reqwest = { version = "0.11.24", features = ["blocking", "json", "rustls-tls", "stream"] }
 ring = { version = "0.17.8", features = ["std"] }
 schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid1"] }
 semver = { version = "1.0.22", features = ["serde"] }
-serde = { version = "1.0.196", features = ["alloc", "derive", "rc"] }
+serde = { version = "1.0.197", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1.0.114", features = ["raw_value", "unbounded_depth"] }
 sha2 = { version = "0.10.8", features = ["oid"] }
 similar = { version = "2.3.0", features = ["inline", "unicode"] }
@@ -97,10 +97,9 @@ slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "rele
 socket2 = { version = "0.5.5", default-features = false, features = ["all"] }
 spin = { version = "0.9.8" }
 string_cache = { version = "0.8.7" }
-strum = { version = "0.25.0", features = ["derive"] }
 subtle = { version = "2.5.0" }
 syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] }
-syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.48", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
+syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.51", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
 time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] }
 tokio = { version = "1.36.0", features = ["full", "test-util"] }
 tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] }
@@ -192,11 +191,11 @@ rand_chacha = { version = "0.3.1", default-features = false, features = ["std"]
 regex = { version = "1.10.3" }
 regex-automata = { version = "0.4.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
 regex-syntax = { version = "0.8.2" }
-reqwest = { version = "0.11.22", features = ["blocking", "json", "rustls-tls", "stream"] }
+reqwest = { version = "0.11.24", features = ["blocking", "json", "rustls-tls", "stream"] }
 ring = { version = "0.17.8", features = ["std"] }
 schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid1"] }
 semver = { version = "1.0.22", features = ["serde"] }
-serde = { version = "1.0.196", features = ["alloc", "derive", "rc"] }
+serde = { version = "1.0.197", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1.0.114", features = ["raw_value", "unbounded_depth"] }
 sha2 = { version = "0.10.8", features = ["oid"] }
 similar = { version = "2.3.0", features = ["inline", "unicode"] }
@@ -204,10 +203,9 @@ slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "rele
 socket2 = { version = "0.5.5", default-features = false, features = ["all"] }
 spin = { version = "0.9.8" }
 string_cache = { version = "0.8.7" }
-strum = { version = "0.25.0", features = ["derive"] }
 subtle = { version = "2.5.0" }
 syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] }
-syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.48", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
+syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.51", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
 time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] }
 time-macros = { version = "0.2.13", default-features = false, features = ["formatting", "parsing"] }
 tokio = { version = "1.36.0", features = ["full", "test-util"] }

From 3498c3c35edfd67d3efee65525a526181b6d3825 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Wed, 28 Feb 2024 22:38:22 -0800
Subject: [PATCH 050/157] chore(deps): update rust crate rayon to 1.9 (#5159)

---
 Cargo.lock | 8 ++++----
 Cargo.toml | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2c1e4acd50..0d1e149700 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6864,9 +6864,9 @@ dependencies = [
 
 [[package]]
 name = "rayon"
-version = "1.8.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1"
+checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd"
 dependencies = [
  "either",
  "rayon-core",
@@ -6874,9 +6874,9 @@ dependencies = [
 
 [[package]]
 name = "rayon-core"
-version = "1.12.0"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed"
+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
 dependencies = [
  "crossbeam-deque",
  "crossbeam-utils",
diff --git a/Cargo.toml b/Cargo.toml
index ad3d801876..97300bc0d2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -321,7 +321,7 @@ proptest = "1.4.0"
 quote = "1.0"
 rand = "0.8.5"
 ratatui = "0.26.1"
-rayon = "1.8"
+rayon = "1.9"
 rcgen = "0.12.1"
 reedline = "0.29.0"
 ref-cast = "1.0"

From e7a404e15c4d0df73baba7de9b83a7923c7ba56a Mon Sep 17 00:00:00 2001
From: liffy <629075+lifning@users.noreply.github.com>
Date: Thu, 29 Feb 2024 02:09:41 -0800
Subject: [PATCH 051/157] Push hardcoded PROPOLIS_PORT out of all but the
 instance creation/migration sagas (#5138)

Adds a database column for Nexus to track `propolis_port` in addition to
the `propolis_ip`, such that tests using sled-agent-sim don't collide.

Restrict hardcoded PROPOLIS_PORT to simply being the default in sagas.
It remains 12400 in production, since there's no reason for it not to
be, but for scenarios wherein each "propolis-server" API isn't
necessarily served from an illumos zone with its own IP address, we can
let the port be automatically assigned.

(addressing issue #5051)
---
 common/src/address.rs                      |  2 --
 nexus/db-model/src/schema.rs               |  3 ++-
 nexus/db-model/src/vmm.rs                  |  6 ++++++
 nexus/db-queries/src/db/datastore/vmm.rs   | 14 ++++++++++----
 nexus/src/app/instance.rs                  |  5 ++---
 nexus/src/app/sagas/instance_common.rs     |  4 ++++
 nexus/src/app/sagas/instance_migrate.rs    |  7 ++++---
 nexus/tests/integration_tests/instances.rs |  8 ++++----
 schema/crdb/38.0.0/up.sql                  | 11 +++++++++++
 schema/crdb/dbinit.sql                     |  5 +++--
 sled-agent/src/sim/sled_agent.rs           | 22 ++++++++++------------
 11 files changed, 56 insertions(+), 31 deletions(-)
 create mode 100644 schema/crdb/38.0.0/up.sql

diff --git a/common/src/address.rs b/common/src/address.rs
index 152fb9319e..3a470c189b 100644
--- a/common/src/address.rs
+++ b/common/src/address.rs
@@ -44,8 +44,6 @@ pub const DNS_PORT: u16 = 53;
 pub const DNS_HTTP_PORT: u16 = 5353;
 pub const SLED_AGENT_PORT: u16 = 12345;
 
-/// The port propolis-server listens on inside the propolis zone.
-pub const PROPOLIS_PORT: u16 = 12400;
 pub const COCKROACH_PORT: u16 = 32221;
 pub const CRUCIBLE_PORT: u16 = 32345;
 pub const CLICKHOUSE_PORT: u16 = 8123;
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index d8344c2258..55d3e9b43f 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(37, 0, 1);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(38, 0, 0);
 
 table! {
     disk (id) {
@@ -380,6 +380,7 @@ table! {
         instance_id -> Uuid,
         sled_id -> Uuid,
         propolis_ip -> Inet,
+        propolis_port -> Int4,
         state -> crate::InstanceStateEnum,
         time_state_updated -> Timestamptz,
         state_generation -> Int8,
diff --git a/nexus/db-model/src/vmm.rs b/nexus/db-model/src/vmm.rs
index fe1158d5bb..ca3be120d4 100644
--- a/nexus/db-model/src/vmm.rs
+++ b/nexus/db-model/src/vmm.rs
@@ -14,6 +14,7 @@
 
 use super::{Generation, InstanceState};
 use crate::schema::vmm;
+use crate::SqlU16;
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 use uuid::Uuid;
@@ -43,6 +44,9 @@ pub struct Vmm {
     /// The IP address at which this VMM is serving the Propolis server API.
     pub propolis_ip: ipnetwork::IpNetwork,
 
+    /// The socket port on which this VMM is serving the Propolis server API.
+    pub propolis_port: SqlU16,
+
     /// Runtime state for the VMM.
     #[diesel(embed)]
     pub runtime: VmmRuntimeState,
@@ -61,6 +65,7 @@ impl Vmm {
         instance_id: Uuid,
         sled_id: Uuid,
         propolis_ip: ipnetwork::IpNetwork,
+        propolis_port: u16,
         initial_state: VmmInitialState,
     ) -> Self {
         use omicron_common::api::external::InstanceState as ApiInstanceState;
@@ -78,6 +83,7 @@ impl Vmm {
             instance_id,
             sled_id,
             propolis_ip,
+            propolis_port: SqlU16(propolis_port),
             runtime: VmmRuntimeState {
                 state: InstanceState::new(api_state),
                 time_state_updated: now,
diff --git a/nexus/db-queries/src/db/datastore/vmm.rs b/nexus/db-queries/src/db/datastore/vmm.rs
index 18afde84f0..b9bfd7697e 100644
--- a/nexus/db-queries/src/db/datastore/vmm.rs
+++ b/nexus/db-queries/src/db/datastore/vmm.rs
@@ -23,6 +23,7 @@ use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::LookupType;
 use omicron_common::api::external::ResourceType;
 use omicron_common::api::external::UpdateResult;
+use std::net::SocketAddr;
 use uuid::Uuid;
 
 impl DataStore {
@@ -134,7 +135,7 @@ impl DataStore {
         Ok(updated)
     }
 
-    /// Forcibly overwrites the Propolis IP in the supplied VMM's record with
+    /// Forcibly overwrites the Propolis IP/Port in the supplied VMM's record with
     /// the supplied Propolis IP.
     ///
     /// This is used in tests to overwrite the IP for a VMM that is backed by a
@@ -142,15 +143,20 @@ impl DataStore {
     /// allocated by the instance start procedure. (Unfortunately, this can't be
     /// marked #[cfg(test)] because the integration tests require this
     /// functionality.)
-    pub async fn vmm_overwrite_ip_for_test(
+    pub async fn vmm_overwrite_addr_for_test(
         &self,
         opctx: &OpContext,
         vmm_id: &Uuid,
-        new_ip: ipnetwork::IpNetwork,
+        new_addr: SocketAddr,
     ) -> UpdateResult<Vmm> {
+        let new_ip = ipnetwork::IpNetwork::from(new_addr.ip());
+        let new_port = new_addr.port();
         let vmm = diesel::update(dsl::vmm)
             .filter(dsl::id.eq(*vmm_id))
-            .set(dsl::propolis_ip.eq(new_ip))
+            .set((
+                dsl::propolis_ip.eq(new_ip),
+                dsl::propolis_port.eq(new_port as i32),
+            ))
             .returning(Vmm::as_returning())
             .get_result_async(&*self.pool_connection_authorized(opctx).await?)
             .await
diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs
index 4b52b597ba..88a427af6a 100644
--- a/nexus/src/app/instance.rs
+++ b/nexus/src/app/instance.rs
@@ -29,7 +29,6 @@ use nexus_db_queries::db::identity::Resource;
 use nexus_db_queries::db::lookup;
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_types::external_api::views;
-use omicron_common::address::PROPOLIS_PORT;
 use omicron_common::api::external::http_pagination::PaginatedBy;
 use omicron_common::api::external::ByteCount;
 use omicron_common::api::external::CreateResult;
@@ -1223,7 +1222,7 @@ impl super::Nexus {
                     propolis_id: *propolis_id,
                     propolis_addr: SocketAddr::new(
                         initial_vmm.propolis_ip.ip(),
-                        PROPOLIS_PORT,
+                        initial_vmm.propolis_port.into(),
                     )
                     .to_string(),
                 },
@@ -1762,7 +1761,7 @@ impl super::Nexus {
                 | InstanceState::Rebooting
                 | InstanceState::Migrating
                 | InstanceState::Repairing => {
-                    Ok(SocketAddr::new(vmm.propolis_ip.ip(), PROPOLIS_PORT))
+                    Ok(SocketAddr::new(vmm.propolis_ip.ip(), vmm.propolis_port.into()))
                 }
                 InstanceState::Creating
                 | InstanceState::Starting
diff --git a/nexus/src/app/sagas/instance_common.rs b/nexus/src/app/sagas/instance_common.rs
index 445abd5daf..e915c026dd 100644
--- a/nexus/src/app/sagas/instance_common.rs
+++ b/nexus/src/app/sagas/instance_common.rs
@@ -24,6 +24,9 @@ use uuid::Uuid;
 
 use super::NexusActionContext;
 
+/// The port propolis-server listens on inside the propolis zone.
+const DEFAULT_PROPOLIS_PORT: u16 = 12400;
+
 /// Reserves resources for a new VMM whose instance has `ncpus` guest logical
 /// processors and `guest_memory` bytes of guest RAM. The selected sled is
 /// random within the set of sleds allowed by the supplied `constraints`.
@@ -98,6 +101,7 @@ pub async fn create_and_insert_vmm_record(
         instance_id,
         sled_id,
         IpAddr::V6(propolis_ip).into(),
+        DEFAULT_PROPOLIS_PORT,
         initial_state,
     );
 
diff --git a/nexus/src/app/sagas/instance_migrate.rs b/nexus/src/app/sagas/instance_migrate.rs
index ff3ff66e78..da3b3e93ea 100644
--- a/nexus/src/app/sagas/instance_migrate.rs
+++ b/nexus/src/app/sagas/instance_migrate.rs
@@ -12,7 +12,6 @@ use crate::app::sagas::{
 use crate::external_api::params;
 use nexus_db_queries::db::{identity::Resource, lookup::LookupPath};
 use nexus_db_queries::{authn, authz, db};
-use omicron_common::address::PROPOLIS_PORT;
 use serde::Deserialize;
 use serde::Serialize;
 use sled_agent_client::types::{
@@ -424,8 +423,10 @@ async fn sim_instance_migrate(
     let db_instance =
         sagactx.lookup::<db::model::Instance>("set_migration_ids")?;
 
-    let src_vmm_addr =
-        SocketAddr::new(params.src_vmm.propolis_ip.ip(), PROPOLIS_PORT);
+    let src_vmm_addr = SocketAddr::new(
+        params.src_vmm.propolis_ip.ip(),
+        params.src_vmm.propolis_port.into(),
+    );
 
     let src_propolis_id = db_instance.runtime().propolis_id.unwrap();
     let dst_vmm = sagactx.lookup::<db::model::Vmm>("dst_vmm_record")?;
diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs
index 09f91a2288..0df2b83008 100644
--- a/nexus/tests/integration_tests/instances.rs
+++ b/nexus/tests/integration_tests/instances.rs
@@ -3762,7 +3762,7 @@ async fn test_instance_serial(cptestctx: &ControlPlaneTestContext) {
     let nexus = &apictx.nexus;
     let instance_name = "kris-picks";
 
-    cptestctx
+    let propolis_addr = cptestctx
         .sled_agent
         .sled_agent
         .start_local_mock_propolis_server(&cptestctx.logctx.log)
@@ -3819,12 +3819,12 @@ async fn test_instance_serial(cptestctx: &ControlPlaneTestContext) {
         .runtime()
         .propolis_id
         .expect("running instance should have vmm");
-    let localhost = std::net::IpAddr::V6(std::net::Ipv6Addr::LOCALHOST);
     let updated_vmm = datastore
-        .vmm_overwrite_ip_for_test(&opctx, &propolis_id, localhost.into())
+        .vmm_overwrite_addr_for_test(&opctx, &propolis_id, propolis_addr)
         .await
         .unwrap();
-    assert_eq!(updated_vmm.propolis_ip.ip(), localhost);
+    assert_eq!(updated_vmm.propolis_ip.ip(), propolis_addr.ip());
+    assert_eq!(updated_vmm.propolis_port.0, propolis_addr.port());
 
     // Query serial output history endpoint
     // This is the first line of output generated by the mock propolis-server.
diff --git a/schema/crdb/38.0.0/up.sql b/schema/crdb/38.0.0/up.sql
new file mode 100644
index 0000000000..725a9e2c90
--- /dev/null
+++ b/schema/crdb/38.0.0/up.sql
@@ -0,0 +1,11 @@
+-- 12400 was the hardcoded PROPOLIS_PORT prior to this addition; default to
+-- that for all existing instances when creating the column.
+-- in production, this value will always end up still being 12400.
+-- however, nexus' testability in scenarios where each "propolis-server" API
+-- isn't necessarily served from an illumos zone with its own IP address, but
+-- rather in a sled-agent-sim thread during a cargo nextest run? useful to
+-- allow some dynamic range there to avoid flakes caused by port collisions.
+ALTER TABLE omicron.public.vmm
+ADD COLUMN IF NOT EXISTS propolis_port INT4 NOT NULL
+    CHECK (propolis_port BETWEEN 0 AND 65535)
+    DEFAULT 12400;
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 837be42c35..40a6fd463f 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -3310,7 +3310,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.vmm (
     time_state_updated TIMESTAMPTZ NOT NULL,
     state_generation INT NOT NULL,
     sled_id UUID NOT NULL,
-    propolis_ip INET NOT NULL
+    propolis_ip INET NOT NULL,
+    propolis_port INT4 NOT NULL CHECK (propolis_port BETWEEN 0 AND 65535) DEFAULT 12400
 );
 
 /*
@@ -3551,7 +3552,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '37.0.1', NULL)
+    ( TRUE, NOW(), NOW(), '38.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;
diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs
index d4c92fca51..27a06e4617 100644
--- a/sled-agent/src/sim/sled_agent.rs
+++ b/sled-agent/src/sim/sled_agent.rs
@@ -27,7 +27,6 @@ use illumos_utils::opte::params::{
     DeleteVirtualNetworkInterfaceHost, SetVirtualNetworkInterfaceHost,
 };
 use nexus_client::types::PhysicalDiskKind;
-use omicron_common::address::PROPOLIS_PORT;
 use omicron_common::api::external::{
     ByteCount, DiskState, Error, Generation, ResourceType,
 };
@@ -684,14 +683,15 @@ impl SledAgent {
     }
 
     /// Used for integration tests that require a component to talk to a
-    /// mocked propolis-server API.
-    // TODO: fix schemas so propolis-server's port isn't hardcoded in nexus
-    // such that we can run more than one of these.
-    // (this is only needed by test_instance_serial at present)
+    /// mocked propolis-server API. Returns the socket on which the dropshot
+    /// service is listening, which *must* be patched into Nexus with
+    /// `nexus_db_queries::db::datastore::vmm_overwrite_addr_for_test` after
+    /// the instance creation saga if functionality touching propolis-server
+    /// is to be tested (e.g. serial console connection).
     pub async fn start_local_mock_propolis_server(
         &self,
         log: &Logger,
-    ) -> Result<(), Error> {
+    ) -> Result<SocketAddr, Error> {
         let mut mock_lock = self.mock_propolis.lock().await;
         if mock_lock.is_some() {
             return Err(Error::ObjectAlreadyExists {
@@ -700,7 +700,7 @@ impl SledAgent {
             });
         }
         let propolis_bind_address =
-            SocketAddr::new(Ipv6Addr::LOCALHOST.into(), PROPOLIS_PORT);
+            SocketAddr::new(Ipv6Addr::LOCALHOST.into(), 0);
         let dropshot_config = dropshot::ConfigDropshot {
             bind_address: propolis_bind_address,
             ..Default::default()
@@ -721,12 +721,10 @@ impl SledAgent {
             Error::unavail(&format!("initializing propolis-server: {}", error))
         })?
         .start();
-        let client = propolis_client::Client::new(&format!(
-            "http://{}",
-            srv.local_addr()
-        ));
+        let addr = srv.local_addr();
+        let client = propolis_client::Client::new(&format!("http://{}", addr));
         *mock_lock = Some((srv, client));
-        Ok(())
+        Ok(addr)
     }
 
     pub fn inventory(&self, addr: SocketAddr) -> anyhow::Result<Inventory> {

From 1bc50078a589560ad771665435856685fab89723 Mon Sep 17 00:00:00 2001
From: David Crespo <david-crespo@users.noreply.github.com>
Date: Thu, 29 Feb 2024 04:37:37 -0600
Subject: [PATCH 052/157] Bump web console (fix rounding bug on cap & util)
 (#5165)

https://github.com/oxidecomputer/console/compare/e991ec5c...25fb506d

* [25fb506d](https://github.com/oxidecomputer/console/commit/25fb506d)
hard code en-US locale in NumberFormat step of round()
* [0ba26501](https://github.com/oxidecomputer/console/commit/0ba26501)
oxidecomputer/console#2011
* [bce7285e](https://github.com/oxidecomputer/console/commit/bce7285e)
oxidecomputer/console#2009
* [6947e026](https://github.com/oxidecomputer/console/commit/6947e026)
fall back to EmptyCell on empty description in properties table (closes
oxidecomputer/console#2007)
* [7e60dfaa](https://github.com/oxidecomputer/console/commit/7e60dfaa)
oxidecomputer/console#2008
* [629f1c29](https://github.com/oxidecomputer/console/commit/629f1c29)
bump dax in bump-omicron script to fix quote escaping bug
---
 tools/console_version | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/console_version b/tools/console_version
index ee7f619b53..f5534b5c50 100644
--- a/tools/console_version
+++ b/tools/console_version
@@ -1,2 +1,2 @@
-COMMIT="e991ec5cf148d83f135b6a692dca9b8df05acef0"
-SHA2="18d53e4485e63d9e2273a889e70785648052e8a231df89f3bcbc2ed01a0eeeb1"
+COMMIT="25fb506d0527f5508dc90b118672c0bf8ce14dd3"
+SHA2="1083f693c66c275d0cf76e120aa923a51c3188dd9a7996eb51dc0fe4f611507a"

From 7daf5e1efe1054119219a164b3eba64515363c69 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Thu, 29 Feb 2024 16:14:58 +0000
Subject: [PATCH 053/157] chore(deps): update taiki-e/install-action digest to
 45dfafe (#5169)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [taiki-e/install-action](https://togithub.com/taiki-e/install-action)
| action | digest | [`1d776b1` ->
`45dfafe`](https://togithub.com/taiki-e/install-action/compare/1d776b1...45dfafe)
|

---

### Configuration

📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone
America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone
America/Los_Angeles.

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update
again.

---

- [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check
this box

---

This PR has been generated by [Renovate
Bot](https://togithub.com/renovatebot/renovate).

<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNy4yMjAuMiIsInVwZGF0ZWRJblZlciI6IjM3LjIyMC4yIiwidGFyZ2V0QnJhbmNoIjoibWFpbiJ9-->

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index 0f942e7d76..49914f36db 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@1d776b18af134fca43744080130085d256938196 # v2
+        uses: taiki-e/install-action@45dfafe7696c1865602c71c94d46a16a45f8e88b # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From 10ada767920e7c8d988e43a613feab49d1157fc3 Mon Sep 17 00:00:00 2001
From: Dan Cross <crossd@gmail.com>
Date: Thu, 29 Feb 2024 11:44:58 -0500
Subject: [PATCH 054/157] env.sh: prefer `readlink` to `cd && echo ${PWD}`
 (#5162)

A trivial change to make this script more robust for those of us who
alias `cd` to something.
---
 env.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/env.sh b/env.sh
index 483a89f597..74f3d1caf4 100644
--- a/env.sh
+++ b/env.sh
@@ -5,7 +5,7 @@
 # See also: ./.envrc
 
 set -o xtrace
-OMICRON_WS="$(cd $(dirname "${BASH_SOURCE[0]}") && echo $PWD)"
+OMICRON_WS="$(readlink -f $(dirname "${BASH_SOURCE[0]}"))"
 export PATH="$OMICRON_WS/out/cockroachdb/bin:$PATH"
 export PATH="$OMICRON_WS/out/clickhouse:$PATH"
 export PATH="$OMICRON_WS/out/dendrite-stub/bin:$PATH"

From c42b50531cf408d20385a1f2275088ae8e5dfbeb Mon Sep 17 00:00:00 2001
From: Laura Abbott <labbott@users.noreply.github.com>
Date: Thu, 29 Feb 2024 12:35:02 -0500
Subject: [PATCH 055/157] SP v1.0.9 (#5170)

---
 tools/hubris_checksums | 17 +++++++++--------
 tools/hubris_version   |  2 +-
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/tools/hubris_checksums b/tools/hubris_checksums
index dca8ea0ab6..fb1c8c1930 100644
--- a/tools/hubris_checksums
+++ b/tools/hubris_checksums
@@ -1,8 +1,9 @@
-28f432735a15f40101c133e4e9974d1681a33bb7b3386ccbe15b465b613f4826  build-gimlet-c-image-default-v1.0.8.zip
-db927999398f0723d5d614db78a5abb4a1d515c711ffba944477bdac10c48907  build-gimlet-d-image-default-v1.0.8.zip
-629a53b5d9d4bf3d410687d0ecedf4837a54233ce62b6223d209494777cc7ebc  build-gimlet-e-image-default-v1.0.8.zip
-e3c2a243257929a65de638f3be425370f084aeeefafbd1773d01ee71cf0b8ea7  build-gimlet-f-image-default-v1.0.8.zip
-556595b42d05508ebfdac9dd71b38fe9b72e0cb30f6aa4be626c02141e375a71  build-psc-b-image-default-v1.0.8.zip
-39fbf92cbc935b4eaecb81a9768357828cf3e79b5c74d36c9a655ae9023cc50c  build-psc-c-image-default-v1.0.8.zip
-4225dff721b034fe7cf1dc26277557e1f15f2710014dd46dfa7c92ff04c7e054  build-sidecar-b-image-default-v1.0.8.zip
-ae8f12d7b66d0bcc372dd79abf515255f6ca97bb0339c570a058684e04e12cf8  build-sidecar-c-image-default-v1.0.8.zip
+4ebca97672fafd3b2716c17f00c8a93f531c1ccb44137c756c77c13383b203da  build-gimlet-c-image-default-v1.0.9.zip
+a945dbcd4b1825e95db32bbf2fbdc56013501396dcdc5ebe6ded361decfcb1f6  build-gimlet-d-image-default-v1.0.9.zip
+527c4fe9488c5834f0fd68fa59756ab01eb0c36d725ba8b8ade1375e2dcb4c3e  build-gimlet-e-image-default-v1.0.9.zip
+69d09a1367b5cfecf43c5b733f84adcb0e4f73b49e062f1c4574b769fc2f1de9  build-gimlet-f-image-default-v1.0.9.zip
+0fa33a057dec7301280338cf5a80a5198b5d95475dba350a807979f1a53ce5c7  build-psc-b-image-default-v1.0.9.zip
+88e7aa91fe4f976753ccfc8ba9a57a0521c54d9ef35504cf16e7b1ce78c87465  build-psc-c-image-default-v1.0.9.zip
+cfbf21fef84b38c2a5ca3c0b45703c20c2605c6bf5d599b53b26e3a2d09d756d  build-sidecar-b-image-default-v1.0.9.zip
+c98b89993f1a9b5219bbabf06897db2f92d60f6a3afb4374cb8e6e79d57fdfe4  build-sidecar-c-image-default-v1.0.9.zip
+
diff --git a/tools/hubris_version b/tools/hubris_version
index 4ee8ac61fe..37a180a853 100644
--- a/tools/hubris_version
+++ b/tools/hubris_version
@@ -1 +1 @@
-TAGS=(gimlet-v1.0.8 psc-v1.0.8 sidecar-v1.0.8)
+TAGS=(gimlet-v1.0.9 psc-v1.0.9 sidecar-v1.0.9)

From 55a0760a77a37333c51594229e06c9b598349c90 Mon Sep 17 00:00:00 2001
From: David Pacheco <dap@oxidecomputer.com>
Date: Thu, 29 Feb 2024 10:50:05 -0800
Subject: [PATCH 056/157] rename reconfigurator packages (#5163)

---
 Cargo.lock                                    | 112 +++++++++---------
 Cargo.toml                                    |  12 +-
 nexus/Cargo.toml                              |   4 +-
 .../tests/config.test.toml                    |   1 -
 nexus/db-queries/Cargo.toml                   |   2 +-
 .../db-queries/src/db/datastore/deployment.rs |   4 +-
 nexus/reconfigurator/README.md                |  13 ++
 .../execution}/Cargo.toml                     |   4 +-
 .../execution}/build.rs                       |   0
 .../execution}/src/datasets.rs                |   0
 .../execution}/src/dns.rs                     |   2 +-
 .../execution}/src/lib.rs                     |   2 +-
 .../execution}/src/omicron_zones.rs           |   0
 .../execution}/src/resource_allocation.rs     |   0
 .../execution/tests/config.test.toml          |   1 +
 .../planning}/Cargo.toml                      |   2 +-
 .../planning}/src/blueprint_builder.rs        |   0
 .../planning}/src/ip_allocator.rs             |   0
 .../planning}/src/lib.rs                      |   0
 .../planning}/src/planner.rs                  |   0
 .../src/app/background/blueprint_execution.rs |   4 +-
 nexus/src/app/deployment.rs                   |   4 +-
 nexus/types/src/deployment.rs                 |  13 +-
 23 files changed, 97 insertions(+), 83 deletions(-)
 delete mode 120000 nexus/blueprint-execution/tests/config.test.toml
 create mode 100644 nexus/reconfigurator/README.md
 rename nexus/{blueprint-execution => reconfigurator/execution}/Cargo.toml (92%)
 rename nexus/{blueprint-execution => reconfigurator/execution}/build.rs (100%)
 rename nexus/{blueprint-execution => reconfigurator/execution}/src/datasets.rs (100%)
 rename nexus/{blueprint-execution => reconfigurator/execution}/src/dns.rs (99%)
 rename nexus/{blueprint-execution => reconfigurator/execution}/src/lib.rs (97%)
 rename nexus/{blueprint-execution => reconfigurator/execution}/src/omicron_zones.rs (100%)
 rename nexus/{blueprint-execution => reconfigurator/execution}/src/resource_allocation.rs (100%)
 create mode 120000 nexus/reconfigurator/execution/tests/config.test.toml
 rename nexus/{deployment => reconfigurator/planning}/Cargo.toml (92%)
 rename nexus/{deployment => reconfigurator/planning}/src/blueprint_builder.rs (100%)
 rename nexus/{deployment => reconfigurator/planning}/src/ip_allocator.rs (100%)
 rename nexus/{deployment => reconfigurator/planning}/src/lib.rs (100%)
 rename nexus/{deployment => reconfigurator/planning}/src/planner.rs (100%)

diff --git a/Cargo.lock b/Cargo.lock
index 0d1e149700..c3fc5d86cf 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4262,39 +4262,6 @@ dependencies = [
  "rustc_version 0.1.7",
 ]
 
-[[package]]
-name = "nexus-blueprint-execution"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "chrono",
- "dns-service-client",
- "futures",
- "httptest",
- "illumos-utils",
- "internal-dns",
- "ipnet",
- "nexus-db-model",
- "nexus-db-queries",
- "nexus-deployment",
- "nexus-inventory",
- "nexus-test-utils",
- "nexus-test-utils-macros",
- "nexus-types",
- "omicron-common",
- "omicron-nexus",
- "omicron-rpaths",
- "omicron-test-utils",
- "omicron-workspace-hack",
- "pq-sys",
- "reqwest",
- "sled-agent-client",
- "slog",
- "slog-error-chain",
- "tokio",
- "uuid",
-]
-
 [[package]]
 name = "nexus-client"
 version = "0.1.0"
@@ -4386,8 +4353,8 @@ dependencies = [
  "macaddr",
  "newtype_derive",
  "nexus-db-model",
- "nexus-deployment",
  "nexus-inventory",
+ "nexus-reconfigurator-planning",
  "nexus-test-utils",
  "nexus-types",
  "omicron-common",
@@ -4444,26 +4411,6 @@ dependencies = [
  "serde_json",
 ]
 
-[[package]]
-name = "nexus-deployment"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "chrono",
- "internal-dns",
- "ipnet",
- "ipnetwork",
- "nexus-inventory",
- "nexus-types",
- "omicron-common",
- "omicron-test-utils",
- "omicron-workspace-hack",
- "sled-agent-client",
- "slog",
- "thiserror",
- "uuid",
-]
-
 [[package]]
 name = "nexus-inventory"
 version = "0.1.0"
@@ -4501,6 +4448,59 @@ dependencies = [
  "syn 2.0.51",
 ]
 
+[[package]]
+name = "nexus-reconfigurator-execution"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "chrono",
+ "dns-service-client",
+ "futures",
+ "httptest",
+ "illumos-utils",
+ "internal-dns",
+ "ipnet",
+ "nexus-db-model",
+ "nexus-db-queries",
+ "nexus-inventory",
+ "nexus-reconfigurator-planning",
+ "nexus-test-utils",
+ "nexus-test-utils-macros",
+ "nexus-types",
+ "omicron-common",
+ "omicron-nexus",
+ "omicron-rpaths",
+ "omicron-test-utils",
+ "omicron-workspace-hack",
+ "pq-sys",
+ "reqwest",
+ "sled-agent-client",
+ "slog",
+ "slog-error-chain",
+ "tokio",
+ "uuid",
+]
+
+[[package]]
+name = "nexus-reconfigurator-planning"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "chrono",
+ "internal-dns",
+ "ipnet",
+ "ipnetwork",
+ "nexus-inventory",
+ "nexus-types",
+ "omicron-common",
+ "omicron-test-utils",
+ "omicron-workspace-hack",
+ "sled-agent-client",
+ "slog",
+ "thiserror",
+ "uuid",
+]
+
 [[package]]
 name = "nexus-test-interface"
 version = "0.1.0"
@@ -5031,12 +5031,12 @@ dependencies = [
  "macaddr",
  "mg-admin-client",
  "mime_guess",
- "nexus-blueprint-execution",
  "nexus-db-model",
  "nexus-db-queries",
  "nexus-defaults",
- "nexus-deployment",
  "nexus-inventory",
+ "nexus-reconfigurator-execution",
+ "nexus-reconfigurator-planning",
  "nexus-test-interface",
  "nexus-test-utils",
  "nexus-test-utils-macros",
diff --git a/Cargo.toml b/Cargo.toml
index 97300bc0d2..e19ed35c32 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -37,14 +37,14 @@ members = [
     "key-manager",
     "nexus",
     "nexus/authz-macros",
-    "nexus/blueprint-execution",
     "nexus/db-macros",
     "nexus/db-model",
     "nexus/db-queries",
     "nexus/defaults",
-    "nexus/deployment",
     "nexus/inventory",
     "nexus/macros-common",
+    "nexus/reconfigurator/execution",
+    "nexus/reconfigurator/planning",
     "nexus/test-interface",
     "nexus/test-utils-macros",
     "nexus/test-utils",
@@ -114,14 +114,14 @@ default-members = [
     "key-manager",
     "nexus",
     "nexus/authz-macros",
-    "nexus/blueprint-execution",
     "nexus/macros-common",
     "nexus/db-macros",
     "nexus/db-model",
     "nexus/db-queries",
     "nexus/defaults",
-    "nexus/deployment",
     "nexus/inventory",
+    "nexus/reconfigurator/execution",
+    "nexus/reconfigurator/planning",
     "nexus/types",
     "oximeter/collector",
     "oximeter/db",
@@ -253,14 +253,14 @@ mockall = "0.12"
 newtype_derive = "0.1.6"
 mg-admin-client = { path = "clients/mg-admin-client" }
 multimap = "0.10.0"
-nexus-blueprint-execution = { path = "nexus/blueprint-execution" }
 nexus-client = { path = "clients/nexus-client" }
 nexus-db-model = { path = "nexus/db-model" }
 nexus-db-queries = { path = "nexus/db-queries" }
 nexus-defaults = { path = "nexus/defaults" }
-nexus-deployment = { path = "nexus/deployment" }
 nexus-inventory = { path = "nexus/inventory" }
 nexus-macros-common = { path = "nexus/macros-common" }
+nexus-reconfigurator-execution = { path = "nexus/reconfigurator/execution" }
+nexus-reconfigurator-planning = { path = "nexus/reconfigurator/planning" }
 omicron-certificates = { path = "certificates" }
 omicron-passwords = { path = "passwords" }
 omicron-workspace-hack = "0.1.0"
diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml
index 6e9f2f135d..581e16e84d 100644
--- a/nexus/Cargo.toml
+++ b/nexus/Cargo.toml
@@ -77,12 +77,12 @@ tough.workspace = true
 trust-dns-resolver.workspace = true
 uuid.workspace = true
 
-nexus-blueprint-execution.workspace = true
 nexus-defaults.workspace = true
 nexus-db-model.workspace = true
 nexus-db-queries.workspace = true
-nexus-deployment.workspace = true
 nexus-inventory.workspace = true
+nexus-reconfigurator-execution.workspace = true
+nexus-reconfigurator-planning.workspace = true
 nexus-types.workspace = true
 omicron-common.workspace = true
 omicron-passwords.workspace = true
diff --git a/nexus/blueprint-execution/tests/config.test.toml b/nexus/blueprint-execution/tests/config.test.toml
deleted file mode 120000
index 52f00171fd..0000000000
--- a/nexus/blueprint-execution/tests/config.test.toml
+++ /dev/null
@@ -1 +0,0 @@
-../../tests/config.test.toml
\ No newline at end of file
diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml
index 539a913476..9d0c22c373 100644
--- a/nexus/db-queries/Cargo.toml
+++ b/nexus/db-queries/Cargo.toml
@@ -69,8 +69,8 @@ gateway-client.workspace = true
 illumos-utils.workspace = true
 internal-dns.workspace = true
 itertools.workspace = true
-nexus-deployment.workspace = true
 nexus-inventory.workspace = true
+nexus-reconfigurator-planning.workspace = true
 nexus-test-utils.workspace = true
 omicron-sled-agent.workspace = true
 omicron-test-utils.workspace = true
diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs
index 00a75a21da..020916928d 100644
--- a/nexus/db-queries/src/db/datastore/deployment.rs
+++ b/nexus/db-queries/src/db/datastore/deployment.rs
@@ -1055,9 +1055,9 @@ impl RunQueryDsl<DbConnection> for InsertTargetQuery {}
 mod tests {
     use super::*;
     use crate::db::datastore::test_utils::datastore_test;
-    use nexus_deployment::blueprint_builder::BlueprintBuilder;
-    use nexus_deployment::blueprint_builder::Ensure;
     use nexus_inventory::now_db_precision;
+    use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder;
+    use nexus_reconfigurator_planning::blueprint_builder::Ensure;
     use nexus_test_utils::db::test_setup_database;
     use nexus_types::deployment::Policy;
     use nexus_types::deployment::SledResources;
diff --git a/nexus/reconfigurator/README.md b/nexus/reconfigurator/README.md
new file mode 100644
index 0000000000..7c6a98e666
--- /dev/null
+++ b/nexus/reconfigurator/README.md
@@ -0,0 +1,13 @@
+# Reconfigurator
+
+**Reconfigurator** is a project and control plane facility for dynamically reconfiguring the software running on the Oxide rack.  This will be used for a variety of use cases:
+
+* Adding, removing, or replacing sleds (gracefully or otherwise)
+* Upgrading any Oxide-delivered software (see below)
+* Scale-up and scale-down of control plane services
+* Automated response to hardware failure
+
+For those with access (primarily, Oxide employees), see:
+
+* [RFD 418 Towards automated system update](https://rfd.shared.oxide.computer/rfd/418).
+* [Reconfigurator GitHub Project](https://github.com/orgs/oxidecomputer/projects/44?pane=info).
diff --git a/nexus/blueprint-execution/Cargo.toml b/nexus/reconfigurator/execution/Cargo.toml
similarity index 92%
rename from nexus/blueprint-execution/Cargo.toml
rename to nexus/reconfigurator/execution/Cargo.toml
index 164559b468..2f4807d38d 100644
--- a/nexus/blueprint-execution/Cargo.toml
+++ b/nexus/reconfigurator/execution/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "nexus-blueprint-execution"
+name = "nexus-reconfigurator-execution"
 version = "0.1.0"
 edition = "2021"
 
@@ -33,7 +33,7 @@ omicron-workspace-hack.workspace = true
 chrono.workspace = true
 httptest.workspace = true
 ipnet.workspace = true
-nexus-deployment.workspace = true
+nexus-reconfigurator-planning.workspace = true
 nexus-inventory.workspace = true
 nexus-test-utils.workspace = true
 nexus-test-utils-macros.workspace = true
diff --git a/nexus/blueprint-execution/build.rs b/nexus/reconfigurator/execution/build.rs
similarity index 100%
rename from nexus/blueprint-execution/build.rs
rename to nexus/reconfigurator/execution/build.rs
diff --git a/nexus/blueprint-execution/src/datasets.rs b/nexus/reconfigurator/execution/src/datasets.rs
similarity index 100%
rename from nexus/blueprint-execution/src/datasets.rs
rename to nexus/reconfigurator/execution/src/datasets.rs
diff --git a/nexus/blueprint-execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs
similarity index 99%
rename from nexus/blueprint-execution/src/dns.rs
rename to nexus/reconfigurator/execution/src/dns.rs
index 54611e9f66..0fa8eb1c10 100644
--- a/nexus/blueprint-execution/src/dns.rs
+++ b/nexus/reconfigurator/execution/src/dns.rs
@@ -320,8 +320,8 @@ mod test {
     use crate::Sled;
     use internal_dns::ServiceName;
     use internal_dns::DNS_ZONE;
-    use nexus_deployment::blueprint_builder::BlueprintBuilder;
     use nexus_inventory::CollectionBuilder;
+    use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder;
     use nexus_types::deployment::Blueprint;
     use nexus_types::deployment::OmicronZoneConfig;
     use nexus_types::deployment::OmicronZoneType;
diff --git a/nexus/blueprint-execution/src/lib.rs b/nexus/reconfigurator/execution/src/lib.rs
similarity index 97%
rename from nexus/blueprint-execution/src/lib.rs
rename to nexus/reconfigurator/execution/src/lib.rs
index 531c4f57a8..74b4764d7a 100644
--- a/nexus/blueprint-execution/src/lib.rs
+++ b/nexus/reconfigurator/execution/src/lib.rs
@@ -4,7 +4,7 @@
 
 //! Execution of Nexus blueprints
 //!
-//! See `nexus_deployment` crate-level docs for background.
+//! See `nexus_reconfigurator_planning` crate-level docs for background.
 
 use anyhow::{anyhow, Context};
 use nexus_db_queries::context::OpContext;
diff --git a/nexus/blueprint-execution/src/omicron_zones.rs b/nexus/reconfigurator/execution/src/omicron_zones.rs
similarity index 100%
rename from nexus/blueprint-execution/src/omicron_zones.rs
rename to nexus/reconfigurator/execution/src/omicron_zones.rs
diff --git a/nexus/blueprint-execution/src/resource_allocation.rs b/nexus/reconfigurator/execution/src/resource_allocation.rs
similarity index 100%
rename from nexus/blueprint-execution/src/resource_allocation.rs
rename to nexus/reconfigurator/execution/src/resource_allocation.rs
diff --git a/nexus/reconfigurator/execution/tests/config.test.toml b/nexus/reconfigurator/execution/tests/config.test.toml
new file mode 120000
index 0000000000..7e4238a9d6
--- /dev/null
+++ b/nexus/reconfigurator/execution/tests/config.test.toml
@@ -0,0 +1 @@
+../../../tests/config.test.toml
\ No newline at end of file
diff --git a/nexus/deployment/Cargo.toml b/nexus/reconfigurator/planning/Cargo.toml
similarity index 92%
rename from nexus/deployment/Cargo.toml
rename to nexus/reconfigurator/planning/Cargo.toml
index 115dec98a5..35ead9494e 100644
--- a/nexus/deployment/Cargo.toml
+++ b/nexus/reconfigurator/planning/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "nexus-deployment"
+name = "nexus-reconfigurator-planning"
 version = "0.1.0"
 edition = "2021"
 
diff --git a/nexus/deployment/src/blueprint_builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder.rs
similarity index 100%
rename from nexus/deployment/src/blueprint_builder.rs
rename to nexus/reconfigurator/planning/src/blueprint_builder.rs
diff --git a/nexus/deployment/src/ip_allocator.rs b/nexus/reconfigurator/planning/src/ip_allocator.rs
similarity index 100%
rename from nexus/deployment/src/ip_allocator.rs
rename to nexus/reconfigurator/planning/src/ip_allocator.rs
diff --git a/nexus/deployment/src/lib.rs b/nexus/reconfigurator/planning/src/lib.rs
similarity index 100%
rename from nexus/deployment/src/lib.rs
rename to nexus/reconfigurator/planning/src/lib.rs
diff --git a/nexus/deployment/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs
similarity index 100%
rename from nexus/deployment/src/planner.rs
rename to nexus/reconfigurator/planning/src/planner.rs
diff --git a/nexus/src/app/background/blueprint_execution.rs b/nexus/src/app/background/blueprint_execution.rs
index 4ba60ab566..3c2530a3d3 100644
--- a/nexus/src/app/background/blueprint_execution.rs
+++ b/nexus/src/app/background/blueprint_execution.rs
@@ -69,7 +69,7 @@ impl BackgroundTask for BlueprintExecutor {
                 });
             }
 
-            let result = nexus_blueprint_execution::realize_blueprint(
+            let result = nexus_reconfigurator_execution::realize_blueprint(
                 opctx,
                 &self.datastore,
                 blueprint,
@@ -258,7 +258,7 @@ mod test {
 
         // Make sure that requests get made to the sled agent.  This is not a
         // careful check of exactly what gets sent.  For that, see the tests in
-        // nexus-blueprint-execution.
+        // nexus-reconfigurator-execution.
         for s in [&mut s1, &mut s2] {
             s.expect(
                 Expectation::matching(all_of![request::method_path(
diff --git a/nexus/src/app/deployment.rs b/nexus/src/app/deployment.rs
index adf6119c5c..31ba9fe065 100644
--- a/nexus/src/app/deployment.rs
+++ b/nexus/src/app/deployment.rs
@@ -9,8 +9,8 @@ use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db::datastore::SQL_BATCH_SIZE;
 use nexus_db_queries::db::pagination::Paginator;
-use nexus_deployment::blueprint_builder::BlueprintBuilder;
-use nexus_deployment::planner::Planner;
+use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder;
+use nexus_reconfigurator_planning::planner::Planner;
 use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::BlueprintMetadata;
 use nexus_types::deployment::BlueprintTarget;
diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs
index 0f601f3db5..ef3c03a302 100644
--- a/nexus/types/src/deployment.rs
+++ b/nexus/types/src/deployment.rs
@@ -4,12 +4,13 @@
 
 //! Types representing deployed software and configuration
 //!
-//! For more on this, see the crate-level documentation for `nexus/deployment`.
+//! For more on this, see the crate-level documentation for
+//! `nexus/reconfigurator/planning`.
 //!
 //! This lives in nexus/types because it's used by both nexus/db-model and
-//! nexus/deployment.  (It could as well just live in nexus/db-model, but
-//! nexus/deployment does not currently know about nexus/db-model and it's
-//! convenient to separate these concerns.)
+//! nexus/reconfigurator/planning.  (It could as well just live in
+//! nexus/db-model, but nexus/reconfigurator/planning does not currently know
+//! about nexus/db-model and it's convenient to separate these concerns.)
 
 use crate::external_api::views::SledPolicy;
 use crate::external_api::views::SledState;
@@ -98,8 +99,8 @@ impl SledResources {
 /// Describes a complete set of software and configuration for the system
 // Blueprints are a fundamental part of how the system modifies itself.  Each
 // blueprint completely describes all of the software and configuration
-// that the control plane manages.  See the nexus/deployment crate-level
-// documentation for details.
+// that the control plane manages.  See the nexus/reconfigurator/planning
+// crate-level documentation for details.
 //
 // Blueprints are different from policy.  Policy describes the things that an
 // operator would generally want to control.  The blueprint describes the

From b993768e4d62515f33a058e4082dd7a721c9cee1 Mon Sep 17 00:00:00 2001
From: Laura Abbott <labbott@users.noreply.github.com>
Date: Thu, 29 Feb 2024 17:16:10 -0500
Subject: [PATCH 057/157] Bump Gimlet to v1.0.10 (#5171)

This specifcially contains some additional debugging information
---
 tools/hubris_checksums | 9 ++++-----
 tools/hubris_version   | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/tools/hubris_checksums b/tools/hubris_checksums
index fb1c8c1930..65a2739fcd 100644
--- a/tools/hubris_checksums
+++ b/tools/hubris_checksums
@@ -1,9 +1,8 @@
-4ebca97672fafd3b2716c17f00c8a93f531c1ccb44137c756c77c13383b203da  build-gimlet-c-image-default-v1.0.9.zip
-a945dbcd4b1825e95db32bbf2fbdc56013501396dcdc5ebe6ded361decfcb1f6  build-gimlet-d-image-default-v1.0.9.zip
-527c4fe9488c5834f0fd68fa59756ab01eb0c36d725ba8b8ade1375e2dcb4c3e  build-gimlet-e-image-default-v1.0.9.zip
-69d09a1367b5cfecf43c5b733f84adcb0e4f73b49e062f1c4574b769fc2f1de9  build-gimlet-f-image-default-v1.0.9.zip
+05e4d531c2db947379c08bda099a53055e18e86d4c7fbcba1004f7f7983da701  build-gimlet-c-image-default-v1.0.10.zip
+9cba42b8d6c4dbd1bcc805b44e14e6bac7fa590f1faaaa47664f7aa254e91602  build-gimlet-d-image-default-v1.0.10.zip
+3ace1b1d692e3c23815665b932dadb42b58abc1942fdd4564c1884a13fa0f0cb  build-gimlet-e-image-default-v1.0.10.zip
+b3b05e1926bb0f66be8587a67084c76cd760da5db0f1e288c265a03cf80637e9  build-gimlet-f-image-default-v1.0.10.zip
 0fa33a057dec7301280338cf5a80a5198b5d95475dba350a807979f1a53ce5c7  build-psc-b-image-default-v1.0.9.zip
 88e7aa91fe4f976753ccfc8ba9a57a0521c54d9ef35504cf16e7b1ce78c87465  build-psc-c-image-default-v1.0.9.zip
 cfbf21fef84b38c2a5ca3c0b45703c20c2605c6bf5d599b53b26e3a2d09d756d  build-sidecar-b-image-default-v1.0.9.zip
 c98b89993f1a9b5219bbabf06897db2f92d60f6a3afb4374cb8e6e79d57fdfe4  build-sidecar-c-image-default-v1.0.9.zip
-
diff --git a/tools/hubris_version b/tools/hubris_version
index 37a180a853..cb9e30d980 100644
--- a/tools/hubris_version
+++ b/tools/hubris_version
@@ -1 +1 @@
-TAGS=(gimlet-v1.0.9 psc-v1.0.9 sidecar-v1.0.9)
+TAGS=(gimlet-v1.0.10 psc-v1.0.9 sidecar-v1.0.9)

From bc1efb35c7b98c049d041fa2fde3613a57490e17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Karen=20C=C3=A1rcamo?= <karencfv@users.noreply.github.com>
Date: Fri, 1 Mar 2024 20:08:15 +1300
Subject: [PATCH 058/157] Add logging to identify
 client::tests::test_replicated test flake (#5175)

To help us identify what is going on with the test flake

Related https://github.com/oxidecomputer/omicron/issues/5166
---
 oximeter/db/src/client.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/oximeter/db/src/client.rs b/oximeter/db/src/client.rs
index ca996dc894..2e3029a938 100644
--- a/oximeter/db/src/client.rs
+++ b/oximeter/db/src/client.rs
@@ -1944,19 +1944,22 @@ mod tests {
         let sql = String::from(
             "INSERT INTO oximeter.measurements_string (datum) VALUES ('hiya');",
         );
-        client_2.execute_with_body(sql).await.unwrap();
+        let result = client_2.execute_with_body(sql.clone()).await.unwrap();
+        info!(log, "Inserted datum to client #2"; "sql" => sql, "result" => result);
 
         // Make sure replicas are synched
         let sql = String::from(
             "SYSTEM SYNC REPLICA oximeter.measurements_string_local;",
         );
-        client_1.execute_with_body(sql).await.unwrap();
+        let result = client_1.execute_with_body(sql.clone()).await.unwrap();
+        info!(log, "Synced replicas via client #1"; "sql" => sql, "result" => result);
 
         // Make sure data exists in the other replica
         let sql = String::from(
             "SELECT * FROM oximeter.measurements_string FORMAT JSONEachRow;",
         );
-        let result = client_1.execute_with_body(sql).await.unwrap();
+        let result = client_1.execute_with_body(sql.clone()).await.unwrap();
+        info!(log, "Retrieved values via client #1"; "sql" => sql, "result" => result.clone());
         assert!(result.contains("hiya"));
 
         client_1.wipe_replicated_db().await?;

From f6efad4126986f72d3bedfdc04cb4ed30a926f0b Mon Sep 17 00:00:00 2001
From: David Crespo <david-crespo@users.noreply.github.com>
Date: Fri, 1 Mar 2024 11:38:13 -0600
Subject: [PATCH 059/157] Floating IP attach fix: fip by ID + instance by name
 and vice versa (#5173)

When I tried attaching a floating IP to an instance in the web console
on dogfood, I got this error from the API.

<img
src="https://github.com/oxidecomputer/omicron/assets/3612203/0c7b6a2f-f31e-44d1-bc21-07c5f1926ef7"
width="400">


It turns out we were taking the project as specified for determining the
floating IP and jamming it into the instance selector, regardless of
whether it needed it (selecting instance by name) or not (selecting
instance by ID). I added a bit to the floating IP attach test and
confirmed that it fails without the fix and passes with the fix.

Even with this fix in place, there is still one more loose end to tie
up: if the floating IP is specified by ID, and therefore does not have
the project, but the _instance_ is specified by name, this will fail
because there is no project to include for instance specification
purposes. The only way I can see of making this work without asking the
user to somehow specify the project for the instance is to fetch the
floating IP and pull the project off of that to include on the instance
selector. This makes sense because you can only ever attach a floating
IP to an instance in the same project.

Edit: did that in 6c144e82e04e1a64da9bd1a6ccc6026efb76f98e. Kinda sad to
do an extra DB fetch, but I don't see another way to preserve the
ability to do name or ID on the instance.
---
 nexus/src/app/external_ip.rs                  | 26 +++++++-
 nexus/src/app/instance.rs                     |  2 +-
 nexus/tests/integration_tests/external_ips.rs | 59 +++++++++++++++++++
 nexus/types/src/external_api/params.rs        |  2 +-
 4 files changed, 86 insertions(+), 3 deletions(-)

diff --git a/nexus/src/app/external_ip.rs b/nexus/src/app/external_ip.rs
index 45b05fbb0b..22907d4ea9 100644
--- a/nexus/src/app/external_ip.rs
+++ b/nexus/src/app/external_ip.rs
@@ -146,10 +146,34 @@ impl super::Nexus {
     ) -> UpdateResult<views::FloatingIp> {
         match target.kind {
             params::FloatingIpParentKind::Instance => {
+                // Handle the case where floating IP is specified by name (and
+                // therefore a project is given) but instance is specified by
+                // ID (and therefore the lookup doesn't want a project), as well
+                // as the converse: floating IP specified by ID (and no project
+                // given) but instance specified by name, and therefore needs
+                // a project. In the latter case, we have to fetch the floating
+                // IP by its ID in order to get the project to include with
+                // the instance.
+                let project =
+                    match (target.parent.clone(), fip_selector.clone().project)
+                    {
+                        (NameOrId::Id(_), _) => None,
+                        (NameOrId::Name(_), Some(p)) => Some(p),
+                        (NameOrId::Name(_), None) => {
+                            let fip_lookup = self.floating_ip_lookup(
+                                opctx,
+                                fip_selector.clone(),
+                            )?;
+                            let (.., fip) = fip_lookup.fetch().await?;
+                            Some(fip.project_id.into())
+                        }
+                    };
+
                 let instance_selector = params::InstanceSelector {
-                    project: fip_selector.project,
+                    project,
                     instance: target.parent,
                 };
+
                 let instance =
                     self.instance_lookup(opctx, instance_selector)?;
                 let attach_params = &params::ExternalIpCreate::Floating {
diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs
index 88a427af6a..200fad9f05 100644
--- a/nexus/src/app/instance.rs
+++ b/nexus/src/app/instance.rs
@@ -201,7 +201,7 @@ impl super::Nexus {
                 ..
             } => {
                 Err(Error::invalid_request(
-                    "when providing instance as an ID project should not be specified",
+                    "when providing instance as an ID, project should not be specified",
                 ))
             }
             _ => {
diff --git a/nexus/tests/integration_tests/external_ips.rs b/nexus/tests/integration_tests/external_ips.rs
index ee59c6a034..39c27174a1 100644
--- a/nexus/tests/integration_tests/external_ips.rs
+++ b/nexus/tests/integration_tests/external_ips.rs
@@ -643,6 +643,65 @@ async fn test_external_ip_live_attach_detach(
             "instance does not have an ephemeral IP attached".to_string()
         );
     }
+
+    // Finally, two kind of funny tests. There is special logic in the handler
+    // for the case where the floating IP is specified by name but the instance
+    // by ID and vice versa, so we want to test both combinations.
+
+    // Attach to an instance by instance ID with floating IP selected by name
+    let floating_ip_name = fips[0].identity.name.as_str();
+    let instance_id = instances[0].identity.id;
+    let url = attach_floating_ip_url(floating_ip_name, PROJECT_NAME);
+    let body = params::FloatingIpAttach {
+        kind: params::FloatingIpParentKind::Instance,
+        parent: instance_id.into(),
+    };
+    let attached: views::FloatingIp = NexusRequest::new(
+        RequestBuilder::new(client, Method::POST, &url)
+            .body(Some(&body))
+            .expect_status(Some(StatusCode::ACCEPTED)),
+    )
+    .authn_as(AuthnMode::PrivilegedUser)
+    .execute()
+    .await
+    .unwrap()
+    .parsed_body()
+    .unwrap();
+
+    assert_eq!(attached.identity.name.as_str(), floating_ip_name);
+
+    let instance_name = instances[0].identity.name.as_str();
+    let eip_list =
+        fetch_instance_external_ips(client, instance_name, PROJECT_NAME).await;
+    assert_eq!(eip_list.len(), 1);
+    assert_eq!(eip_list[0].ip(), fips[0].ip);
+
+    // now the other way: floating IP by ID and instance by name
+    let floating_ip_id = fips[1].identity.id;
+    let instance_name = instances[1].identity.name.as_str();
+    let url = format!("/v1/floating-ips/{floating_ip_id}/attach");
+    let body = params::FloatingIpAttach {
+        kind: params::FloatingIpParentKind::Instance,
+        parent: instance_name.parse::<Name>().unwrap().into(),
+    };
+    let attached: views::FloatingIp = NexusRequest::new(
+        RequestBuilder::new(client, Method::POST, &url)
+            .body(Some(&body))
+            .expect_status(Some(StatusCode::ACCEPTED)),
+    )
+    .authn_as(AuthnMode::PrivilegedUser)
+    .execute()
+    .await
+    .unwrap()
+    .parsed_body()
+    .unwrap();
+
+    assert_eq!(attached.identity.id, floating_ip_id);
+
+    let eip_list =
+        fetch_instance_external_ips(client, instance_name, PROJECT_NAME).await;
+    assert_eq!(eip_list.len(), 1);
+    assert_eq!(eip_list[0].ip(), fips[1].ip);
 }
 
 #[nexus_test]
diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs
index 567b1ff4ad..46260d20d0 100644
--- a/nexus/types/src/external_api/params.rs
+++ b/nexus/types/src/external_api/params.rs
@@ -167,7 +167,7 @@ pub struct OptionalProjectSelector {
     pub project: Option<NameOrId>,
 }
 
-#[derive(Deserialize, JsonSchema)]
+#[derive(Deserialize, JsonSchema, Clone)]
 pub struct FloatingIpSelector {
     /// Name or ID of the project, only required if `floating_ip` is provided as a `Name`
     pub project: Option<NameOrId>,

From 6c54c826cd74f272bffc139d97ad2b728d8dd232 Mon Sep 17 00:00:00 2001
From: Kyle Simpson <kyle@oxide.computer>
Date: Fri, 1 Mar 2024 21:30:31 +0000
Subject: [PATCH 060/157] Fix: prevent cross-project floating IP attach (#5177)

This PR ensures that attached FIPs have the same project ID as their
target instance, and prevents double lookup of a FIP in the `FIP [name +
project] + Instance [ID]` attach case.
---
 .../src/db/datastore/external_ip.rs           | 15 ++--
 nexus/src/app/external_ip.rs                  | 53 +++++-------
 nexus/src/app/instance.rs                     | 54 ++++++++++--
 nexus/src/app/sagas/instance_common.rs        |  8 +-
 nexus/src/app/sagas/instance_create.rs        |  8 +-
 nexus/src/app/sagas/instance_ip_attach.rs     | 72 +++++++---------
 nexus/src/app/sagas/instance_ip_detach.rs     |  7 +-
 nexus/src/app/sagas/mod.rs                    |  2 +-
 nexus/src/external_api/http_entrypoints.rs    |  6 +-
 nexus/tests/integration_tests/external_ips.rs | 83 ++++++++++++++++++-
 10 files changed, 213 insertions(+), 95 deletions(-)

diff --git a/nexus/db-queries/src/db/datastore/external_ip.rs b/nexus/db-queries/src/db/datastore/external_ip.rs
index d15e1b7ca8..3fdfbc1d4b 100644
--- a/nexus/db-queries/src/db/datastore/external_ip.rs
+++ b/nexus/db-queries/src/db/datastore/external_ip.rs
@@ -37,12 +37,12 @@ use chrono::Utc;
 use diesel::prelude::*;
 use nexus_db_model::Instance;
 use nexus_db_model::IpAttachState;
-use nexus_types::external_api::params;
 use nexus_types::identity::Resource;
 use omicron_common::api::external::http_pagination::PaginatedBy;
 use omicron_common::api::external::CreateResult;
 use omicron_common::api::external::DeleteResult;
 use omicron_common::api::external::Error;
+use omicron_common::api::external::IdentityMetadataCreateParams;
 use omicron_common::api::external::ListResultVec;
 use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::ResourceType;
@@ -227,7 +227,8 @@ impl DataStore {
         &self,
         opctx: &OpContext,
         project_id: Uuid,
-        params: params::FloatingIpCreate,
+        identity: IdentityMetadataCreateParams,
+        ip: Option<IpAddr>,
         pool: Option<authz::IpPool>,
     ) -> CreateResult<ExternalIp> {
         let ip_id = Uuid::new_v4();
@@ -256,11 +257,11 @@ impl DataStore {
 
         let pool_id = pool.id();
 
-        let data = if let Some(ip) = params.ip {
+        let data = if let Some(ip) = ip {
             IncompleteExternalIp::for_floating_explicit(
                 ip_id,
-                &Name(params.identity.name),
-                &params.identity.description,
+                &Name(identity.name),
+                &identity.description,
                 project_id,
                 ip,
                 pool_id,
@@ -268,8 +269,8 @@ impl DataStore {
         } else {
             IncompleteExternalIp::for_floating(
                 ip_id,
-                &Name(params.identity.name),
-                &params.identity.description,
+                &Name(identity.name),
+                &identity.description,
                 project_id,
                 pool_id,
             )
diff --git a/nexus/src/app/external_ip.rs b/nexus/src/app/external_ip.rs
index 22907d4ea9..19a2cdee36 100644
--- a/nexus/src/app/external_ip.rs
+++ b/nexus/src/app/external_ip.rs
@@ -109,9 +109,11 @@ impl super::Nexus {
         let (.., authz_project) =
             project_lookup.lookup_for(authz::Action::CreateChild).await?;
 
-        let pool = match &params.pool {
+        let params::FloatingIpCreate { identity, pool, ip } = params;
+
+        let pool = match pool {
             Some(pool) => Some(
-                self.ip_pool_lookup(opctx, pool)?
+                self.ip_pool_lookup(opctx, &pool)?
                     .lookup_for(authz::Action::Read)
                     .await?
                     .0,
@@ -121,7 +123,7 @@ impl super::Nexus {
 
         Ok(self
             .db_datastore
-            .allocate_floating_ip(opctx, authz_project.id(), params, pool)
+            .allocate_floating_ip(opctx, authz_project.id(), identity, ip, pool)
             .await?
             .try_into()
             .unwrap())
@@ -144,45 +146,32 @@ impl super::Nexus {
         fip_selector: params::FloatingIpSelector,
         target: params::FloatingIpAttach,
     ) -> UpdateResult<views::FloatingIp> {
+        let fip_lookup = self.floating_ip_lookup(opctx, fip_selector)?;
+        let (.., authz_project, authz_fip) =
+            fip_lookup.lookup_for(authz::Action::Modify).await?;
+
         match target.kind {
             params::FloatingIpParentKind::Instance => {
-                // Handle the case where floating IP is specified by name (and
-                // therefore a project is given) but instance is specified by
-                // ID (and therefore the lookup doesn't want a project), as well
-                // as the converse: floating IP specified by ID (and no project
-                // given) but instance specified by name, and therefore needs
-                // a project. In the latter case, we have to fetch the floating
-                // IP by its ID in order to get the project to include with
-                // the instance.
-                let project =
-                    match (target.parent.clone(), fip_selector.clone().project)
-                    {
-                        (NameOrId::Id(_), _) => None,
-                        (NameOrId::Name(_), Some(p)) => Some(p),
-                        (NameOrId::Name(_), None) => {
-                            let fip_lookup = self.floating_ip_lookup(
-                                opctx,
-                                fip_selector.clone(),
-                            )?;
-                            let (.., fip) = fip_lookup.fetch().await?;
-                            Some(fip.project_id.into())
-                        }
-                    };
-
+                // Handle the cases where the FIP and instance are specified by
+                // name and ID (or ID and name) respectively. We remove the project
+                // from the instance lookup if using the instance's ID, and insert
+                // the floating IP's project ID otherwise.
                 let instance_selector = params::InstanceSelector {
-                    project,
+                    project: match &target.parent {
+                        NameOrId::Id(_) => None,
+                        NameOrId::Name(_) => Some(authz_project.id().into()),
+                    },
                     instance: target.parent,
                 };
 
                 let instance =
                     self.instance_lookup(opctx, instance_selector)?;
-                let attach_params = &params::ExternalIpCreate::Floating {
-                    floating_ip: fip_selector.floating_ip,
-                };
-                self.instance_attach_external_ip(
+
+                self.instance_attach_floating_ip(
                     opctx,
                     &instance,
-                    attach_params,
+                    authz_fip,
+                    authz_project,
                 )
                 .await
                 .and_then(FloatingIp::try_from)
diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs
index 200fad9f05..f6cf90718f 100644
--- a/nexus/src/app/instance.rs
+++ b/nexus/src/app/instance.rs
@@ -49,6 +49,7 @@ use propolis_client::support::tungstenite::Message as WebSocketMessage;
 use propolis_client::support::InstanceSerialConsoleHelper;
 use propolis_client::support::WSClientOffset;
 use propolis_client::support::WebSocketStream;
+use sagas::instance_common::ExternalIpAttach;
 use sled_agent_client::types::InstanceMigrationSourceParams;
 use sled_agent_client::types::InstanceMigrationTargetParams;
 use sled_agent_client::types::InstanceProperties;
@@ -201,7 +202,7 @@ impl super::Nexus {
                 ..
             } => {
                 Err(Error::invalid_request(
-                    "when providing instance as an ID, project should not be specified",
+                    "when providing instance as an ID project should not be specified",
                 ))
             }
             _ => {
@@ -1952,20 +1953,63 @@ impl super::Nexus {
         Ok(())
     }
 
-    /// Attach an external IP to an instance.
-    pub(crate) async fn instance_attach_external_ip(
+    /// Attach an ephemeral IP to an instance.
+    pub(crate) async fn instance_attach_ephemeral_ip(
+        self: &Arc<Self>,
+        opctx: &OpContext,
+        instance_lookup: &lookup::Instance<'_>,
+        pool: Option<NameOrId>,
+    ) -> UpdateResult<views::ExternalIp> {
+        let (.., authz_project, authz_instance) =
+            instance_lookup.lookup_for(authz::Action::Modify).await?;
+
+        self.instance_attach_external_ip(
+            opctx,
+            authz_instance,
+            authz_project.id(),
+            ExternalIpAttach::Ephemeral { pool },
+        )
+        .await
+    }
+
+    /// Attach an ephemeral IP to an instance.
+    pub(crate) async fn instance_attach_floating_ip(
         self: &Arc<Self>,
         opctx: &OpContext,
         instance_lookup: &lookup::Instance<'_>,
-        ext_ip: &params::ExternalIpCreate,
+        authz_fip: authz::FloatingIp,
+        authz_fip_project: authz::Project,
     ) -> UpdateResult<views::ExternalIp> {
         let (.., authz_project, authz_instance) =
             instance_lookup.lookup_for(authz::Action::Modify).await?;
 
+        if authz_fip_project.id() != authz_project.id() {
+            return Err(Error::invalid_request(
+                "floating IP must be in the same project as the instance",
+            ));
+        }
+
+        self.instance_attach_external_ip(
+            opctx,
+            authz_instance,
+            authz_project.id(),
+            ExternalIpAttach::Floating { floating_ip: authz_fip },
+        )
+        .await
+    }
+
+    /// Attach an external IP to an instance.
+    pub(crate) async fn instance_attach_external_ip(
+        self: &Arc<Self>,
+        opctx: &OpContext,
+        authz_instance: authz::Instance,
+        project_id: Uuid,
+        ext_ip: ExternalIpAttach,
+    ) -> UpdateResult<views::ExternalIp> {
         let saga_params = sagas::instance_ip_attach::Params {
             create_params: ext_ip.clone(),
             authz_instance,
-            project_id: authz_project.id(),
+            project_id,
             serialized_authn: authn::saga::Serialized::for_opctx(opctx),
         };
 
diff --git a/nexus/src/app/sagas/instance_common.rs b/nexus/src/app/sagas/instance_common.rs
index e915c026dd..b941739393 100644
--- a/nexus/src/app/sagas/instance_common.rs
+++ b/nexus/src/app/sagas/instance_common.rs
@@ -16,8 +16,8 @@ use nexus_db_queries::authz;
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_db_queries::db::queries::external_ip::SAFE_TRANSIENT_INSTANCE_STATES;
 use nexus_db_queries::{authn, context::OpContext, db, db::DataStore};
-use omicron_common::api::external::Error;
 use omicron_common::api::external::InstanceState;
+use omicron_common::api::external::{Error, NameOrId};
 use serde::{Deserialize, Serialize};
 use steno::ActionError;
 use uuid::Uuid;
@@ -469,3 +469,9 @@ pub async fn instance_ip_remove_opte(
 
     Ok(())
 }
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub enum ExternalIpAttach {
+    Ephemeral { pool: Option<NameOrId> },
+    Floating { floating_ip: authz::FloatingIp },
+}
diff --git a/nexus/src/app/sagas/instance_create.rs b/nexus/src/app/sagas/instance_create.rs
index 88dd0ae36e..0950754572 100644
--- a/nexus/src/app/sagas/instance_create.rs
+++ b/nexus/src/app/sagas/instance_create.rs
@@ -733,7 +733,7 @@ async fn sic_allocate_instance_external_ip(
         }
         // Set the parent of an existing floating IP to the new instance's ID.
         params::ExternalIpCreate::Floating { floating_ip } => {
-            let (.., authz_fip) = match floating_ip {
+            let (.., authz_project, authz_fip) = match floating_ip {
                 NameOrId::Name(name) => LookupPath::new(&opctx, datastore)
                     .project_id(saga_params.project_id)
                     .floating_ip_name(db::model::Name::ref_cast(name)),
@@ -745,6 +745,12 @@ async fn sic_allocate_instance_external_ip(
             .await
             .map_err(ActionError::action_failed)?;
 
+            if authz_project.id() != saga_params.project_id {
+                return Err(ActionError::action_failed(Error::invalid_request(
+                    "floating IP must be in the same project as the instance"
+                )));
+            }
+
             datastore
                 .floating_ip_begin_attach(&opctx, &authz_fip, instance_id, true)
                 .await
diff --git a/nexus/src/app/sagas/instance_ip_attach.rs b/nexus/src/app/sagas/instance_ip_attach.rs
index be7f81368e..3cd6ac1c46 100644
--- a/nexus/src/app/sagas/instance_ip_attach.rs
+++ b/nexus/src/app/sagas/instance_ip_attach.rs
@@ -4,17 +4,15 @@
 
 use super::instance_common::{
     instance_ip_add_nat, instance_ip_add_opte, instance_ip_get_instance_state,
-    instance_ip_move_state, instance_ip_remove_opte, ModifyStateForExternalIp,
+    instance_ip_move_state, instance_ip_remove_opte, ExternalIpAttach,
+    ModifyStateForExternalIp,
 };
 use super::{ActionRegistry, NexusActionContext, NexusSaga};
 use crate::app::sagas::declare_saga_actions;
-use crate::app::{authn, authz, db};
-use crate::external_api::params;
+use crate::app::{authn, authz};
 use nexus_db_model::{IpAttachState, Ipv4NatEntry};
-use nexus_db_queries::db::lookup::LookupPath;
 use nexus_types::external_api::views;
-use omicron_common::api::external::{Error, NameOrId};
-use ref_cast::RefCast;
+use omicron_common::api::external::Error;
 use serde::Deserialize;
 use serde::Serialize;
 use steno::ActionError;
@@ -72,7 +70,7 @@ declare_saga_actions! {
 
 #[derive(Debug, Deserialize, Serialize)]
 pub struct Params {
-    pub create_params: params::ExternalIpCreate,
+    pub create_params: ExternalIpAttach,
     pub authz_instance: authz::Instance,
     pub project_id: Uuid,
     /// Authentication context to use to fetch the instance's current state from
@@ -93,7 +91,7 @@ async fn siia_begin_attach_ip(
 
     match &params.create_params {
         // Allocate a new IP address from the target, possibly default, pool
-        params::ExternalIpCreate::Ephemeral { pool } => {
+        ExternalIpAttach::Ephemeral { pool } => {
             let pool = if let Some(name_or_id) = pool {
                 Some(
                     osagactx
@@ -125,33 +123,19 @@ async fn siia_begin_attach_ip(
                 })
         }
         // Set the parent of an existing floating IP to the new instance's ID.
-        params::ExternalIpCreate::Floating { floating_ip } => {
-            let (.., authz_fip) = match floating_ip {
-                NameOrId::Name(name) => LookupPath::new(&opctx, datastore)
-                    .project_id(params.project_id)
-                    .floating_ip_name(db::model::Name::ref_cast(name)),
-                NameOrId::Id(id) => {
-                    LookupPath::new(&opctx, datastore).floating_ip_id(*id)
-                }
-            }
-            .lookup_for(authz::Action::Modify)
+        ExternalIpAttach::Floating { floating_ip } => datastore
+            .floating_ip_begin_attach(
+                &opctx,
+                &floating_ip,
+                params.authz_instance.id(),
+                false,
+            )
             .await
-            .map_err(ActionError::action_failed)?;
-
-            datastore
-                .floating_ip_begin_attach(
-                    &opctx,
-                    &authz_fip,
-                    params.authz_instance.id(),
-                    false,
-                )
-                .await
-                .map_err(ActionError::action_failed)
-                .map(|(external_ip, do_saga)| ModifyStateForExternalIp {
-                    external_ip: Some(external_ip),
-                    do_saga,
-                })
-        }
+            .map_err(ActionError::action_failed)
+            .map(|(external_ip, do_saga)| ModifyStateForExternalIp {
+                external_ip: Some(external_ip),
+                do_saga,
+            }),
     }
 }
 
@@ -349,7 +333,7 @@ impl NexusSaga for SagaInstanceIpAttach {
 #[cfg(test)]
 pub(crate) mod test {
     use super::*;
-    use crate::app::{saga::create_saga_dag, sagas::test_helpers};
+    use crate::app::{db, saga::create_saga_dag, sagas::test_helpers};
     use async_bb8_diesel::AsyncRunQueryDsl;
     use diesel::{
         ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper,
@@ -357,12 +341,13 @@ pub(crate) mod test {
     use dropshot::test_util::ClientTestContext;
     use nexus_db_model::{ExternalIp, IpKind};
     use nexus_db_queries::context::OpContext;
+    use nexus_db_queries::db::lookup::LookupPath;
     use nexus_test_utils::resource_helpers::{
         create_default_ip_pool, create_floating_ip, create_instance,
         create_project,
     };
     use nexus_test_utils_macros::nexus_test;
-    use omicron_common::api::external::{Name, SimpleIdentity};
+    use omicron_common::api::external::SimpleIdentity;
 
     type ControlPlaneTestContext =
         nexus_test_utils::ControlPlaneTestContext<crate::Server>;
@@ -391,17 +376,22 @@ pub(crate) mod test {
         datastore: &db::DataStore,
         use_floating: bool,
     ) -> Params {
+        let project_name = db::model::Name(PROJECT_NAME.parse().unwrap());
         let create_params = if use_floating {
-            params::ExternalIpCreate::Floating {
-                floating_ip: FIP_NAME.parse::<Name>().unwrap().into(),
-            }
+            let (.., floating_ip) = LookupPath::new(opctx, datastore)
+                .project_name(&project_name)
+                .floating_ip_name(&db::model::Name(FIP_NAME.parse().unwrap()))
+                .lookup_for(authz::Action::Modify)
+                .await
+                .unwrap();
+            ExternalIpAttach::Floating { floating_ip }
         } else {
-            params::ExternalIpCreate::Ephemeral { pool: None }
+            ExternalIpAttach::Ephemeral { pool: None }
         };
 
         let (.., authz_project, authz_instance) =
             LookupPath::new(opctx, datastore)
-                .project_name(&db::model::Name(PROJECT_NAME.parse().unwrap()))
+                .project_name(&project_name)
                 .instance_name(&db::model::Name(INSTANCE_NAME.parse().unwrap()))
                 .lookup_for(authz::Action::Modify)
                 .await
diff --git a/nexus/src/app/sagas/instance_ip_detach.rs b/nexus/src/app/sagas/instance_ip_detach.rs
index da6c92077d..7a71824376 100644
--- a/nexus/src/app/sagas/instance_ip_detach.rs
+++ b/nexus/src/app/sagas/instance_ip_detach.rs
@@ -357,6 +357,8 @@ pub(crate) mod test {
             .project_name(&proj_name)
             .instance_name(&inst_name);
 
+        let (.., authz_proj, authz_instance, _) = lookup.fetch().await.unwrap();
+
         for use_float in [false, true] {
             let params = instance_ip_attach::test::new_test_params(
                 opctx, datastore, use_float,
@@ -365,8 +367,9 @@ pub(crate) mod test {
             nexus
                 .instance_attach_external_ip(
                     opctx,
-                    &lookup,
-                    &params.create_params,
+                    authz_instance.clone(),
+                    authz_proj.id(),
+                    params.create_params,
                 )
                 .await
                 .unwrap();
diff --git a/nexus/src/app/sagas/mod.rs b/nexus/src/app/sagas/mod.rs
index e9f800c61b..aef8442090 100644
--- a/nexus/src/app/sagas/mod.rs
+++ b/nexus/src/app/sagas/mod.rs
@@ -24,7 +24,7 @@ pub mod disk_create;
 pub mod disk_delete;
 pub mod finalize_disk;
 pub mod image_delete;
-mod instance_common;
+pub(crate) mod instance_common;
 pub mod instance_create;
 pub mod instance_delete;
 pub mod instance_ip_attach;
diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs
index b007cc6217..0a626db234 100644
--- a/nexus/src/external_api/http_entrypoints.rs
+++ b/nexus/src/external_api/http_entrypoints.rs
@@ -4088,12 +4088,10 @@ async fn instance_ephemeral_ip_attach(
         let instance_lookup =
             nexus.instance_lookup(&opctx, instance_selector)?;
         let ip = nexus
-            .instance_attach_external_ip(
+            .instance_attach_ephemeral_ip(
                 &opctx,
                 &instance_lookup,
-                &params::ExternalIpCreate::Ephemeral {
-                    pool: ip_to_create.into_inner().pool,
-                },
+                ip_to_create.into_inner().pool,
             )
             .await?;
         Ok(HttpResponseAccepted(ip))
diff --git a/nexus/tests/integration_tests/external_ips.rs b/nexus/tests/integration_tests/external_ips.rs
index 39c27174a1..c1c71e9afc 100644
--- a/nexus/tests/integration_tests/external_ips.rs
+++ b/nexus/tests/integration_tests/external_ips.rs
@@ -36,8 +36,10 @@ use nexus_types::external_api::views::FloatingIp;
 use nexus_types::identity::Resource;
 use omicron_common::address::IpRange;
 use omicron_common::address::Ipv4Range;
+use omicron_common::api::external::ByteCount;
 use omicron_common::api::external::IdentityMetadataCreateParams;
 use omicron_common::api::external::Instance;
+use omicron_common::api::external::InstanceCpuCount;
 use omicron_common::api::external::Name;
 use omicron_common::api::external::NameOrId;
 use uuid::Uuid;
@@ -70,6 +72,10 @@ pub fn attach_floating_ip_url(
     format!("/v1/floating-ips/{floating_ip_name}/attach?project={project_name}")
 }
 
+pub fn attach_floating_ip_uuid(floating_ip_uuid: &Uuid) -> String {
+    format!("/v1/floating-ips/{floating_ip_uuid}/attach")
+}
+
 pub fn detach_floating_ip_url(
     floating_ip_name: &str,
     project_name: &str,
@@ -705,7 +711,82 @@ async fn test_external_ip_live_attach_detach(
 }
 
 #[nexus_test]
-async fn test_external_ip_attach_detach_fail_if_in_use_by_other(
+async fn test_floating_ip_attach_fail_between_projects(
+    cptestctx: &ControlPlaneTestContext,
+) {
+    let client = &cptestctx.external_client;
+    let apictx = &cptestctx.server.apictx();
+    let _nexus = &apictx.nexus;
+
+    create_default_ip_pool(&client).await;
+    let _project = create_project(client, PROJECT_NAME).await;
+    let _project2 = create_project(client, "proj2").await;
+
+    // Create a floating IP in another project.
+    let fip =
+        create_floating_ip(client, FIP_NAMES[0], "proj2", None, None).await;
+
+    // Create a new instance *then* bind the FIP to it, both by ID.
+    let instance =
+        instance_for_external_ips(client, INSTANCE_NAMES[0], true, false, &[])
+            .await;
+
+    let url = attach_floating_ip_uuid(&fip.identity.id);
+    let error: HttpErrorResponseBody = NexusRequest::new(
+        RequestBuilder::new(client, Method::POST, &url)
+            .body(Some(&params::FloatingIpAttach {
+                kind: params::FloatingIpParentKind::Instance,
+                parent: instance.identity.id.into(),
+            }))
+            .expect_status(Some(StatusCode::BAD_REQUEST)),
+    )
+    .authn_as(AuthnMode::PrivilegedUser)
+    .execute()
+    .await
+    .unwrap()
+    .parsed_body()
+    .unwrap();
+    assert_eq!(
+        error.message,
+        "floating IP must be in the same project as the instance".to_string()
+    );
+
+    // Create a new instance with a FIP, referenced by ID.
+    let url = format!("/v1/instances?project={PROJECT_NAME}");
+    let error = object_create_error(
+        client,
+        &url,
+        &params::InstanceCreate {
+            identity: IdentityMetadataCreateParams {
+                name: INSTANCE_NAMES[1].parse().unwrap(),
+                description: "".into(),
+            },
+            ncpus: InstanceCpuCount(4),
+            memory: ByteCount::from_gibibytes_u32(1),
+            hostname: "the-host".parse().unwrap(),
+            user_data:
+                b"#cloud-config\nsystem_info:\n  default_user:\n    name: oxide"
+                    .to_vec(),
+            ssh_public_keys: Some(Vec::new()),
+            network_interfaces:
+                params::InstanceNetworkInterfaceAttachment::Default,
+            external_ips: vec![params::ExternalIpCreate::Floating {
+                floating_ip: fip.identity.id.into(),
+            }],
+            disks: vec![],
+            start: true,
+        },
+        StatusCode::BAD_REQUEST,
+    )
+    .await;
+    assert_eq!(
+        error.message,
+        "floating IP must be in the same project as the instance".to_string()
+    );
+}
+
+#[nexus_test]
+async fn test_external_ip_attach_fail_if_in_use_by_other(
     cptestctx: &ControlPlaneTestContext,
 ) {
     let client = &cptestctx.external_client;

From 4ab41a074aa1492e6a78eeb4294f4a178f71c922 Mon Sep 17 00:00:00 2001
From: Charlie Park <charlie@oxidecomputer.com>
Date: Fri, 1 Mar 2024 13:55:42 -0800
Subject: [PATCH 061/157] Enable Updates for Floating IPs (#5176)

Fixes #5016

This PR adds the ability to update names and descriptions for floating
IPs.

---------

Co-authored-by: David Crespo <david-crespo@users.noreply.github.com>
---
 nexus/db-model/src/external_ip.rs             | 19 +++++
 .../src/db/datastore/external_ip.rs           | 27 +++++++
 nexus/src/app/external_ip.rs                  | 16 +++++
 nexus/src/external_api/http_entrypoints.rs    | 46 ++++++++++--
 nexus/tests/integration_tests/endpoints.rs    | 11 +++
 nexus/tests/integration_tests/external_ips.rs | 55 ++++++++++++++
 nexus/tests/output/nexus_tags.txt             |  1 +
 nexus/types/src/external_api/params.rs        |  6 ++
 openapi/nexus.json                            | 72 +++++++++++++++++++
 9 files changed, 249 insertions(+), 4 deletions(-)

diff --git a/nexus/db-model/src/external_ip.rs b/nexus/db-model/src/external_ip.rs
index b30f91c7c0..0f484f7610 100644
--- a/nexus/db-model/src/external_ip.rs
+++ b/nexus/db-model/src/external_ip.rs
@@ -16,6 +16,7 @@ use db_macros::Resource;
 use diesel::Queryable;
 use diesel::Selectable;
 use ipnetwork::IpNetwork;
+use nexus_types::external_api::params;
 use nexus_types::external_api::shared;
 use nexus_types::external_api::views;
 use omicron_common::address::NUM_SOURCE_NAT_PORTS;
@@ -538,6 +539,24 @@ impl From<FloatingIp> for views::FloatingIp {
     }
 }
 
+#[derive(AsChangeset)]
+#[diesel(table_name = external_ip)]
+pub struct FloatingIpUpdate {
+    pub name: Option<Name>,
+    pub description: Option<String>,
+    pub time_modified: DateTime<Utc>,
+}
+
+impl From<params::FloatingIpUpdate> for FloatingIpUpdate {
+    fn from(params: params::FloatingIpUpdate) -> Self {
+        Self {
+            name: params.identity.name.map(Name),
+            description: params.identity.description,
+            time_modified: Utc::now(),
+        }
+    }
+}
+
 impl TryFrom<ExternalIp> for InstanceExternalIpBody {
     type Error = Error;
 
diff --git a/nexus/db-queries/src/db/datastore/external_ip.rs b/nexus/db-queries/src/db/datastore/external_ip.rs
index 3fdfbc1d4b..f561a024e8 100644
--- a/nexus/db-queries/src/db/datastore/external_ip.rs
+++ b/nexus/db-queries/src/db/datastore/external_ip.rs
@@ -35,6 +35,7 @@ use crate::db::update_and_check::UpdateStatus;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use chrono::Utc;
 use diesel::prelude::*;
+use nexus_db_model::FloatingIpUpdate;
 use nexus_db_model::Instance;
 use nexus_db_model::IpAttachState;
 use nexus_types::identity::Resource;
@@ -824,6 +825,32 @@ impl DataStore {
         .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
     }
 
+    /// Update a Floating IP
+    pub async fn floating_ip_update(
+        &self,
+        opctx: &OpContext,
+        authz_fip: &authz::FloatingIp,
+        update: FloatingIpUpdate,
+    ) -> UpdateResult<ExternalIp> {
+        use db::schema::external_ip::dsl;
+
+        opctx.authorize(authz::Action::Modify, authz_fip).await?;
+
+        diesel::update(dsl::external_ip)
+            .filter(dsl::id.eq(authz_fip.id()))
+            .filter(dsl::time_deleted.is_null())
+            .set(update)
+            .returning(ExternalIp::as_returning())
+            .get_result_async(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| {
+                public_error_from_diesel(
+                    e,
+                    ErrorHandler::NotFoundByResource(authz_fip),
+                )
+            })
+    }
+
     /// Delete a Floating IP, verifying first that it is not in use.
     pub async fn floating_ip_delete(
         &self,
diff --git a/nexus/src/app/external_ip.rs b/nexus/src/app/external_ip.rs
index 19a2cdee36..9908512c0f 100644
--- a/nexus/src/app/external_ip.rs
+++ b/nexus/src/app/external_ip.rs
@@ -129,6 +129,22 @@ impl super::Nexus {
             .unwrap())
     }
 
+    pub(crate) async fn floating_ip_update(
+        &self,
+        opctx: &OpContext,
+        ip_lookup: lookup::FloatingIp<'_>,
+        params: params::FloatingIpUpdate,
+    ) -> UpdateResult<FloatingIp> {
+        let (.., authz_fip) =
+            ip_lookup.lookup_for(authz::Action::Modify).await?;
+        Ok(self
+            .db_datastore
+            .floating_ip_update(opctx, &authz_fip, params.clone().into())
+            .await?
+            .try_into()
+            .unwrap())
+    }
+
     pub(crate) async fn floating_ip_delete(
         &self,
         opctx: &OpContext,
diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs
index 0a626db234..8fe02a0fd8 100644
--- a/nexus/src/external_api/http_entrypoints.rs
+++ b/nexus/src/external_api/http_entrypoints.rs
@@ -7,10 +7,10 @@
 use super::{
     console_api, device_auth, params,
     views::{
-        self, Certificate, Group, IdentityProvider, Image, IpPool, IpPoolRange,
-        PhysicalDisk, Project, Rack, Role, Silo, SiloQuotas, SiloUtilization,
-        Sled, Snapshot, SshKey, User, UserBuiltin, Utilization, Vpc, VpcRouter,
-        VpcSubnet,
+        self, Certificate, FloatingIp, Group, IdentityProvider, Image, IpPool,
+        IpPoolRange, PhysicalDisk, Project, Rack, Role, Silo, SiloQuotas,
+        SiloUtilization, Sled, Snapshot, SshKey, User, UserBuiltin,
+        Utilization, Vpc, VpcRouter, VpcSubnet,
     },
 };
 use crate::external_api::shared;
@@ -141,6 +141,7 @@ pub(crate) fn external_api() -> NexusApiDescription {
         api.register(floating_ip_list)?;
         api.register(floating_ip_create)?;
         api.register(floating_ip_view)?;
+        api.register(floating_ip_update)?;
         api.register(floating_ip_delete)?;
         api.register(floating_ip_attach)?;
         api.register(floating_ip_detach)?;
@@ -1921,6 +1922,43 @@ async fn floating_ip_create(
     apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
 }
 
+/// Update floating IP
+#[endpoint {
+    method = PUT,
+    path = "/v1/floating-ips/{floating_ip}",
+    tags = ["floating-ips"],
+}]
+async fn floating_ip_update(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    path_params: Path<params::FloatingIpPath>,
+    query_params: Query<params::OptionalProjectSelector>,
+    updated_floating_ip: TypedBody<params::FloatingIpUpdate>,
+) -> Result<HttpResponseOk<FloatingIp>, HttpError> {
+    let apictx = rqctx.context();
+    let handler = async {
+        let nexus = &apictx.nexus;
+        let path = path_params.into_inner();
+        let query = query_params.into_inner();
+        let updated_floating_ip_params = updated_floating_ip.into_inner();
+        let opctx = crate::context::op_context_for_external_api(&rqctx).await?;
+        let floating_ip_selector = params::FloatingIpSelector {
+            project: query.project,
+            floating_ip: path.floating_ip,
+        };
+        let floating_ip_lookup =
+            nexus.floating_ip_lookup(&opctx, floating_ip_selector)?;
+        let floating_ip = nexus
+            .floating_ip_update(
+                &opctx,
+                floating_ip_lookup,
+                updated_floating_ip_params,
+            )
+            .await?;
+        Ok(HttpResponseOk(floating_ip))
+    };
+    apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
 /// Delete floating IP
 #[endpoint {
     method = DELETE,
diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs
index c5c69df232..f18b2d961d 100644
--- a/nexus/tests/integration_tests/endpoints.rs
+++ b/nexus/tests/integration_tests/endpoints.rs
@@ -772,6 +772,14 @@ pub static DEMO_FLOAT_IP_CREATE: Lazy<params::FloatingIpCreate> =
         pool: None,
     });
 
+pub static DEMO_FLOAT_IP_UPDATE: Lazy<params::FloatingIpUpdate> =
+    Lazy::new(|| params::FloatingIpUpdate {
+        identity: IdentityMetadataUpdateParams {
+            name: None,
+            description: Some(String::from("an updated Floating IP")),
+        },
+    });
+
 pub static DEMO_FLOAT_IP_ATTACH: Lazy<params::FloatingIpAttach> =
     Lazy::new(|| params::FloatingIpAttach {
         kind: params::FloatingIpParentKind::Instance,
@@ -2277,6 +2285,9 @@ pub static VERIFY_ENDPOINTS: Lazy<Vec<VerifyEndpoint>> = Lazy::new(|| {
             unprivileged_access: UnprivilegedAccess::None,
             allowed_methods: vec![
                 AllowedMethod::Get,
+                AllowedMethod::Put(
+                    serde_json::to_value(&*DEMO_FLOAT_IP_UPDATE).unwrap()
+                  ),
                 AllowedMethod::Delete,
             ],
         },
diff --git a/nexus/tests/integration_tests/external_ips.rs b/nexus/tests/integration_tests/external_ips.rs
index c1c71e9afc..25c74bfd7f 100644
--- a/nexus/tests/integration_tests/external_ips.rs
+++ b/nexus/tests/integration_tests/external_ips.rs
@@ -28,6 +28,8 @@ use nexus_test_utils::resource_helpers::object_create;
 use nexus_test_utils::resource_helpers::object_create_error;
 use nexus_test_utils::resource_helpers::object_delete;
 use nexus_test_utils::resource_helpers::object_delete_error;
+use nexus_test_utils::resource_helpers::object_get;
+use nexus_test_utils::resource_helpers::object_put;
 use nexus_test_utils_macros::nexus_test;
 use nexus_types::external_api::params;
 use nexus_types::external_api::shared;
@@ -38,6 +40,7 @@ use omicron_common::address::IpRange;
 use omicron_common::address::Ipv4Range;
 use omicron_common::api::external::ByteCount;
 use omicron_common::api::external::IdentityMetadataCreateParams;
+use omicron_common::api::external::IdentityMetadataUpdateParams;
 use omicron_common::api::external::Instance;
 use omicron_common::api::external::InstanceCpuCount;
 use omicron_common::api::external::Name;
@@ -387,6 +390,58 @@ async fn test_floating_ip_create_name_in_use(
     );
 }
 
+#[nexus_test]
+async fn test_floating_ip_update(cptestctx: &ControlPlaneTestContext) {
+    let client = &cptestctx.external_client;
+
+    create_default_ip_pool(&client).await;
+    let project = create_project(client, PROJECT_NAME).await;
+
+    // Create the Floating IP
+    let fip = create_floating_ip(
+        client,
+        FIP_NAMES[0],
+        project.identity.name.as_str(),
+        None,
+        None,
+    )
+    .await;
+
+    let floating_ip_url = get_floating_ip_by_id_url(&fip.identity.id);
+
+    // Verify that the Floating IP was created correctly
+    let fetched_floating_ip: FloatingIp =
+        object_get(client, &floating_ip_url).await;
+
+    assert_eq!(fip.identity, fetched_floating_ip.identity);
+
+    // Set up the updated values
+    let new_fip_name: &str = "updated";
+    let new_fip_desc: &str = "updated description";
+    let updates: params::FloatingIpUpdate = params::FloatingIpUpdate {
+        identity: IdentityMetadataUpdateParams {
+            name: Some(String::from(new_fip_name).parse().unwrap()),
+            description: Some(String::from(new_fip_desc).parse().unwrap()),
+        },
+    };
+
+    // Update the Floating IP
+    let new_fip: FloatingIp =
+        object_put(client, &floating_ip_url, &updates).await;
+
+    assert_eq!(new_fip.identity.name.as_str(), new_fip_name);
+    assert_eq!(new_fip.identity.description, new_fip_desc);
+    assert_eq!(new_fip.project_id, project.identity.id);
+    assert_eq!(new_fip.identity.time_created, fip.identity.time_created);
+    assert_ne!(new_fip.identity.time_modified, fip.identity.time_modified);
+
+    // Verify that the Floating IP was updated correctly
+    let fetched_modified_floating_ip: FloatingIp =
+        object_get(client, &floating_ip_url).await;
+
+    assert_eq!(new_fip.identity, fetched_modified_floating_ip.identity);
+}
+
 #[nexus_test]
 async fn test_floating_ip_delete(cptestctx: &ControlPlaneTestContext) {
     let client = &cptestctx.external_client;
diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt
index adb36a24af..ecffadcb4d 100644
--- a/nexus/tests/output/nexus_tags.txt
+++ b/nexus/tests/output/nexus_tags.txt
@@ -17,6 +17,7 @@ floating_ip_create                       POST     /v1/floating-ips
 floating_ip_delete                       DELETE   /v1/floating-ips/{floating_ip}
 floating_ip_detach                       POST     /v1/floating-ips/{floating_ip}/detach
 floating_ip_list                         GET      /v1/floating-ips
+floating_ip_update                       PUT      /v1/floating-ips/{floating_ip}
 floating_ip_view                         GET      /v1/floating-ips/{floating_ip}
 
 API operations found with tag "hidden"
diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs
index 46260d20d0..31cb1d3e5c 100644
--- a/nexus/types/src/external_api/params.rs
+++ b/nexus/types/src/external_api/params.rs
@@ -890,6 +890,12 @@ pub struct FloatingIpCreate {
     pub pool: Option<NameOrId>,
 }
 
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
+pub struct FloatingIpUpdate {
+    #[serde(flatten)]
+    pub identity: IdentityMetadataUpdateParams,
+}
+
 /// The type of resource that a floating IP is attached to
 #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
 #[serde(rename_all = "snake_case")]
diff --git a/openapi/nexus.json b/openapi/nexus.json
index b0aa84d67a..3d31331a90 100644
--- a/openapi/nexus.json
+++ b/openapi/nexus.json
@@ -964,6 +964,60 @@
           }
         }
       },
+      "put": {
+        "tags": [
+          "floating-ips"
+        ],
+        "summary": "Update floating IP",
+        "operationId": "floating_ip_update",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "floating_ip",
+            "description": "Name or ID of the floating IP",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/NameOrId"
+            }
+          },
+          {
+            "in": "query",
+            "name": "project",
+            "description": "Name or ID of the project",
+            "schema": {
+              "$ref": "#/components/schemas/NameOrId"
+            }
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/FloatingIpUpdate"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/FloatingIp"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      },
       "delete": {
         "tags": [
           "floating-ips"
@@ -11919,6 +11973,24 @@
           "items"
         ]
       },
+      "FloatingIpUpdate": {
+        "description": "Updateable identity-related parameters",
+        "type": "object",
+        "properties": {
+          "description": {
+            "nullable": true,
+            "type": "string"
+          },
+          "name": {
+            "nullable": true,
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Name"
+              }
+            ]
+          }
+        }
+      },
       "Group": {
         "description": "View of a Group",
         "type": "object",

From ac78673a8de8475213ea6c8d32c93473e9c704cf Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 1 Mar 2024 14:10:20 -0800
Subject: [PATCH 062/157] Use the right error conversion to external::Error for
 service errors (#5178)

Service errors already define their own conversion to external errors:


https://github.com/oxidecomputer/omicron/blob/f6efad4126986f72d3bedfdc04cb4ed30a926f0b/sled-agent/src/services.rs#L286

But this conversion was not used by the Sled Agent because it treated
ALL `sled_agent::Error` types as internal errors:


https://github.com/oxidecomputer/omicron/blob/f6efad4126986f72d3bedfdc04cb4ed30a926f0b/sled-agent/src/sled_agent.rs#L154-L160

This PR starts converting better error types by enabling dispatch to
more specific error converters.
---
 sled-agent/src/sled_agent.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs
index 1a634a6346..07c4dbbf4e 100644
--- a/sled-agent/src/sled_agent.rs
+++ b/sled-agent/src/sled_agent.rs
@@ -153,8 +153,12 @@ pub enum Error {
 
 impl From<Error> for omicron_common::api::external::Error {
     fn from(err: Error) -> Self {
-        omicron_common::api::external::Error::InternalError {
-            internal_message: err.to_string(),
+        match err {
+            // Service errors can convert themselves into the external error
+            Error::Services(err) => err.into(),
+            _ => omicron_common::api::external::Error::InternalError {
+                internal_message: err.to_string(),
+            },
         }
     }
 }

From 18cb8ed2382eb782ebf6f522289e355b0441583c Mon Sep 17 00:00:00 2001
From: Rain <rain@oxide.computer>
Date: Fri, 1 Mar 2024 17:08:23 -0800
Subject: [PATCH 063/157] [meta] patch whoami (#5184)

Patch it to fix memory corruption on illumos.
---
 Cargo.lock | 29 +++++++++++++++++++++++++++--
 Cargo.toml |  5 +++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index c3fc5d86cf..7f3ea50279 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -883,6 +883,12 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "cfg_aliases"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
+
 [[package]]
 name = "chacha20"
 version = "0.9.1"
@@ -4623,6 +4629,18 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "nix"
+version = "0.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4"
+dependencies = [
+ "bitflags 2.4.0",
+ "cfg-if",
+ "cfg_aliases",
+ "libc",
+]
+
 [[package]]
 name = "nodrop"
 version = "0.1.14"
@@ -10138,6 +10156,12 @@ version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
+[[package]]
+name = "wasite"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.87"
@@ -10247,9 +10271,10 @@ dependencies = [
 [[package]]
 name = "whoami"
 version = "1.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50"
+source = "git+https://github.com/oxidecomputer/whoami?branch=use-nix#da2a26e695849f1ffc394627ba27a6a473650705"
 dependencies = [
+ "nix 0.28.0",
+ "wasite",
  "wasm-bindgen",
  "web-sys",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index e19ed35c32..6f23ab929c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -621,6 +621,11 @@ opt-level = 3
 git = 'https://github.com/oxidecomputer/pq-sys'
 branch = "oxide/omicron"
 
+# See https://github.com/oxidecomputer/security-operations/issues/43.
+[patch.crates-io.whoami]
+git = "https://github.com/oxidecomputer/whoami"
+branch = "use-nix"
+
 # Using the workspace-hack via this patch directive means that it only applies
 # while building within this workspace. If another workspace imports a crate
 # from here via a git dependency, it will not have the workspace-hack applied

From dcd3d9efcadc6ff1f6ac4a34726f41d62bec296f Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Sat, 2 Mar 2024 09:49:40 +0000
Subject: [PATCH 064/157] chore(deps): update taiki-e/install-action digest to
 c6ffb58 (#5174)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [taiki-e/install-action](https://togithub.com/taiki-e/install-action)
| action | digest | [`45dfafe` ->
`c6ffb58`](https://togithub.com/taiki-e/install-action/compare/45dfafe...c6ffb58)
|

---

### Configuration

📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone
America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone
America/Los_Angeles.

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update
again.

---

- [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check
this box

---

This PR has been generated by [Renovate
Bot](https://togithub.com/renovatebot/renovate).

<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNy4yMjEuMCIsInVwZGF0ZWRJblZlciI6IjM3LjIyMy4wIiwidGFyZ2V0QnJhbmNoIjoibWFpbiJ9-->

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index 49914f36db..e3e032806c 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@45dfafe7696c1865602c71c94d46a16a45f8e88b # v2
+        uses: taiki-e/install-action@c6ffb5827ec9fecafc811f72ec5d7854680dc6ca # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From 77669cdc49ebf07e451ee77109d0567129db3174 Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Mon, 4 Mar 2024 11:49:11 -0500
Subject: [PATCH 065/157] Log all possible sled targets when creating a
 reservation (#5182)

We hope this gives us some insight into
https://github.com/oxidecomputer/omicron/issues/5181.
---
 nexus/db-queries/src/db/datastore/sled.rs | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs
index 8809f4d60d..8d812f69cc 100644
--- a/nexus/db-queries/src/db/datastore/sled.rs
+++ b/nexus/db-queries/src/db/datastore/sled.rs
@@ -247,11 +247,20 @@ impl DataStore {
                     }
 
                     sql_function!(fn random() -> diesel::sql_types::Float);
+
+                    // We only actually care about one target here, so this
+                    // query should have a `.limit(1)` attached. We fetch all
+                    // sled targets to leave additional debugging information in
+                    // the logs, for now.
                     let sled_targets = sled_targets
                         .order(random())
-                        .limit(1)
                         .get_results_async::<Uuid>(&conn)
                         .await?;
+                    info!(
+                        opctx.log,
+                        "found {} available sled targets", sled_targets.len();
+                        "sled_ids" => ?sled_targets,
+                    );
 
                     if sled_targets.is_empty() {
                         return Err(err.bail(SledReservationError::NotFound));

From 9f51dcbeb19ac5d317477c1e5553d13f4ad329d5 Mon Sep 17 00:00:00 2001
From: Andy Fiddaman <andy@oxidecomputer.com>
Date: Mon, 4 Mar 2024 18:15:05 +0000
Subject: [PATCH 066/157] Upgrade clickhouse from v22.8.9.24 to v23.8.7.24
 (#5127)

---
 oximeter/db/src/client.rs                 |   8 +-
 oximeter/db/src/configs/keeper_config.xml |   2 +-
 smf/clickhouse_keeper/keeper_config.xml   |   2 +-
 smf/clickhouse_keeper/method_script.sh    |  30 ++---
 smf/profile/profile                       |   4 +-
 test-utils/src/dev/clickhouse.rs          | 131 ++++++++++++++--------
 tools/ci_download_clickhouse              |   4 +-
 tools/clickhouse_checksums                |   6 +-
 tools/clickhouse_version                  |   2 +-
 tools/install_builder_prerequisites.sh    |   6 +-
 10 files changed, 122 insertions(+), 73 deletions(-)

diff --git a/oximeter/db/src/client.rs b/oximeter/db/src/client.rs
index 2e3029a938..abea11aa06 100644
--- a/oximeter/db/src/client.rs
+++ b/oximeter/db/src/client.rs
@@ -50,6 +50,10 @@ use tokio::fs;
 use tokio::sync::Mutex;
 use uuid::Uuid;
 
+const CLICKHOUSE_DB_MISSING: &'static str = "Database oximeter does not exist";
+const CLICKHOUSE_DB_VERSION_MISSING: &'static str =
+    "Table oximeter.version does not exist";
+
 #[usdt::provider(provider = "clickhouse_client")]
 mod probes {
     fn query__start(_: &usdt::UniqueId, sql: &str) {}
@@ -856,10 +860,10 @@ impl Client {
             })?,
             Err(Error::Database(err))
                 // Case 1: The database has not been created.
-                if err.contains("Database oximeter doesn't exist") ||
+                if err.contains(CLICKHOUSE_DB_MISSING) ||
                 // Case 2: The database has been created, but it's old (exists
                 // prior to the version table).
-                    err.contains("Table oximeter.version doesn't exist") =>
+                    err.contains(CLICKHOUSE_DB_VERSION_MISSING) =>
             {
                 warn!(self.log, "oximeter database does not exist, or is out-of-date");
                 0
diff --git a/oximeter/db/src/configs/keeper_config.xml b/oximeter/db/src/configs/keeper_config.xml
index 19ab99f909..ae9d2a9cc5 100644
--- a/oximeter/db/src/configs/keeper_config.xml
+++ b/oximeter/db/src/configs/keeper_config.xml
@@ -9,7 +9,7 @@
     </logger>
 
     <!-- To allow all use :: -->
-    <listen_host from_env="CH_LISTEN_ADDR"/> 
+    <listen_host from_env="CH_LISTEN_ADDR"/>
     <path replace="true" from_env="CH_DATASTORE"/>
 
     <keeper_server>
diff --git a/smf/clickhouse_keeper/keeper_config.xml b/smf/clickhouse_keeper/keeper_config.xml
index ec114694cc..2d64c2c0a0 100644
--- a/smf/clickhouse_keeper/keeper_config.xml
+++ b/smf/clickhouse_keeper/keeper_config.xml
@@ -9,7 +9,7 @@
     </logger>
 
     <!-- To allow all use :: -->
-    <listen_host from_env="CH_LISTEN_ADDR"/> 
+    <listen_host from_env="CH_LISTEN_ADDR"/>
     <path replace="true" from_env="CH_DATASTORE"/>
 
     <keeper_server>
diff --git a/smf/clickhouse_keeper/method_script.sh b/smf/clickhouse_keeper/method_script.sh
index 8499e0001f..a017409a98 100755
--- a/smf/clickhouse_keeper/method_script.sh
+++ b/smf/clickhouse_keeper/method_script.sh
@@ -39,20 +39,22 @@ KEEPER_HOST_01="${keepers[0]}"
 KEEPER_HOST_02="${keepers[1]}"
 KEEPER_HOST_03="${keepers[2]}"
 
-# Generate unique reproduceable number IDs by removing letters from KEEPER_IDENTIFIER_*
-# Keeper IDs must be numbers, and they cannot be reused (i.e. when a keeper node is 
-# unrecoverable the ID must be changed to something new). 
-# By trimming the hosts we can make sure all keepers will always be up to date when 
-# a new keeper is spun up. Clickhouse does not allow very large numbers, so we will
-# be reducing to 7 characters. This should be enough entropy given the small amount
-# of keepers we have.
+# Generate unique reproduceable number IDs by removing letters from
+# KEEPER_IDENTIFIER_* Keeper IDs must be numbers, and they cannot be reused
+# (i.e. when a keeper node is unrecoverable the ID must be changed to something
+# new). By trimming the hosts we can make sure all keepers will always be up to
+# date when a new keeper is spun up. Clickhouse does not allow very large
+# numbers, so we will be reducing to 7 characters. This should be enough
+# entropy given the small amount of keepers we have.
 KEEPER_ID_01="$( echo "${KEEPER_HOST_01}" | tr -dc [:digit:] | cut -c1-7)"
 KEEPER_ID_02="$( echo "${KEEPER_HOST_02}" | tr -dc [:digit:] | cut -c1-7)"
 KEEPER_ID_03="$( echo "${KEEPER_HOST_03}" | tr -dc [:digit:] | cut -c1-7)"
 
-# Identify the node type this is as this will influence how the config is constructed
-# TODO(https://github.com/oxidecomputer/omicron/issues/3824): There are probably much better ways to do this service name lookup, but this works
-# for now. The services contain the same IDs as the hostnames.
+# Identify the node type this is as this will influence how the config is
+# constructed
+# TODO(https://github.com/oxidecomputer/omicron/issues/3824): There are
+# probably much better ways to do this service name lookup, but this works for
+# now. The services contain the same IDs as the hostnames.
 KEEPER_SVC="$(zonename | tr -dc [:digit:] | cut -c1-7)"
 if [[ $KEEPER_ID_01 == $KEEPER_SVC ]]
 then
@@ -68,9 +70,9 @@ else
     exit "$SMF_EXIT_ERR_CONFIG"
 fi
 
-# Setting environment variables this way is best practice, but has the downside of
-# obscuring the field values to anyone ssh=ing into the zone.  To mitigate this,
-# we will be saving them to ${DATASTORE}/config_env_vars
+# Setting environment variables this way is best practice, but has the downside
+# of obscuring the field values to anyone ssh=ing into the zone.  To mitigate
+# this, we will be saving them to ${DATASTORE}/config_env_vars
 export CH_LOG="${DATASTORE}/clickhouse-keeper.log"
 export CH_ERROR_LOG="${DATASTORE}/clickhouse-keeper.err.log"
 export CH_LISTEN_ADDR=${LISTEN_ADDR}
@@ -103,7 +105,7 @@ CH_KEEPER_HOST_03="${CH_KEEPER_HOST_03}""
 
 echo $content >> "${DATASTORE}/config_env_vars"
 
-# The clickhouse binary must be run from within the directory that contains it. 
+# The clickhouse binary must be run from within the directory that contains it.
 # Otherwise, it does not automatically detect the configuration files, nor does
 # it append them when necessary
 cd /opt/oxide/clickhouse_keeper/
diff --git a/smf/profile/profile b/smf/profile/profile
index 73256cd6fd..a8378f7403 100644
--- a/smf/profile/profile
+++ b/smf/profile/profile
@@ -12,8 +12,8 @@ case "$HOSTNAME" in
     oxz_crucible*)
 	PATH+=:/opt/oxide/crucible/bin
 	;;
-    oxz_clockhouse*)
-	PATH+=:/opt/oxide/clickhouse
+    oxz_clickhouse*)
+	PATH+=:/opt/oxide/clickhouse:/opt/oxide/clickhouse_keeper
 	;;
     oxz_external_dns*|oxz_internal_dns*)
 	PATH+=:/opt/oxide/dns-server/bin
diff --git a/test-utils/src/dev/clickhouse.rs b/test-utils/src/dev/clickhouse.rs
index 220662d9bb..8c415d949e 100644
--- a/test-utils/src/dev/clickhouse.rs
+++ b/test-utils/src/dev/clickhouse.rs
@@ -29,6 +29,19 @@ const CLICKHOUSE_TIMEOUT: Duration = Duration::from_secs(30);
 // Timeout used when starting a ClickHouse keeper subprocess.
 const CLICKHOUSE_KEEPER_TIMEOUT: Duration = Duration::from_secs(30);
 
+// The string to look for in a keeper log file that indicates that the server
+// is ready.
+const KEEPER_READY: &'static str = "Server initialized, waiting for quorum";
+
+// The string to look for in a clickhouse log file that indicates that the
+// server is ready.
+const CLICKHOUSE_READY: &'static str =
+    "<Information> Application: Ready for connections";
+
+// The string to look for in a clickhouse log file when trying to determine the
+// port number on which it is listening.
+const CLICKHOUSE_PORT: &'static str = "Application: Listening for http://[::1]";
+
 /// A `ClickHouseInstance` is used to start and manage a ClickHouse single node server process.
 #[derive(Debug)]
 pub struct ClickHouseInstance {
@@ -157,9 +170,11 @@ impl ClickHouseInstance {
             .env("CH_REPLICA_NUMBER", r_number)
             .env("CH_REPLICA_HOST_01", "::1")
             .env("CH_REPLICA_HOST_02", "::1")
-            // ClickHouse servers have a small quirk, where when setting the keeper hosts as IPv6 localhost
-            // addresses in the replica configuration file, they must be wrapped in square brackets
-            // Otherwise, when running any query, a "Service not found" error appears.
+            // ClickHouse servers have a small quirk, where when setting the
+            // keeper hosts as IPv6 localhost addresses in the replica
+            // configuration file, they must be wrapped in square brackets
+            // Otherwise, when running any query, a "Service not found" error
+            // appears.
             .env("CH_KEEPER_HOST_01", "[::1]")
             .env("CH_KEEPER_HOST_02", "[::1]")
             .env("CH_KEEPER_HOST_03", "[::1]")
@@ -171,7 +186,12 @@ impl ClickHouseInstance {
         let data_path = data_dir.root_path().to_path_buf();
         let address = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), port);
 
-        let result = wait_for_ready(data_dir.log_path()).await;
+        let result = wait_for_ready(
+            data_dir.log_path(),
+            CLICKHOUSE_TIMEOUT,
+            CLICKHOUSE_READY,
+        )
+        .await;
         match result {
             Ok(()) => Ok(Self {
                 data_dir: Some(data_dir),
@@ -192,9 +212,9 @@ impl ClickHouseInstance {
         k_id: u16,
         config_path: PathBuf,
     ) -> Result<Self, anyhow::Error> {
-        // We assume that only 3 keepers will be run, and the ID of the keeper can only
-        // be one of "1", "2" or "3". This is to avoid having to pass the IDs of the
-        // other keepers as part of the function's parameters.
+        // We assume that only 3 keepers will be run, and the ID of the keeper
+        // can only be one of "1", "2" or "3". This is to avoid having to pass
+        // the IDs of the other keepers as part of the function's parameters.
         if ![1, 2, 3].contains(&k_id) {
             return Err(ClickHouseError::InvalidKeeperId.into());
         }
@@ -240,7 +260,12 @@ impl ClickHouseInstance {
         let data_path = data_dir.root_path().to_path_buf();
         let address = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), port);
 
-        let result = wait_for_ready(data_dir.keeper_log_path()).await;
+        let result = wait_for_ready(
+            data_dir.keeper_log_path(),
+            CLICKHOUSE_KEEPER_TIMEOUT,
+            KEEPER_READY,
+        )
+        .await;
         match result {
             Ok(()) => Ok(Self {
                 data_dir: Some(data_dir),
@@ -641,8 +666,9 @@ pub async fn wait_for_port(
     Ok(p)
 }
 
-// Parse the ClickHouse log file at the given path, looking for a line reporting the port number of
-// the HTTP server. This is only used if the port is chosen by the OS, not the caller.
+// Parse the ClickHouse log file at the given path, looking for a line
+// reporting the port number of the HTTP server. This is only used if the port
+// is chosen by the OS, not the caller.
 async fn discover_local_listening_port(
     path: &Utf8Path,
     timeout: Duration,
@@ -655,23 +681,21 @@ async fn discover_local_listening_port(
 
 // Parse the clickhouse log for a port number.
 //
-// NOTE: This function loops forever until the expected line is found. It should be run under a
-// timeout, or some other mechanism for cancelling it.
+// NOTE: This function loops forever until the expected line is found. It
+// should be run under a timeout, or some other mechanism for cancelling it.
 async fn find_clickhouse_port_in_log(
     path: &Utf8Path,
 ) -> Result<u16, ClickHouseError> {
     let mut reader = BufReader::new(File::open(path).await?);
-    const NEEDLE: &str =
-        "<Information> Application: Listening for http://[::1]";
     let mut lines = reader.lines();
     loop {
         let line = lines.next_line().await?;
         match line {
             Some(line) => {
-                if let Some(needle_start) = line.find(NEEDLE) {
+                if let Some(needle_start) = line.find(CLICKHOUSE_PORT) {
                     // Our needle ends with `http://[::1]`; we'll split on the
                     // colon we expect to follow it to find the port.
-                    let address_start = needle_start + NEEDLE.len();
+                    let address_start = needle_start + CLICKHOUSE_PORT.len();
                     return line[address_start..]
                         .trim()
                         .split(':')
@@ -697,11 +721,12 @@ async fn find_clickhouse_port_in_log(
 // Wait for the ClickHouse log file to report it is ready to receive connections
 pub async fn wait_for_ready(
     log_path: Utf8PathBuf,
+    timeout: Duration,
+    needle: &str,
 ) -> Result<(), anyhow::Error> {
     let p = poll::wait_for_condition(
         || async {
-            let result =
-                discover_ready(&log_path, CLICKHOUSE_KEEPER_TIMEOUT).await;
+            let result = discover_ready(&log_path, timeout, needle).await;
             match result {
                 Ok(ready) => Ok(ready),
                 Err(e) => {
@@ -721,40 +746,41 @@ pub async fn wait_for_ready(
             }
         },
         &Duration::from_millis(500),
-        &CLICKHOUSE_KEEPER_TIMEOUT,
+        &timeout,
     )
     .await
     .context("waiting to discover if ClickHouse is ready for connections")?;
     Ok(p)
 }
 
-// Parse the ClickHouse log file at the given path, looking for a line reporting that the server
-// is ready for connections.
+// Parse the ClickHouse log file at the given path, looking for a line
+// reporting that the server is ready for connections.
 async fn discover_ready(
     path: &Utf8Path,
     timeout: Duration,
+    needle: &str,
 ) -> Result<(), ClickHouseError> {
     let timeout = Instant::now() + timeout;
-    tokio::time::timeout_at(timeout, clickhouse_ready_from_log(path))
+    tokio::time::timeout_at(timeout, clickhouse_ready_from_log(path, needle))
         .await
         .map_err(|_| ClickHouseError::Timeout)?
 }
 
 // Parse the clickhouse log to know if the server is ready for connections.
 //
-// NOTE: This function loops forever until the expected line is found. It should be run under a
-// timeout, or some other mechanism for cancelling it.
+// NOTE: This function loops forever until the expected line is found. It
+// should be run under a timeout, or some other mechanism for cancelling it.
 async fn clickhouse_ready_from_log(
     path: &Utf8Path,
+    needle: &str,
 ) -> Result<(), ClickHouseError> {
     let mut reader = BufReader::new(File::open(path).await?);
-    const READY: &str = "<Information> Application: Ready for connections";
     let mut lines = reader.lines();
     loop {
         let line = lines.next_line().await?;
         match line {
             Some(line) => {
-                if let Some(_) = line.find(READY) {
+                if let Some(_) = line.find(needle) {
                     return Ok(());
                 }
             }
@@ -775,7 +801,7 @@ async fn clickhouse_ready_from_log(
 mod tests {
     use super::{
         discover_local_listening_port, discover_ready, ClickHouseError,
-        CLICKHOUSE_TIMEOUT,
+        CLICKHOUSE_PORT, CLICKHOUSE_READY, CLICKHOUSE_TIMEOUT,
     };
     use camino_tempfile::NamedUtf8TempFile;
     use std::process::Stdio;
@@ -824,14 +850,16 @@ mod tests {
         writeln!(file, "A garbage line").unwrap();
         writeln!(
             file,
-            "2023.07.31 20:12:38.936192 [ 82373 ] <Information> Application: Ready for connections.",
+            "2023.07.31 20:12:38.936192 [ 82373 ] <Information> {}",
+            CLICKHOUSE_READY,
         )
         .unwrap();
         writeln!(file, "Another garbage line").unwrap();
         file.flush().unwrap();
 
         assert!(matches!(
-            discover_ready(file.path(), CLICKHOUSE_TIMEOUT).await,
+            discover_ready(file.path(), CLICKHOUSE_TIMEOUT, CLICKHOUSE_READY)
+                .await,
             Ok(())
         ));
     }
@@ -849,17 +877,23 @@ mod tests {
         writeln!(file, "Another garbage line").unwrap();
         file.flush().unwrap();
         assert!(matches!(
-            discover_ready(file.path(), Duration::from_secs(1)).await,
+            discover_ready(
+                file.path(),
+                Duration::from_secs(1),
+                CLICKHOUSE_READY
+            )
+            .await,
             Err(ClickHouseError::Timeout {})
         ));
     }
 
     // A regression test for #131.
     //
-    // The function `discover_local_listening_port` initially read from the log file until EOF, but
-    // there's no guarantee that ClickHouse has written the port we're searching for before the
-    // reader consumes the whole file. This test confirms that the file is read until the line is
-    // found, ignoring EOF, at least until the timeout is hit.
+    // The function `discover_local_listening_port` initially read from the log
+    // file until EOF, but there's no guarantee that ClickHouse has written the
+    // port we're searching for before the reader consumes the whole file. This
+    // test confirms that the file is read until the line is found, ignoring
+    // EOF, at least until the timeout is hit.
     #[tokio::test]
     async fn test_discover_local_listening_port_slow_write() {
         // In this case the writer is slightly "slower" than the reader.
@@ -880,9 +914,11 @@ mod tests {
         assert!(read_log_file(reader_timeout, writer_interval).await.is_err());
     }
 
-    // Implementation of the above tests, simulating simultaneous reading/writing of the log file
+    // Implementation of the above tests, simulating simultaneous
+    // reading/writing of the log file
     //
-    // This uses Tokio's test utilities to manage time, rather than relying on timeouts.
+    // This uses Tokio's test utilities to manage time, rather than relying on
+    // timeouts.
     async fn read_log_file(
         reader_timeout: Duration,
         writer_interval: Duration,
@@ -904,12 +940,14 @@ mod tests {
 
         // Start a task that slowly writes lines to the log file.
         //
-        // NOTE: This looks overly complicated, and it is. We have to wrap this in a mutex because
-        // both this function, and the writer task we're spawning, need access to the file. They
-        // may complete in any order, and so it's not possible to give one of them ownership over
-        // the `NamedTempFile`. If the owning task completes, that may delete the file before the
-        // other task accesses it. So we need interior mutability (because one of the references is
-        // mutable for writing), and _this_ scope must own it.
+        // NOTE: This looks overly complicated, and it is. We have to wrap this
+        // in a mutex because both this function, and the writer task we're
+        // spawning, need access to the file. They may complete in any order,
+        // and so it's not possible to give one of them ownership over the
+        // `NamedTempFile`. If the owning task completes, that may delete the
+        // file before the other task accesses it. So we need interior
+        // mutability (because one of the references is mutable for writing),
+        // and _this_ scope must own it.
         let file = Arc::new(Mutex::new(NamedUtf8TempFile::new()?));
         let path = file.lock().await.path().to_path_buf();
         let writer_file = file.clone();
@@ -927,13 +965,13 @@ mod tests {
             // (https://github.com/oxidecomputer/omicron/issues/3580).
             write_and_wait(
                 &mut file,
-                "<Information> Application: List".to_string(),
+                (&CLICKHOUSE_PORT[..30]).to_string(),
                 writer_interval,
             )
             .await;
             write_and_wait(
                 &mut file,
-                format!("ening for http://[::1]:{}\n", EXPECTED_PORT),
+                format!("{}:{}\n", &CLICKHOUSE_PORT[30..], EXPECTED_PORT),
                 writer_interval,
             )
             .await;
@@ -950,8 +988,9 @@ mod tests {
 
         // "Run" the test.
         //
-        // Note that the futures for the reader/writer tasks must be pinned to the stack, so that
-        // they may be polled on multiple passes through the select loop without consuming them.
+        // Note that the futures for the reader/writer tasks must be pinned to
+        // the stack, so that they may be polled on multiple passes through the
+        // select loop without consuming them.
         tokio::pin!(writer_task);
         tokio::pin!(reader_task);
         let mut poll_writer = true;
diff --git a/tools/ci_download_clickhouse b/tools/ci_download_clickhouse
index 675566fad7..93201a772b 100755
--- a/tools/ci_download_clickhouse
+++ b/tools/ci_download_clickhouse
@@ -19,7 +19,8 @@ DOWNLOAD_DIR="$TARGET_DIR/downloads"
 # Location where the final clickhouse directory should end up.
 DEST_DIR="./$TARGET_DIR/clickhouse"
 
-# If you change this, you must also update the md5sums below
+# If you change the version in clickhouse_version, you must also update the
+# md5sums in clickhouse_checksums
 CIDL_VERSION="$(cat "$SOURCE_DIR/clickhouse_version")"
 source "$SOURCE_DIR/clickhouse_checksums"
 
@@ -113,7 +114,6 @@ function configure_os
 			CIDL_PLATFORM="illumos"
 			CIDL_MD5="$CIDL_MD5_ILLUMOS"
 			CIDL_MD5FUNC="do_md5sum"
-			CIDL_DASHREV=-1
 			;;
 		*)
 			fail "unsupported OS: $1"
diff --git a/tools/clickhouse_checksums b/tools/clickhouse_checksums
index 92a5237301..afddb15cab 100644
--- a/tools/clickhouse_checksums
+++ b/tools/clickhouse_checksums
@@ -1,3 +1,3 @@
-CIDL_MD5_DARWIN="20603974a929926591fca70ff1df0e45"
-CIDL_MD5_LINUX="ea909519bd9d989fd5d090fd9bdd42f1"
-CIDL_MD5_ILLUMOS="7702939ce5b4b51846a1ba39f1392306"
+CIDL_MD5_DARWIN="3e20c3284b7e6b0cfcfedf622ecf547a"
+CIDL_MD5_LINUX="f6c30a25a86deac3bad6c50dcf758fd5"
+CIDL_MD5_ILLUMOS="409222de8ecb59e5dd97dcc942ccdffe"
diff --git a/tools/clickhouse_version b/tools/clickhouse_version
index 93b98bf738..0a988071cd 100644
--- a/tools/clickhouse_version
+++ b/tools/clickhouse_version
@@ -1 +1 @@
-v22.8.9.24
\ No newline at end of file
+v23.8.7.24
diff --git a/tools/install_builder_prerequisites.sh b/tools/install_builder_prerequisites.sh
index 0427629960..5fa8ec11ba 100755
--- a/tools/install_builder_prerequisites.sh
+++ b/tools/install_builder_prerequisites.sh
@@ -128,6 +128,7 @@ function install_packages {
     fi
   elif [[ "${HOST_OS}" == "SunOS" ]]; then
     CLANGVER=15
+    RTVER=13
     PGVER=13
     packages=(
       "pkg:/package/pkg"
@@ -135,6 +136,8 @@ function install_packages {
       "library/postgresql-$PGVER"
       "pkg-config"
       "library/libxmlsec1"
+      "system/library/gcc-runtime@$RTVER"
+      "system/library/g++-runtime@$RTVER"
       # "bindgen leverages libclang to preprocess, parse, and type check C and C++ header files."
       "pkg:/ooce/developer/clang-$CLANGVER"
       "system/library/gcc-runtime"
@@ -159,7 +162,8 @@ function install_packages {
     }
 
     pkg mediator -a
-    pkg list -v "${packages[@]}"
+    pkg publisher
+    pkg list -afv "${packages[@]}"
   elif [[ "${HOST_OS}" == "Darwin" ]]; then
     packages=(
       'coreutils'

From 55cc240236a41fa0288aaa0c7a1ea3b3b5e66c42 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxidecomputer.com>
Date: Mon, 4 Mar 2024 10:39:18 -0800
Subject: [PATCH 067/157] Update Propolis and Crucible to latest (#5187)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Crucible changes:
Per client, queue-based backpressure (#1186)
A builder for the Downstairs Downstairs struct. (#1152) Update Rust to
v1.76.0 (#1153)
Deactivate the read only parent after a scrub (#1180) Start byte-based
backpressure earlier (#1179)
Tweak CI scripts to fix warnings (#1178)
Make `gw_ds_complete` less public (#1175)
Verify extent under repair is valid after copying files (#1159) Remove
individual panic setup, use global panic settings (#1174) [smf] Use new
zone network config service (#1096)
Move a few methods into downstairs (#1160)
Remove extra clone in upstairs read (#1163)
Make `crucible-downstairs` not depend on upstairs (#1165) Update Rust
crate rusqlite to 0.31 (#1171)
Update Rust crate reedline to 0.29.0 (#1170)
Update Rust crate clap to 4.5 (#1169)
Update Rust crate indicatif to 0.17.8 (#1168)
Update progenitor to bc0bb4b (#1164)
Do not 500 on snapshot delete for deleted region (#1162) Drop jobs from
Offline downstairs. (#1157)
`Mutex<Work>` → `Work` (#1156)
Added a contributing.md (#1158)
Remove ExtentFlushClose::source_downstairs (#1154) Remove unnecessary
mutexes from Downstairs (#1132)

Propolis changes:
PHD: improve Windows reliability (#651)
Update progenitor and omicron deps
Clean up VMM resource on server shutdown
Remove Inventory mechanism
Update vergen dependency
Properly handle pre/post illumos#16183 fixups
PHD: add `pfexec` to xtask phd-runner invocation (#647) PHD: add Windows
Server 2016 adapter & improve WS2016/2019 reliability (#646) PHD: use
`clap` for more `cargo xtask phd` args (#645) PHD: several `cargo xtask
phd` CLI fixes (#642)
PHD: Use ZFS clones for file-backed disks (#640)
PHD: improve ctrl-c handling (#634)

Co-authored-by: Alan Hanson <alan@oxide.computer>
---
 Cargo.lock            | 16 ++++++++--------
 Cargo.toml            | 12 ++++++------
 package-manifest.toml | 12 ++++++------
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7f3ea50279..7a491135b5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -463,7 +463,7 @@ dependencies = [
 [[package]]
 name = "bhyve_api"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=488a70c644e332dae98447292a95e45074c8107c#488a70c644e332dae98447292a95e45074c8107c"
+source = "git+https://github.com/oxidecomputer/propolis?rev=a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c#a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c"
 dependencies = [
  "bhyve_api_sys",
  "libc",
@@ -473,7 +473,7 @@ dependencies = [
 [[package]]
 name = "bhyve_api_sys"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=488a70c644e332dae98447292a95e45074c8107c#488a70c644e332dae98447292a95e45074c8107c"
+source = "git+https://github.com/oxidecomputer/propolis?rev=a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c#a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c"
 dependencies = [
  "libc",
  "strum 0.26.1",
@@ -1315,7 +1315,7 @@ dependencies = [
 [[package]]
 name = "crucible-agent-client"
 version = "0.0.1"
-source = "git+https://github.com/oxidecomputer/crucible?rev=796dce526dd7ed7b52a0429a486ccba4a9da1ce5#796dce526dd7ed7b52a0429a486ccba4a9da1ce5"
+source = "git+https://github.com/oxidecomputer/crucible?rev=1c1574fb721f98f2df1b23e3fd27d83be018882e#1c1574fb721f98f2df1b23e3fd27d83be018882e"
 dependencies = [
  "anyhow",
  "chrono",
@@ -1331,7 +1331,7 @@ dependencies = [
 [[package]]
 name = "crucible-pantry-client"
 version = "0.0.1"
-source = "git+https://github.com/oxidecomputer/crucible?rev=796dce526dd7ed7b52a0429a486ccba4a9da1ce5#796dce526dd7ed7b52a0429a486ccba4a9da1ce5"
+source = "git+https://github.com/oxidecomputer/crucible?rev=1c1574fb721f98f2df1b23e3fd27d83be018882e#1c1574fb721f98f2df1b23e3fd27d83be018882e"
 dependencies = [
  "anyhow",
  "chrono",
@@ -1348,7 +1348,7 @@ dependencies = [
 [[package]]
 name = "crucible-smf"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/crucible?rev=796dce526dd7ed7b52a0429a486ccba4a9da1ce5#796dce526dd7ed7b52a0429a486ccba4a9da1ce5"
+source = "git+https://github.com/oxidecomputer/crucible?rev=1c1574fb721f98f2df1b23e3fd27d83be018882e#1c1574fb721f98f2df1b23e3fd27d83be018882e"
 dependencies = [
  "crucible-workspace-hack",
  "libc",
@@ -6623,7 +6623,7 @@ dependencies = [
 [[package]]
 name = "propolis-client"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=488a70c644e332dae98447292a95e45074c8107c#488a70c644e332dae98447292a95e45074c8107c"
+source = "git+https://github.com/oxidecomputer/propolis?rev=a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c#a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c"
 dependencies = [
  "async-trait",
  "base64",
@@ -6644,7 +6644,7 @@ dependencies = [
 [[package]]
 name = "propolis-mock-server"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=488a70c644e332dae98447292a95e45074c8107c#488a70c644e332dae98447292a95e45074c8107c"
+source = "git+https://github.com/oxidecomputer/propolis?rev=a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c#a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c"
 dependencies = [
  "anyhow",
  "atty",
@@ -6674,7 +6674,7 @@ dependencies = [
 [[package]]
 name = "propolis_types"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=488a70c644e332dae98447292a95e45074c8107c#488a70c644e332dae98447292a95e45074c8107c"
+source = "git+https://github.com/oxidecomputer/propolis?rev=a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c#a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c"
 dependencies = [
  "schemars",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index 6f23ab929c..1d701c550e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -182,9 +182,9 @@ cookie = "0.18"
 criterion = { version = "0.5.1", features = [ "async_tokio" ] }
 crossbeam = "0.8"
 crossterm = { version = "0.27.0", features = ["event-stream"] }
-crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "796dce526dd7ed7b52a0429a486ccba4a9da1ce5" }
-crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "796dce526dd7ed7b52a0429a486ccba4a9da1ce5" }
-crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "796dce526dd7ed7b52a0429a486ccba4a9da1ce5" }
+crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "1c1574fb721f98f2df1b23e3fd27d83be018882e" }
+crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "1c1574fb721f98f2df1b23e3fd27d83be018882e" }
+crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "1c1574fb721f98f2df1b23e3fd27d83be018882e" }
 csv = "1.3.0"
 curve25519-dalek = "4"
 datatest-stable = "0.2.3"
@@ -314,9 +314,9 @@ prettyplease = { version = "0.2.16", features = ["verbatim"] }
 proc-macro2 = "1.0"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" }
 progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" }
-bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "488a70c644e332dae98447292a95e45074c8107c" }
-propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "488a70c644e332dae98447292a95e45074c8107c" }
-propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "488a70c644e332dae98447292a95e45074c8107c" }
+bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c" }
+propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c" }
+propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c" }
 proptest = "1.4.0"
 quote = "1.0"
 rand = "0.8.5"
diff --git a/package-manifest.toml b/package-manifest.toml
index c474a52736..065da4bf42 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -441,10 +441,10 @@ only_for_targets.image = "standard"
 # 3. Use source.type = "manual" instead of "prebuilt"
 source.type = "prebuilt"
 source.repo = "crucible"
-source.commit = "fe0c5c7909707a0f826025be4fe2bbf5f6e0206f"
+source.commit = "1c1574fb721f98f2df1b23e3fd27d83be018882e"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image/<commit>/crucible.sha256.txt
-source.sha256 = "5da4f93b16fc7c0f3cc3a67919dbaa3f143cc07b703183a236f5c98b61504d15"
+source.sha256 = "a14ce9d6b17c6c8427d526e044bb1196ee81316674936873ac9dc1d7ca51459d"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -453,10 +453,10 @@ service_name = "crucible_pantry_prebuilt"
 only_for_targets.image = "standard"
 source.type = "prebuilt"
 source.repo = "crucible"
-source.commit = "fe0c5c7909707a0f826025be4fe2bbf5f6e0206f"
+source.commit = "1c1574fb721f98f2df1b23e3fd27d83be018882e"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image/<commit>/crucible-pantry.sha256.txt
-source.sha256 = "ed5027cc37c5ba4f2b9a568528f5bb49deedccaaa60bd770311c7bface6aa02b"
+source.sha256 = "8839a3748cd2926f61244c99344d0802e59ea4673a5f31a2aaad5d759176aa80"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -468,10 +468,10 @@ service_name = "propolis-server"
 only_for_targets.image = "standard"
 source.type = "prebuilt"
 source.repo = "propolis"
-source.commit = "c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d"
+source.commit = "a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image/<commit>/propolis-server.sha256.txt
-source.sha256 = "0203b7f702377c877c4132851ca102d68cd8fd2c20e4fd5b59d950cbb07fd9ff"
+source.sha256 = "7998a71c4e905a4bf9ef2d2863b7695d0bb4d1b34322aa3ce20d173d4a7ee77a"
 output.type = "zone"
 
 [package.mg-ddm-gz]

From 7f43a7e3f1d8290fd721faa0d13136a2c6c0c8e1 Mon Sep 17 00:00:00 2001
From: Laura Abbott <labbott@users.noreply.github.com>
Date: Mon, 4 Mar 2024 15:44:12 -0500
Subject: [PATCH 068/157] Bump PSC/Sidecar SP to v1.0.10 (#5188)

This syncs them with the gimlet
---
 tools/hubris_checksums | 8 ++++----
 tools/hubris_version   | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tools/hubris_checksums b/tools/hubris_checksums
index 65a2739fcd..83a22f9a43 100644
--- a/tools/hubris_checksums
+++ b/tools/hubris_checksums
@@ -2,7 +2,7 @@
 9cba42b8d6c4dbd1bcc805b44e14e6bac7fa590f1faaaa47664f7aa254e91602  build-gimlet-d-image-default-v1.0.10.zip
 3ace1b1d692e3c23815665b932dadb42b58abc1942fdd4564c1884a13fa0f0cb  build-gimlet-e-image-default-v1.0.10.zip
 b3b05e1926bb0f66be8587a67084c76cd760da5db0f1e288c265a03cf80637e9  build-gimlet-f-image-default-v1.0.10.zip
-0fa33a057dec7301280338cf5a80a5198b5d95475dba350a807979f1a53ce5c7  build-psc-b-image-default-v1.0.9.zip
-88e7aa91fe4f976753ccfc8ba9a57a0521c54d9ef35504cf16e7b1ce78c87465  build-psc-c-image-default-v1.0.9.zip
-cfbf21fef84b38c2a5ca3c0b45703c20c2605c6bf5d599b53b26e3a2d09d756d  build-sidecar-b-image-default-v1.0.9.zip
-c98b89993f1a9b5219bbabf06897db2f92d60f6a3afb4374cb8e6e79d57fdfe4  build-sidecar-c-image-default-v1.0.9.zip
+2d1313c4f282c49a411c32c643347dcb63bb063ae47539b2a1e4003da5a20402  build-psc-b-image-default-v1.0.10.zip
+bb16ee95be863fe5bb7a155d07732a42fc56178ce08bcb9323f46315e9616e62  build-psc-c-image-default-v1.0.10.zip
+97763288ca52a779334422cd7f3aff892341f38f79d1022a1d7ff49b85fb8356  build-sidecar-b-image-default-v1.0.10.zip
+814098557e67f5f8720e95fb76b4f694e7edbd26cd3aa120266b4238641d7029  build-sidecar-c-image-default-v1.0.10.zip
diff --git a/tools/hubris_version b/tools/hubris_version
index cb9e30d980..8098ff8759 100644
--- a/tools/hubris_version
+++ b/tools/hubris_version
@@ -1 +1 @@
-TAGS=(gimlet-v1.0.10 psc-v1.0.9 sidecar-v1.0.9)
+TAGS=(gimlet-v1.0.10 psc-v1.0.10 sidecar-v1.0.10)

From 5cf0a29e5630ebb37df54d04bbdacd2e9a6a9a9d Mon Sep 17 00:00:00 2001
From: David Crespo <david-crespo@users.noreply.github.com>
Date: Mon, 4 Mar 2024 15:37:12 -0600
Subject: [PATCH 069/157] Bump web console (React Canary) (#5193)

https://github.com/oxidecomputer/console/compare/25fb506d...269c2c82

* [269c2c82](https://github.com/oxidecomputer/console/commit/269c2c82)
standardize on ~/ instead of app/
* [e7ff4efb](https://github.com/oxidecomputer/console/commit/e7ff4efb)
eliminate @oxide/ui alias and barrel file
* [cd8df25a](https://github.com/oxidecomputer/console/commit/cd8df25a)
oxidecomputer/console#2028
* [a72768f8](https://github.com/oxidecomputer/console/commit/a72768f8)
oxidecomputer/console#2027
* [67f6d908](https://github.com/oxidecomputer/console/commit/67f6d908)
oxidecomputer/console#2022
* [8f1133ca](https://github.com/oxidecomputer/console/commit/8f1133ca)
oxidecomputer/console#2020
* [555c2f65](https://github.com/oxidecomputer/console/commit/555c2f65)
oxidecomputer/console#2017
* [8690be44](https://github.com/oxidecomputer/console/commit/8690be44)
oxidecomputer/console#2018
* [fa0bcf7a](https://github.com/oxidecomputer/console/commit/fa0bcf7a)
oxidecomputer/console#2016
---
 tools/console_version | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/console_version b/tools/console_version
index f5534b5c50..24d16b17eb 100644
--- a/tools/console_version
+++ b/tools/console_version
@@ -1,2 +1,2 @@
-COMMIT="25fb506d0527f5508dc90b118672c0bf8ce14dd3"
-SHA2="1083f693c66c275d0cf76e120aa923a51c3188dd9a7996eb51dc0fe4f611507a"
+COMMIT="269c2c82018d09d442a932474d84366a54837068"
+SHA2="7a6a04f43c6189384065675ebb7c429de2b8369ffbb7c1989a13aafe779be2d6"

From d9111d3ccc92830a39a7f1aa08bc2920d4e0c9b3 Mon Sep 17 00:00:00 2001
From: Rain <rain@oxide.computer>
Date: Mon, 4 Mar 2024 19:37:00 -0800
Subject: [PATCH 070/157] [meta] update whoami to 1.5.0, remove patch (#5194)

whoami 1.5.0 contains the fix for https://github.com/oxidecomputer/security-operations/issues/43.
---
 Cargo.lock | 26 ++++----------------------
 Cargo.toml |  5 -----
 2 files changed, 4 insertions(+), 27 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7a491135b5..2e822122c2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -883,12 +883,6 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
-[[package]]
-name = "cfg_aliases"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
-
 [[package]]
 name = "chacha20"
 version = "0.9.1"
@@ -4629,18 +4623,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "nix"
-version = "0.28.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4"
-dependencies = [
- "bitflags 2.4.0",
- "cfg-if",
- "cfg_aliases",
- "libc",
-]
-
 [[package]]
 name = "nodrop"
 version = "0.1.14"
@@ -10270,12 +10252,12 @@ dependencies = [
 
 [[package]]
 name = "whoami"
-version = "1.4.1"
-source = "git+https://github.com/oxidecomputer/whoami?branch=use-nix#da2a26e695849f1ffc394627ba27a6a473650705"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fec781d48b41f8163426ed18e8fc2864c12937df9ce54c88ede7bd47270893e"
 dependencies = [
- "nix 0.28.0",
+ "redox_syscall 0.4.1",
  "wasite",
- "wasm-bindgen",
  "web-sys",
 ]
 
diff --git a/Cargo.toml b/Cargo.toml
index 1d701c550e..9d556c7b68 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -621,11 +621,6 @@ opt-level = 3
 git = 'https://github.com/oxidecomputer/pq-sys'
 branch = "oxide/omicron"
 
-# See https://github.com/oxidecomputer/security-operations/issues/43.
-[patch.crates-io.whoami]
-git = "https://github.com/oxidecomputer/whoami"
-branch = "use-nix"
-
 # Using the workspace-hack via this patch directive means that it only applies
 # while building within this workspace. If another workspace imports a crate
 # from here via a git dependency, it will not have the workspace-hack applied

From 25e905b63dba5783aa4fc962f5f2582d30906950 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 4 Mar 2024 23:41:52 -0800
Subject: [PATCH 071/157] Bump mio from 0.8.9 to 0.8.11 (#5195)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Cargo.lock                |  4 ++--
 workspace-hack/Cargo.toml | 16 ++++++++--------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2e822122c2..962f515ebd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4130,9 +4130,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.8.9"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
 dependencies = [
  "libc",
  "log",
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index 01b7e440ae..aa2523d6ff 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -229,45 +229,45 @@ zip = { version = "0.6.6", default-features = false, features = ["bzip2", "defla
 [target.x86_64-unknown-linux-gnu.dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
 dof = { version = "0.3.0", default-features = false, features = ["des"] }
-mio = { version = "0.8.9", features = ["net", "os-ext"] }
+mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.x86_64-unknown-linux-gnu.build-dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
 dof = { version = "0.3.0", default-features = false, features = ["des"] }
-mio = { version = "0.8.9", features = ["net", "os-ext"] }
+mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.x86_64-apple-darwin.dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
-mio = { version = "0.8.9", features = ["net", "os-ext"] }
+mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.x86_64-apple-darwin.build-dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
-mio = { version = "0.8.9", features = ["net", "os-ext"] }
+mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.aarch64-apple-darwin.dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
-mio = { version = "0.8.9", features = ["net", "os-ext"] }
+mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.aarch64-apple-darwin.build-dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
-mio = { version = "0.8.9", features = ["net", "os-ext"] }
+mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.x86_64-unknown-illumos.dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
 dof = { version = "0.3.0", default-features = false, features = ["des"] }
-mio = { version = "0.8.9", features = ["net", "os-ext"] }
+mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] }
@@ -276,7 +276,7 @@ toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", featu
 [target.x86_64-unknown-illumos.build-dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
 dof = { version = "0.3.0", default-features = false, features = ["des"] }
-mio = { version = "0.8.9", features = ["net", "os-ext"] }
+mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] }

From a505c234948df7eff042d9a0dcd458dc520ede14 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Tue, 5 Mar 2024 08:54:14 +0000
Subject: [PATCH 072/157] chore(deps): update rust crate indexmap to 2.2.4
 (#5167)

---
 Cargo.lock                | 34 +++++++++++++++++-----------------
 Cargo.toml                |  2 +-
 workspace-hack/Cargo.toml |  4 ++--
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 962f515ebd..1ed8bbdfef 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1978,7 +1978,7 @@ dependencies = [
  "hostname",
  "http 0.2.11",
  "hyper 0.14.27",
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "multer",
  "openapiv3",
  "paste",
@@ -2781,7 +2781,7 @@ dependencies = [
  "debug-ignore",
  "fixedbitset",
  "guppy-workspace-hack",
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "itertools 0.12.1",
  "nested",
  "once_cell",
@@ -2813,7 +2813,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http 0.2.11",
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "slab",
  "tokio",
  "tokio-util",
@@ -3408,9 +3408,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.2.3"
+version = "2.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177"
+checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4"
 dependencies = [
  "equivalent",
  "hashbrown 0.14.3",
@@ -5397,7 +5397,7 @@ dependencies = [
  "hex",
  "hmac",
  "hyper 0.14.27",
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "inout",
  "ipnetwork",
  "itertools 0.10.5",
@@ -5521,7 +5521,7 @@ version = "0.4.0"
 source = "git+https://github.com/oxidecomputer/openapi-lint?branch=main#ef442ee4343e97b6d9c217d3e7533962fe7d7236"
 dependencies = [
  "heck 0.4.1",
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "lazy_static",
  "openapiv3",
  "regex",
@@ -5533,7 +5533,7 @@ version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cc02deea53ffe807708244e5914f6b099ad7015a207ee24317c22112e17d9c5c"
 dependencies = [
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "serde",
  "serde_json",
 ]
@@ -5790,7 +5790,7 @@ dependencies = [
  "expectorate",
  "futures",
  "highway",
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "itertools 0.12.1",
  "omicron-common",
  "omicron-test-utils",
@@ -6179,7 +6179,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"
 dependencies = [
  "fixedbitset",
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "serde",
  "serde_derive",
 ]
@@ -6571,7 +6571,7 @@ dependencies = [
  "getopts",
  "heck 0.4.1",
  "http 0.2.11",
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "openapiv3",
  "proc-macro2",
  "quote",
@@ -7906,7 +7906,7 @@ dependencies = [
  "chrono",
  "hex",
  "indexmap 1.9.3",
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "serde",
  "serde_derive",
  "serde_json",
@@ -7932,7 +7932,7 @@ version = "0.9.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574"
 dependencies = [
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "itoa",
  "ryu",
  "serde",
@@ -9322,7 +9322,7 @@ version = "0.19.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
 dependencies = [
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "serde",
  "serde_spanned",
  "toml_datetime",
@@ -9335,7 +9335,7 @@ version = "0.22.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2c1b5fd4128cc8d3e0cb74d4ed9a9cc7c7284becd4df68f5f940e1ad123606f6"
 dependencies = [
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "serde",
  "serde_spanned",
  "toml_datetime",
@@ -9854,7 +9854,7 @@ dependencies = [
  "derive-where",
  "either",
  "futures",
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "indicatif",
  "libsw",
  "linear-map",
@@ -10274,7 +10274,7 @@ dependencies = [
  "crossterm",
  "futures",
  "humantime",
- "indexmap 2.2.3",
+ "indexmap 2.2.5",
  "indicatif",
  "itertools 0.12.1",
  "omicron-common",
diff --git a/Cargo.toml b/Cargo.toml
index 9d556c7b68..5a477da0a2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -231,7 +231,7 @@ hyper = "0.14"
 hyper-rustls = "0.26.0"
 hyper-staticfile = "0.9.5"
 illumos-utils = { path = "illumos-utils" }
-indexmap = "2.2.3"
+indexmap = "2.2.5"
 indicatif = { version = "0.17.8", features = ["rayon"] }
 installinator = { path = "installinator" }
 installinator-artifactd = { path = "installinator-artifactd" }
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index aa2523d6ff..e70c7c329e 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -58,7 +58,7 @@ hashbrown = { version = "0.14.3", features = ["raw"] }
 hex = { version = "0.4.3", features = ["serde"] }
 hmac = { version = "0.12.1", default-features = false, features = ["reset"] }
 hyper = { version = "0.14.27", features = ["full"] }
-indexmap = { version = "2.2.3", features = ["serde"] }
+indexmap = { version = "2.2.5", features = ["serde"] }
 inout = { version = "0.1.3", default-features = false, features = ["std"] }
 ipnetwork = { version = "0.20.0", features = ["schemars"] }
 itertools = { version = "0.10.5" }
@@ -164,7 +164,7 @@ hashbrown = { version = "0.14.3", features = ["raw"] }
 hex = { version = "0.4.3", features = ["serde"] }
 hmac = { version = "0.12.1", default-features = false, features = ["reset"] }
 hyper = { version = "0.14.27", features = ["full"] }
-indexmap = { version = "2.2.3", features = ["serde"] }
+indexmap = { version = "2.2.5", features = ["serde"] }
 inout = { version = "0.1.3", default-features = false, features = ["std"] }
 ipnetwork = { version = "0.20.0", features = ["schemars"] }
 itertools = { version = "0.10.5" }

From 1f26c66921b9215bfe11d750514939bcdc11ae12 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Tue, 5 Mar 2024 10:06:38 +0000
Subject: [PATCH 073/157] chore(deps): update taiki-e/install-action digest to
 dabb9c1 (#5199)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [taiki-e/install-action](https://togithub.com/taiki-e/install-action)
| action | digest | [`c6ffb58` ->
`dabb9c1`](https://togithub.com/taiki-e/install-action/compare/c6ffb58...dabb9c1)
|

---

### Configuration

📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone
America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone
America/Los_Angeles.

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update
again.

---

- [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check
this box

---

This PR has been generated by [Renovate
Bot](https://togithub.com/renovatebot/renovate).

<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNy4yMjcuMSIsInVwZGF0ZWRJblZlciI6IjM3LjIyNy4xIiwidGFyZ2V0QnJhbmNoIjoibWFpbiJ9-->

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index e3e032806c..cac987b277 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@c6ffb5827ec9fecafc811f72ec5d7854680dc6ca # v2
+        uses: taiki-e/install-action@dabb9c1ee51c21c545764a0d517f069ff52e6477 # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From b664ded41efee55bf49942604bf7f36fa0abd6f7 Mon Sep 17 00:00:00 2001
From: Alan Hanson <alan@oxidecomputer.com>
Date: Tue, 5 Mar 2024 10:43:07 -0800
Subject: [PATCH 074/157] Add a show-empty option to oxlog (#5186)

This adds an option flag (off by default) that will show matching log
files even if they are zero length.

Co-authored-by: Alan Hanson <alan@oxide.computer>
---
 dev-tools/oxlog/src/bin/oxlog.rs |  7 ++++++-
 dev-tools/oxlog/src/lib.rs       | 24 ++++++++++++++++++------
 2 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/dev-tools/oxlog/src/bin/oxlog.rs b/dev-tools/oxlog/src/bin/oxlog.rs
index ef79605dda..88e067c382 100644
--- a/dev-tools/oxlog/src/bin/oxlog.rs
+++ b/dev-tools/oxlog/src/bin/oxlog.rs
@@ -4,7 +4,7 @@
 
 //! Tool for discovering oxide related logfiles on sleds
 
-use clap::{Args, Parser, Subcommand};
+use clap::{ArgAction, Args, Parser, Subcommand};
 use oxlog::{Filter, LogFile, Zones};
 
 #[derive(Debug, Parser)]
@@ -50,6 +50,10 @@ struct FilterArgs {
     // Print only the extra log files
     #[arg(short, long)]
     extra: bool,
+
+    /// Show log files even if they are empty
+    #[arg(short, long, action=ArgAction::SetTrue)]
+    show_empty: bool,
 }
 
 fn main() -> Result<(), anyhow::Error> {
@@ -68,6 +72,7 @@ fn main() -> Result<(), anyhow::Error> {
                 current: filter.current,
                 archived: filter.archived,
                 extra: filter.extra,
+                show_empty: filter.show_empty,
             };
             let print_metadata = |f: &LogFile| {
                 println!(
diff --git a/dev-tools/oxlog/src/lib.rs b/dev-tools/oxlog/src/lib.rs
index a4774fabbc..625d360368 100644
--- a/dev-tools/oxlog/src/lib.rs
+++ b/dev-tools/oxlog/src/lib.rs
@@ -75,6 +75,9 @@ pub struct Filter {
     /// standard paths or don't follow the naming conventions of SMF service
     /// files. e.g. `/pool/ext/e12f29b8-1ab8-431e-bc96-1c1298947980/crypt/zone/oxz_cockroachdb_8bbea076-ff60-4330-8302-383e18140ef3/root/data/logs/cockroach.log`
     pub extra: bool,
+
+    /// Show a log file even if is has zero size.
+    pub show_empty: bool,
 }
 
 /// Path and metadata about a logfile
@@ -264,17 +267,21 @@ impl Zones {
         // 'archived'. These files have not yet been migrated into the debug
         // directory.
         if filter.current || filter.archived {
-            load_svc_logs(paths.primary.clone(), &mut output);
+            load_svc_logs(
+                paths.primary.clone(),
+                &mut output,
+                filter.show_empty,
+            );
         }
 
         if filter.archived {
             for dir in paths.debug.clone() {
-                load_svc_logs(dir, &mut output);
+                load_svc_logs(dir, &mut output, filter.show_empty);
             }
         }
         if filter.extra {
             for (svc_name, dir) in paths.extra.clone() {
-                load_extra_logs(dir, svc_name, &mut output);
+                load_extra_logs(dir, svc_name, &mut output, filter.show_empty);
             }
         }
         output
@@ -320,7 +327,11 @@ pub fn oxide_smf_service_name_from_log_file_name(
 
 // Given a directory, find all oxide specific SMF service logs and return them
 // mapped to their inferred service name.
-fn load_svc_logs(dir: Utf8PathBuf, logs: &mut BTreeMap<ServiceName, SvcLogs>) {
+fn load_svc_logs(
+    dir: Utf8PathBuf,
+    logs: &mut BTreeMap<ServiceName, SvcLogs>,
+    show_empty: bool,
+) {
     let Ok(entries) = dir.read_dir_utf8() else {
         return;
     };
@@ -344,7 +355,7 @@ fn load_svc_logs(dir: Utf8PathBuf, logs: &mut BTreeMap<ServiceName, SvcLogs>) {
             };
 
             logfile.read_metadata(&entry);
-            if logfile.size == Some(0) {
+            if logfile.size == Some(0) && !show_empty {
                 // skip 0 size files
                 continue;
             }
@@ -368,6 +379,7 @@ fn load_extra_logs(
     dir: Utf8PathBuf,
     svc_name: &str,
     logs: &mut BTreeMap<ServiceName, SvcLogs>,
+    show_empty: bool,
 ) {
     let Ok(entries) = dir.read_dir_utf8() else {
         return;
@@ -384,7 +396,7 @@ fn load_extra_logs(
         path.push(filename);
         let mut logfile = LogFile::new(path);
         logfile.read_metadata(&entry);
-        if logfile.size == Some(0) {
+        if logfile.size == Some(0) && !show_empty {
             // skip 0 size files
             continue;
         }

From 94bc6f542efc54a06bf6e17159e3b896a61959e5 Mon Sep 17 00:00:00 2001
From: Rain <rain@oxide.computer>
Date: Tue, 5 Mar 2024 13:56:44 -0800
Subject: [PATCH 075/157] [common] move config types to a new nexus-config
 crate (#5198)

Part of https://github.com/oxidecomputer/security-operations/issues/43 -- move these types into their own crate so omicron-common no longer depends on tokio-postgres.

There are also other things that can be moved into their own crates, but nothing immediately security-critical.
---
 Cargo.lock                                    | 33 ++++++-
 Cargo.toml                                    |  3 +
 common/Cargo.toml                             |  8 +-
 common/src/lib.rs                             |  2 -
 dev-tools/omdb/Cargo.toml                     |  1 +
 dev-tools/omdb/src/bin/omdb/db.rs             |  2 +-
 dev-tools/omicron-dev/Cargo.toml              |  1 +
 dev-tools/omicron-dev/src/bin/omicron-dev.rs  |  3 +-
 nexus-config/Cargo.toml                       | 22 +++++
 nexus-config/README.adoc                      |  4 +
 nexus-config/src/lib.rs                       | 13 +++
 {common => nexus-config}/src/nexus_config.rs  | 95 +++++++++----------
 .../src/postgres_config.rs                    |  0
 nexus/Cargo.toml                              |  1 +
 nexus/db-model/Cargo.toml                     |  1 +
 nexus/db-model/src/ipv4net.rs                 |  2 +-
 nexus/db-model/src/ipv6net.rs                 |  2 +-
 nexus/db-model/src/vpc_subnet.rs              |  2 +-
 nexus/db-queries/Cargo.toml                   |  1 +
 nexus/db-queries/src/db/config.rs             |  2 +-
 .../src/db/datastore/db_metadata.rs           |  2 +-
 nexus/db-queries/src/db/datastore/mod.rs      |  4 +-
 nexus/db-queries/src/db/datastore/rack.rs     |  2 +-
 nexus/db-queries/src/db/datastore/region.rs   |  2 +-
 .../src/db/queries/network_interface.rs       |  2 +-
 .../src/db/queries/region_allocation.rs       |  2 +-
 nexus/reconfigurator/execution/Cargo.toml     |  1 +
 .../execution/src/resource_allocation.rs      |  2 +-
 nexus/reconfigurator/planning/Cargo.toml      |  1 +
 .../planning/src/blueprint_builder.rs         |  2 +-
 nexus/src/app/background/init.rs              |  4 +-
 nexus/src/app/mod.rs                          | 14 +--
 nexus/src/app/vpc_subnet.rs                   |  2 +-
 nexus/src/bin/nexus.rs                        |  4 +-
 nexus/src/bin/schema-updater.rs               |  4 +-
 nexus/src/config.rs                           | 13 ---
 nexus/src/context.rs                          | 14 +--
 nexus/src/lib.rs                              | 13 ++-
 nexus/test-interface/Cargo.toml               |  1 +
 nexus/test-interface/src/lib.rs               |  6 +-
 nexus/test-utils/Cargo.toml                   |  1 +
 nexus/test-utils/src/lib.rs                   | 53 +++++------
 .../tests/integration_tests/initialization.rs | 56 ++++++-----
 nexus/tests/integration_tests/schema.rs       |  6 +-
 .../integration_tests/subnet_allocation.rs    |  2 +-
 nexus/tests/integration_tests/updates.rs      |  2 +-
 sled-agent/Cargo.toml                         |  1 +
 sled-agent/src/rack_setup/plan/service.rs     |  2 +-
 sled-agent/src/services.rs                    |  6 +-
 sled-agent/src/sim/server.rs                  |  2 +-
 test-utils/Cargo.toml                         |  1 +
 test-utils/src/dev/db.rs                      |  2 +-
 52 files changed, 244 insertions(+), 183 deletions(-)
 create mode 100644 nexus-config/Cargo.toml
 create mode 100644 nexus-config/README.adoc
 create mode 100644 nexus-config/src/lib.rs
 rename {common => nexus-config}/src/nexus_config.rs (96%)
 rename {common => nexus-config}/src/postgres_config.rs (100%)
 delete mode 100644 nexus/src/config.rs

diff --git a/Cargo.lock b/Cargo.lock
index 1ed8bbdfef..18c783037f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4283,6 +4283,26 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "nexus-config"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "camino",
+ "dropshot",
+ "expectorate",
+ "libc",
+ "omicron-common",
+ "omicron-workspace-hack",
+ "schemars",
+ "serde",
+ "serde_json",
+ "serde_with",
+ "tokio-postgres",
+ "toml 0.8.10",
+ "uuid",
+]
+
 [[package]]
 name = "nexus-db-model"
 version = "0.1.0"
@@ -4297,6 +4317,7 @@ dependencies = [
  "ipnetwork",
  "macaddr",
  "newtype_derive",
+ "nexus-config",
  "nexus-defaults",
  "nexus-types",
  "omicron-certificates",
@@ -4352,6 +4373,7 @@ dependencies = [
  "itertools 0.12.1",
  "macaddr",
  "newtype_derive",
+ "nexus-config",
  "nexus-db-model",
  "nexus-inventory",
  "nexus-reconfigurator-planning",
@@ -4460,6 +4482,7 @@ dependencies = [
  "illumos-utils",
  "internal-dns",
  "ipnet",
+ "nexus-config",
  "nexus-db-model",
  "nexus-db-queries",
  "nexus-inventory",
@@ -4490,6 +4513,7 @@ dependencies = [
  "internal-dns",
  "ipnet",
  "ipnetwork",
+ "nexus-config",
  "nexus-inventory",
  "nexus-types",
  "omicron-common",
@@ -4506,6 +4530,7 @@ name = "nexus-test-interface"
 version = "0.1.0"
 dependencies = [
  "async-trait",
+ "nexus-config",
  "nexus-types",
  "omicron-common",
  "omicron-workspace-hack",
@@ -4533,6 +4558,7 @@ dependencies = [
  "http 0.2.11",
  "hyper 0.14.27",
  "internal-dns",
+ "nexus-config",
  "nexus-db-queries",
  "nexus-test-interface",
  "nexus-types",
@@ -4906,8 +4932,6 @@ dependencies = [
  "test-strategy",
  "thiserror",
  "tokio",
- "tokio-postgres",
- "toml 0.8.10",
  "uuid",
 ]
 
@@ -4925,6 +4949,7 @@ dependencies = [
  "gateway-messages",
  "gateway-test-utils",
  "libc",
+ "nexus-config",
  "nexus-test-interface",
  "nexus-test-utils",
  "omicron-common",
@@ -5031,6 +5056,7 @@ dependencies = [
  "macaddr",
  "mg-admin-client",
  "mime_guess",
+ "nexus-config",
  "nexus-db-model",
  "nexus-db-queries",
  "nexus-defaults",
@@ -5129,6 +5155,7 @@ dependencies = [
  "ipnetwork",
  "multimap",
  "nexus-client",
+ "nexus-config",
  "nexus-db-model",
  "nexus-db-queries",
  "nexus-test-utils",
@@ -5259,6 +5286,7 @@ dependencies = [
  "macaddr",
  "mg-admin-client",
  "nexus-client",
+ "nexus-config",
  "omicron-common",
  "omicron-test-utils",
  "omicron-workspace-hack",
@@ -5321,6 +5349,7 @@ dependencies = [
  "hex",
  "http 0.2.11",
  "libc",
+ "nexus-config",
  "omicron-common",
  "omicron-workspace-hack",
  "pem",
diff --git a/Cargo.toml b/Cargo.toml
index 5a477da0a2..474739a932 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -36,6 +36,7 @@ members = [
     "ipcc",
     "key-manager",
     "nexus",
+    "nexus-config",
     "nexus/authz-macros",
     "nexus/db-macros",
     "nexus/db-model",
@@ -113,6 +114,7 @@ default-members = [
     "ipcc",
     "key-manager",
     "nexus",
+    "nexus-config",
     "nexus/authz-macros",
     "nexus/macros-common",
     "nexus/db-macros",
@@ -254,6 +256,7 @@ newtype_derive = "0.1.6"
 mg-admin-client = { path = "clients/mg-admin-client" }
 multimap = "0.10.0"
 nexus-client = { path = "clients/nexus-client" }
+nexus-config = { path = "nexus-config" }
 nexus-db-model = { path = "nexus/db-model" }
 nexus-db-queries = { path = "nexus/db-queries" }
 nexus-defaults = { path = "nexus/defaults" }
diff --git a/common/Cargo.toml b/common/Cargo.toml
index 5628a93397..4451d92bdb 100644
--- a/common/Cargo.toml
+++ b/common/Cargo.toml
@@ -4,6 +4,12 @@ version = "0.1.0"
 edition = "2021"
 license = "MPL-2.0"
 
+# NOTE:
+#
+# This crate is depended on by several other workspaces! Be careful of adding
+# new regular or build dependencies here, as they will be added to those crates
+# as well. (Dev-dependencies are fine.)
+
 [dependencies]
 anyhow.workspace = true
 api_identity.workspace = true
@@ -32,8 +38,6 @@ strum.workspace = true
 test-strategy = { workspace = true, optional = true }
 thiserror.workspace = true
 tokio = { workspace = true, features = ["full"] }
-tokio-postgres.workspace = true
-toml.workspace = true
 uuid.workspace = true
 parse-display.workspace = true
 progenitor.workspace = true
diff --git a/common/src/lib.rs b/common/src/lib.rs
index 0d63de90fb..411bc3e426 100644
--- a/common/src/lib.rs
+++ b/common/src/lib.rs
@@ -26,8 +26,6 @@ pub mod backoff;
 pub mod cmd;
 pub mod disk;
 pub mod ledger;
-pub mod nexus_config;
-pub mod postgres_config;
 pub mod update;
 pub mod vlan;
 
diff --git a/dev-tools/omdb/Cargo.toml b/dev-tools/omdb/Cargo.toml
index 3f566f55ee..df56fda571 100644
--- a/dev-tools/omdb/Cargo.toml
+++ b/dev-tools/omdb/Cargo.toml
@@ -25,6 +25,7 @@ gateway-test-utils.workspace = true
 humantime.workspace = true
 internal-dns.workspace = true
 nexus-client.workspace = true
+nexus-config.workspace = true
 nexus-db-model.workspace = true
 nexus-db-queries.workspace = true
 nexus-types.workspace = true
diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs
index a51deacaf4..c7e9022fc2 100644
--- a/dev-tools/omdb/src/bin/omdb/db.rs
+++ b/dev-tools/omdb/src/bin/omdb/db.rs
@@ -36,6 +36,7 @@ use diesel::OptionalExtension;
 use diesel::TextExpressionMethods;
 use gateway_client::types::SpType;
 use ipnetwork::IpNetwork;
+use nexus_config::PostgresConfigWithUrl;
 use nexus_db_model::Dataset;
 use nexus_db_model::Disk;
 use nexus_db_model::DnsGroup;
@@ -82,7 +83,6 @@ use nexus_types::inventory::RotPageWhich;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::Generation;
 use omicron_common::api::external::MacAddr;
-use omicron_common::postgres_config::PostgresConfigWithUrl;
 use sled_agent_client::types::VolumeConstructionRequest;
 use std::borrow::Cow;
 use std::cmp::Ordering;
diff --git a/dev-tools/omicron-dev/Cargo.toml b/dev-tools/omicron-dev/Cargo.toml
index ce9a6ac32d..6aa480b2c6 100644
--- a/dev-tools/omicron-dev/Cargo.toml
+++ b/dev-tools/omicron-dev/Cargo.toml
@@ -16,6 +16,7 @@ futures.workspace = true
 gateway-messages.workspace = true
 gateway-test-utils.workspace = true
 libc.workspace = true
+nexus-config.workspace = true
 nexus-test-utils = { workspace = true, features = ["omicron-dev"] }
 nexus-test-interface.workspace = true
 omicron-common.workspace = true
diff --git a/dev-tools/omicron-dev/src/bin/omicron-dev.rs b/dev-tools/omicron-dev/src/bin/omicron-dev.rs
index 0eb421478c..5e0c6486d6 100644
--- a/dev-tools/omicron-dev/src/bin/omicron-dev.rs
+++ b/dev-tools/omicron-dev/src/bin/omicron-dev.rs
@@ -11,6 +11,7 @@ use clap::Args;
 use clap::Parser;
 use dropshot::test_util::LogContext;
 use futures::stream::StreamExt;
+use nexus_config::NexusConfig;
 use nexus_test_interface::NexusServer;
 use omicron_common::cmd::fatal;
 use omicron_common::cmd::CmdError;
@@ -473,7 +474,7 @@ async fn cmd_run_all(args: &RunAllArgs) -> Result<(), anyhow::Error> {
 
     // Read configuration.
     let config_str = include_str!("../../../../nexus/examples/config.toml");
-    let mut config: omicron_common::nexus_config::Config =
+    let mut config: NexusConfig =
         toml::from_str(config_str).context("parsing example config")?;
     config.pkg.log = dropshot::ConfigLogging::File {
         // See LogContext::new(),
diff --git a/nexus-config/Cargo.toml b/nexus-config/Cargo.toml
new file mode 100644
index 0000000000..af7eb70f62
--- /dev/null
+++ b/nexus-config/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+name = "nexus-config"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+anyhow.workspace = true
+camino.workspace = true
+dropshot.workspace = true
+omicron-common.workspace = true
+omicron-workspace-hack.workspace = true
+schemars.workspace = true
+serde.workspace = true
+serde_with.workspace = true
+tokio-postgres.workspace = true
+toml.workspace = true
+uuid.workspace = true
+
+[dev-dependencies]
+expectorate.workspace = true
+libc.workspace = true
+serde_json.workspace = true
diff --git a/nexus-config/README.adoc b/nexus-config/README.adoc
new file mode 100644
index 0000000000..6f42e2db0a
--- /dev/null
+++ b/nexus-config/README.adoc
@@ -0,0 +1,4 @@
+= Nexus config
+
+This crate contains type definitions for Nexus configuration, as shared by
+Nexus and sled-agent.
diff --git a/nexus-config/src/lib.rs b/nexus-config/src/lib.rs
new file mode 100644
index 0000000000..c3411147e3
--- /dev/null
+++ b/nexus-config/src/lib.rs
@@ -0,0 +1,13 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Common types used for Nexus configuration.
+//!
+//! The root structure is [`NexusConfig`].
+
+mod nexus_config;
+mod postgres_config;
+
+pub use nexus_config::*;
+pub use postgres_config::*;
diff --git a/common/src/nexus_config.rs b/nexus-config/src/nexus_config.rs
similarity index 96%
rename from common/src/nexus_config.rs
rename to nexus-config/src/nexus_config.rs
index 2545d4cb91..0c40326001 100644
--- a/common/src/nexus_config.rs
+++ b/nexus-config/src/nexus_config.rs
@@ -5,11 +5,13 @@
 //! Configuration parameters to Nexus that are usually only known
 //! at deployment time.
 
-use crate::address::NEXUS_TECHPORT_EXTERNAL_PORT;
-use crate::api::internal::shared::SwitchLocation;
+use crate::PostgresConfigWithUrl;
+
+use omicron_common::address::Ipv6Subnet;
+use omicron_common::address::NEXUS_TECHPORT_EXTERNAL_PORT;
+use omicron_common::address::RACK_PREFIX;
+use omicron_common::api::internal::shared::SwitchLocation;
 
-use super::address::{Ipv6Subnet, RACK_PREFIX};
-use super::postgres_config::PostgresConfigWithUrl;
 use anyhow::anyhow;
 use camino::{Utf8Path, Utf8PathBuf};
 use dropshot::ConfigDropshot;
@@ -28,6 +30,31 @@ use std::net::SocketAddr;
 use std::time::Duration;
 use uuid::Uuid;
 
+#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
+pub struct NexusConfig {
+    /// Configuration parameters known at compile-time.
+    #[serde(flatten)]
+    pub pkg: PackageConfig,
+
+    /// A variety of configuration parameters only known at deployment time.
+    pub deployment: DeploymentConfig,
+}
+
+impl NexusConfig {
+    /// Load a `Config` from the given TOML file
+    ///
+    /// This config object can then be used to create a new `Nexus`.
+    /// The format is described in the README.
+    pub fn from_file<P: AsRef<Utf8Path>>(path: P) -> Result<Self, LoadError> {
+        let path = path.as_ref();
+        let file_contents = std::fs::read_to_string(path)
+            .map_err(|e| (path.to_path_buf(), e))?;
+        let config_parsed: Self = toml::from_str(&file_contents)
+            .map_err(|e| (path.to_path_buf(), e))?;
+        Ok(config_parsed)
+    }
+}
+
 #[derive(Debug)]
 pub struct LoadError {
     pub path: Utf8PathBuf,
@@ -488,31 +515,6 @@ pub struct PackageConfig {
     pub default_region_allocation_strategy: RegionAllocationStrategy,
 }
 
-#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
-pub struct Config {
-    /// Configuration parameters known at compile-time.
-    #[serde(flatten)]
-    pub pkg: PackageConfig,
-
-    /// A variety of configuration parameters only known at deployment time.
-    pub deployment: DeploymentConfig,
-}
-
-impl Config {
-    /// Load a `Config` from the given TOML file
-    ///
-    /// This config object can then be used to create a new `Nexus`.
-    /// The format is described in the README.
-    pub fn from_file<P: AsRef<Utf8Path>>(path: P) -> Result<Self, LoadError> {
-        let path = path.as_ref();
-        let file_contents = std::fs::read_to_string(path)
-            .map_err(|e| (path.to_path_buf(), e))?;
-        let config_parsed: Self = toml::from_str(&file_contents)
-            .map_err(|e| (path.to_path_buf(), e))?;
-        Ok(config_parsed)
-    }
-}
-
 /// List of supported external authn schemes
 ///
 /// Note that the authn subsystem doesn't know about this type.  It allows
@@ -552,24 +554,16 @@ impl std::fmt::Display for SchemeName {
 
 #[cfg(test)]
 mod test {
-    use super::{
-        default_techport_external_server_port, AuthnConfig,
-        BackgroundTaskConfig, BlueprintTasksConfig, Config,
-        ConfigDropshotWithTls, ConsoleConfig, Database, DeploymentConfig,
-        DnsTasksConfig, DpdConfig, ExternalEndpointsConfig, InternalDns,
-        InventoryConfig, LoadError, LoadErrorKind, MgdConfig, NatCleanupConfig,
-        PackageConfig, PhantomDiskConfig, RegionReplacementConfig, SchemeName,
-        TimeseriesDbConfig, Tunables, UpdatesConfig,
-    };
-    use crate::address::{Ipv6Subnet, RACK_PREFIX};
-    use crate::api::internal::shared::SwitchLocation;
-    use crate::nexus_config::{BfdManagerConfig, SyncServiceZoneNatConfig};
+    use super::*;
+
+    use omicron_common::address::{Ipv6Subnet, RACK_PREFIX};
+    use omicron_common::api::internal::shared::SwitchLocation;
+
     use camino::{Utf8Path, Utf8PathBuf};
     use dropshot::ConfigDropshot;
     use dropshot::ConfigLogging;
     use dropshot::ConfigLoggingIfExists;
     use dropshot::ConfigLoggingLevel;
-    use libc;
     use std::collections::HashMap;
     use std::fs;
     use std::net::{Ipv6Addr, SocketAddr};
@@ -594,13 +588,16 @@ mod test {
     /// loads the config from that file, then removes the file.  `label` is used
     /// as a unique string for the filename and error messages.  It should be
     /// unique for each test.
-    fn read_config(label: &str, contents: &str) -> Result<Config, LoadError> {
+    fn read_config(
+        label: &str,
+        contents: &str,
+    ) -> Result<NexusConfig, LoadError> {
         let pathbuf = temp_path(label);
         let path = pathbuf.as_path();
         eprintln!("writing test config {}", path);
         fs::write(path, contents).expect("write to tempfile failed");
 
-        let result = Config::from_file(path);
+        let result = NexusConfig::from_file(path);
         fs::remove_file(path).expect("failed to remove temporary file");
         eprintln!("{:?}", result);
         result
@@ -610,7 +607,7 @@ mod test {
 
     #[test]
     fn test_config_nonexistent() {
-        let error = Config::from_file(Utf8Path::new("/nonexistent"))
+        let error = NexusConfig::from_file(Utf8Path::new("/nonexistent"))
             .expect_err("expected config to fail from /nonexistent");
         let expected = std::io::Error::from_raw_os_error(libc::ENOENT);
         assert_eq!(error, expected);
@@ -726,7 +723,7 @@ mod test {
 
         assert_eq!(
             config,
-            Config {
+            NexusConfig {
                 deployment: DeploymentConfig {
                     id: "28b90dc4-c22a-65ba-f49a-f051fe01208f".parse().unwrap(),
                     rack_id: "38b90dc4-c22a-65ba-f49a-f051fe01208f"
@@ -1020,7 +1017,7 @@ mod test {
         // The example config file should be valid.
         let config_path = "../nexus/examples/config.toml";
         println!("checking {:?}", config_path);
-        let example_config = Config::from_file(config_path)
+        let example_config = NexusConfig::from_file(config_path)
             .expect("example config file is not valid");
 
         // The config file used for the tests should also be valid.  The tests
@@ -1028,7 +1025,7 @@ mod test {
         // helpful to verify this here explicitly as well.
         let config_path = "../nexus/examples/config.toml";
         println!("checking {:?}", config_path);
-        let _ = Config::from_file(config_path)
+        let _ = NexusConfig::from_file(config_path)
             .expect("test config file is not valid");
 
         // The partial config file that's used to deploy Nexus must also be
@@ -1062,7 +1059,7 @@ mod test {
             \n\n\n",
             );
             contents.push_str(&example_deployment);
-            let _: Config = toml::from_str(&contents)
+            let _: NexusConfig = toml::from_str(&contents)
                 .expect("Nexus SMF config file is not valid");
         }
     }
diff --git a/common/src/postgres_config.rs b/nexus-config/src/postgres_config.rs
similarity index 100%
rename from common/src/postgres_config.rs
rename to nexus-config/src/postgres_config.rs
diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml
index 581e16e84d..de79f3429d 100644
--- a/nexus/Cargo.toml
+++ b/nexus/Cargo.toml
@@ -39,6 +39,7 @@ macaddr.workspace = true
 mime_guess.workspace = true
 # Not under "dev-dependencies"; these also need to be implemented for
 # integration tests.
+nexus-config.workspace = true
 nexus-test-interface.workspace = true
 num-integer.workspace = true
 once_cell.workspace = true
diff --git a/nexus/db-model/Cargo.toml b/nexus/db-model/Cargo.toml
index 43b6f6a4b0..b1bb68b8f0 100644
--- a/nexus/db-model/Cargo.toml
+++ b/nexus/db-model/Cargo.toml
@@ -34,6 +34,7 @@ uuid.workspace = true
 db-macros.workspace = true
 omicron-certificates.workspace = true
 omicron-common.workspace = true
+nexus-config.workspace = true
 nexus-defaults.workspace = true
 nexus-types.workspace = true
 omicron-passwords.workspace = true
diff --git a/nexus/db-model/src/ipv4net.rs b/nexus/db-model/src/ipv4net.rs
index fdd9cd3293..eaf8a6eed8 100644
--- a/nexus/db-model/src/ipv4net.rs
+++ b/nexus/db-model/src/ipv4net.rs
@@ -9,8 +9,8 @@ use diesel::pg::Pg;
 use diesel::serialize::{self, ToSql};
 use diesel::sql_types;
 use ipnetwork::IpNetwork;
+use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use omicron_common::api::external;
-use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use serde::Deserialize;
 use serde::Serialize;
 use std::net::Ipv4Addr;
diff --git a/nexus/db-model/src/ipv6net.rs b/nexus/db-model/src/ipv6net.rs
index e47bbc4d33..d516b67ed9 100644
--- a/nexus/db-model/src/ipv6net.rs
+++ b/nexus/db-model/src/ipv6net.rs
@@ -8,8 +8,8 @@ use diesel::pg::Pg;
 use diesel::serialize::{self, ToSql};
 use diesel::sql_types;
 use ipnetwork::IpNetwork;
+use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use omicron_common::api::external;
-use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use rand::{rngs::StdRng, SeedableRng};
 use serde::Deserialize;
 use serde::Serialize;
diff --git a/nexus/db-model/src/vpc_subnet.rs b/nexus/db-model/src/vpc_subnet.rs
index 99f2c5e3ac..407c933ef2 100644
--- a/nexus/db-model/src/vpc_subnet.rs
+++ b/nexus/db-model/src/vpc_subnet.rs
@@ -10,11 +10,11 @@ use crate::schema::vpc_subnet;
 use crate::NetworkInterface;
 use chrono::{DateTime, Utc};
 use db_macros::Resource;
+use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use nexus_types::external_api::params;
 use nexus_types::external_api::views;
 use nexus_types::identity::Resource;
 use omicron_common::api::external;
-use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use serde::Deserialize;
 use serde::Serialize;
 use std::net::IpAddr;
diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml
index 9d0c22c373..e673003036 100644
--- a/nexus/db-queries/Cargo.toml
+++ b/nexus/db-queries/Cargo.toml
@@ -52,6 +52,7 @@ usdt.workspace = true
 
 authz-macros.workspace = true
 db-macros.workspace = true
+nexus-config.workspace = true
 nexus-db-model.workspace = true
 nexus-types.workspace = true
 omicron-common.workspace = true
diff --git a/nexus/db-queries/src/db/config.rs b/nexus/db-queries/src/db/config.rs
index afe51bca66..dc938b07ac 100644
--- a/nexus/db-queries/src/db/config.rs
+++ b/nexus/db-queries/src/db/config.rs
@@ -4,7 +4,7 @@
 
 //! Nexus database configuration
 
-use omicron_common::postgres_config::PostgresConfigWithUrl;
+use nexus_config::PostgresConfigWithUrl;
 use serde::Deserialize;
 use serde::Serialize;
 use serde_with::serde_as;
diff --git a/nexus/db-queries/src/db/datastore/db_metadata.rs b/nexus/db-queries/src/db/datastore/db_metadata.rs
index 1f9ec2c028..cbbeaf2aa4 100644
--- a/nexus/db-queries/src/db/datastore/db_metadata.rs
+++ b/nexus/db-queries/src/db/datastore/db_metadata.rs
@@ -12,9 +12,9 @@ use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection};
 use camino::{Utf8Path, Utf8PathBuf};
 use chrono::Utc;
 use diesel::prelude::*;
+use nexus_config::SchemaConfig;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::SemverVersion;
-use omicron_common::nexus_config::SchemaConfig;
 use slog::Logger;
 use std::collections::BTreeSet;
 use std::ops::Bound;
diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs
index b5eff6cb85..dea6fcb997 100644
--- a/nexus/db-queries/src/db/datastore/mod.rs
+++ b/nexus/db-queries/src/db/datastore/mod.rs
@@ -33,6 +33,7 @@ use diesel::prelude::*;
 use diesel::query_builder::{QueryFragment, QueryId};
 use diesel::query_dsl::methods::LoadQuery;
 use diesel::{ExpressionMethods, QueryDsl};
+use nexus_config::SchemaConfig;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::IdentityMetadataCreateParams;
 use omicron_common::api::external::LookupType;
@@ -41,7 +42,6 @@ use omicron_common::api::external::SemverVersion;
 use omicron_common::backoff::{
     retry_notify, retry_policy_internal_service, BackoffError,
 };
-use omicron_common::nexus_config::SchemaConfig;
 use slog::Logger;
 use std::net::Ipv6Addr;
 use std::num::NonZeroU32;
@@ -392,6 +392,7 @@ mod test {
     use chrono::{Duration, Utc};
     use futures::stream;
     use futures::StreamExt;
+    use nexus_config::RegionAllocationStrategy;
     use nexus_db_model::IpAttachState;
     use nexus_test_utils::db::test_setup_database;
     use nexus_types::external_api::params;
@@ -399,7 +400,6 @@ mod test {
     use omicron_common::api::external::{
         ByteCount, Error, IdentityMetadataCreateParams, LookupType, Name,
     };
-    use omicron_common::nexus_config::RegionAllocationStrategy;
     use omicron_test_utils::dev;
     use std::collections::HashMap;
     use std::collections::HashSet;
diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs
index 46a3e0af2d..32e059bf81 100644
--- a/nexus/db-queries/src/db/datastore/rack.rs
+++ b/nexus/db-queries/src/db/datastore/rack.rs
@@ -867,6 +867,7 @@ mod test {
     use crate::db::model::Sled;
     use async_bb8_diesel::AsyncSimpleConnection;
     use internal_params::DnsRecord;
+    use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
     use nexus_db_model::{DnsGroup, InitialDnsGroup, SledUpdate};
     use nexus_test_utils::db::test_setup_database;
     use nexus_types::external_api::shared::SiloIdentityMode;
@@ -880,7 +881,6 @@ mod test {
         IdentityMetadataCreateParams, MacAddr,
     };
     use omicron_common::api::internal::shared::SourceNatConfig;
-    use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
     use omicron_test_utils::dev;
     use std::collections::HashMap;
     use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddrV6};
diff --git a/nexus/db-queries/src/db/datastore/region.rs b/nexus/db-queries/src/db/datastore/region.rs
index b055a3e85c..912565cb2c 100644
--- a/nexus/db-queries/src/db/datastore/region.rs
+++ b/nexus/db-queries/src/db/datastore/region.rs
@@ -16,11 +16,11 @@ use crate::db::model::Region;
 use crate::transaction_retry::OptionalError;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use diesel::prelude::*;
+use nexus_config::RegionAllocationStrategy;
 use nexus_types::external_api::params;
 use omicron_common::api::external;
 use omicron_common::api::external::DeleteResult;
 use omicron_common::api::external::Error;
-use omicron_common::nexus_config::RegionAllocationStrategy;
 use slog::Logger;
 use uuid::Uuid;
 
diff --git a/nexus/db-queries/src/db/queries/network_interface.rs b/nexus/db-queries/src/db/queries/network_interface.rs
index d96b26c9b3..f6ce3e31e3 100644
--- a/nexus/db-queries/src/db/queries/network_interface.rs
+++ b/nexus/db-queries/src/db/queries/network_interface.rs
@@ -25,11 +25,11 @@ use diesel::QueryResult;
 use diesel::RunQueryDsl;
 use ipnetwork::IpNetwork;
 use ipnetwork::Ipv4Network;
+use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use nexus_db_model::{NetworkInterfaceKind, MAX_NICS_PER_INSTANCE};
 use nexus_db_model::{NetworkInterfaceKindEnum, SqlU8};
 use omicron_common::api::external;
 use omicron_common::api::external::MacAddr;
-use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use once_cell::sync::Lazy;
 use std::net::IpAddr;
 use uuid::Uuid;
diff --git a/nexus/db-queries/src/db/queries/region_allocation.rs b/nexus/db-queries/src/db/queries/region_allocation.rs
index 43e1750812..96fa2058af 100644
--- a/nexus/db-queries/src/db/queries/region_allocation.rs
+++ b/nexus/db-queries/src/db/queries/region_allocation.rs
@@ -21,6 +21,7 @@ use diesel::{
     Insertable, IntoSql, JoinOnDsl, NullableExpressionMethods, QueryDsl,
     RunQueryDsl,
 };
+use nexus_config::RegionAllocationStrategy;
 use nexus_db_model::queries::region_allocation::{
     candidate_datasets, candidate_regions, candidate_zpools, cockroach_md5,
     do_insert, inserted_regions, old_regions, old_zpool_usage,
@@ -31,7 +32,6 @@ use nexus_db_model::to_db_sled_policy;
 use nexus_db_model::SledState;
 use nexus_types::external_api::views::SledPolicy;
 use omicron_common::api::external;
-use omicron_common::nexus_config::RegionAllocationStrategy;
 
 const NOT_ENOUGH_DATASETS_SENTINEL: &'static str = "Not enough datasets";
 const NOT_ENOUGH_ZPOOL_SPACE_SENTINEL: &'static str = "Not enough space";
diff --git a/nexus/reconfigurator/execution/Cargo.toml b/nexus/reconfigurator/execution/Cargo.toml
index 2f4807d38d..62155d9783 100644
--- a/nexus/reconfigurator/execution/Cargo.toml
+++ b/nexus/reconfigurator/execution/Cargo.toml
@@ -12,6 +12,7 @@ dns-service-client.workspace = true
 futures.workspace = true
 illumos-utils.workspace = true
 internal-dns.workspace = true
+nexus-config.workspace = true
 nexus-db-model.workspace = true
 nexus-db-queries.workspace = true
 nexus-types.workspace = true
diff --git a/nexus/reconfigurator/execution/src/resource_allocation.rs b/nexus/reconfigurator/execution/src/resource_allocation.rs
index 7f3ebb9876..8ca44df39e 100644
--- a/nexus/reconfigurator/execution/src/resource_allocation.rs
+++ b/nexus/reconfigurator/execution/src/resource_allocation.rs
@@ -486,6 +486,7 @@ impl<'a> ResourceAllocator<'a> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
     use nexus_test_utils_macros::nexus_test;
     use nexus_types::deployment::OmicronZoneConfig;
     use nexus_types::deployment::OmicronZoneDataset;
@@ -499,7 +500,6 @@ mod tests {
     use omicron_common::api::external::IpNet;
     use omicron_common::api::external::MacAddr;
     use omicron_common::api::external::Vni;
-    use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
     use std::net::IpAddr;
     use std::net::Ipv6Addr;
     use std::net::SocketAddrV6;
diff --git a/nexus/reconfigurator/planning/Cargo.toml b/nexus/reconfigurator/planning/Cargo.toml
index 35ead9494e..3ec616168c 100644
--- a/nexus/reconfigurator/planning/Cargo.toml
+++ b/nexus/reconfigurator/planning/Cargo.toml
@@ -9,6 +9,7 @@ chrono.workspace = true
 internal-dns.workspace = true
 ipnet.workspace = true
 ipnetwork.workspace = true
+nexus-config.workspace = true
 nexus-inventory.workspace = true
 nexus-types.workspace = true
 omicron-common.workspace = true
diff --git a/nexus/reconfigurator/planning/src/blueprint_builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder.rs
index 2263a99ce0..d58d798770 100644
--- a/nexus/reconfigurator/planning/src/blueprint_builder.rs
+++ b/nexus/reconfigurator/planning/src/blueprint_builder.rs
@@ -10,6 +10,7 @@ use anyhow::bail;
 use internal_dns::config::Host;
 use internal_dns::config::ZoneVariant;
 use ipnet::IpAdd;
+use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use nexus_inventory::now_db_precision;
 use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::NetworkInterface;
@@ -34,7 +35,6 @@ use omicron_common::api::external::Generation;
 use omicron_common::api::external::IpNet;
 use omicron_common::api::external::MacAddr;
 use omicron_common::api::external::Vni;
-use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::collections::HashSet;
diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs
index 9ba30bab64..a213f7da72 100644
--- a/nexus/src/app/background/init.rs
+++ b/nexus/src/app/background/init.rs
@@ -18,12 +18,12 @@ use super::phantom_disks;
 use super::region_replacement;
 use super::sync_service_zone_nat::ServiceZoneNatTracker;
 use crate::app::sagas::SagaRequest;
+use nexus_config::BackgroundTaskConfig;
+use nexus_config::DnsTasksConfig;
 use nexus_db_model::DnsGroup;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db::DataStore;
 use omicron_common::api::internal::shared::SwitchLocation;
-use omicron_common::nexus_config::BackgroundTaskConfig;
-use omicron_common::nexus_config::DnsTasksConfig;
 use std::collections::BTreeMap;
 use std::collections::HashMap;
 use std::sync::Arc;
diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs
index 969320617f..781e39ac83 100644
--- a/nexus/src/app/mod.rs
+++ b/nexus/src/app/mod.rs
@@ -7,7 +7,6 @@
 use self::external_endpoints::NexusCertResolver;
 use crate::app::oximeter::LazyTimeseriesClient;
 use crate::app::sagas::SagaRequest;
-use crate::config;
 use crate::populate::populate_start;
 use crate::populate::PopulateArgs;
 use crate::populate::PopulateStatus;
@@ -16,6 +15,10 @@ use crate::DropshotServer;
 use ::oximeter::types::ProducerRegistry;
 use anyhow::anyhow;
 use internal_dns::ServiceName;
+use nexus_config::NexusConfig;
+use nexus_config::RegionAllocationStrategy;
+use nexus_config::Tunables;
+use nexus_config::UpdatesConfig;
 use nexus_db_queries::authn;
 use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
@@ -25,7 +28,6 @@ use omicron_common::address::MGD_PORT;
 use omicron_common::address::MGS_PORT;
 use omicron_common::api::external::Error;
 use omicron_common::api::internal::shared::SwitchLocation;
-use omicron_common::nexus_config::RegionAllocationStrategy;
 use slog::Logger;
 use std::collections::HashMap;
 use std::net::{IpAddr, Ipv6Addr};
@@ -147,10 +149,10 @@ pub struct Nexus {
 
     /// Contents of the trusted root role for the TUF repository.
     #[allow(dead_code)]
-    updates_config: Option<config::UpdatesConfig>,
+    updates_config: Option<UpdatesConfig>,
 
     /// The tunable parameters from a configuration file
-    tunables: config::Tunables,
+    tunables: Tunables,
 
     /// Operational context used for Instance allocation
     opctx_alloc: OpContext,
@@ -200,7 +202,7 @@ impl Nexus {
         resolver: internal_dns::resolver::Resolver,
         pool: db::Pool,
         producer_registry: &ProducerRegistry,
-        config: &config::Config,
+        config: &NexusConfig,
         authz: Arc<authz::Authz>,
     ) -> Result<Arc<Nexus>, String> {
         let pool = Arc::new(pool);
@@ -515,7 +517,7 @@ impl Nexus {
     }
 
     /// Return the tunable configuration parameters, e.g. for use in tests.
-    pub fn tunables(&self) -> &config::Tunables {
+    pub fn tunables(&self) -> &Tunables {
         &self.tunables
     }
 
diff --git a/nexus/src/app/vpc_subnet.rs b/nexus/src/app/vpc_subnet.rs
index b93b90e716..4c5a569201 100644
--- a/nexus/src/app/vpc_subnet.rs
+++ b/nexus/src/app/vpc_subnet.rs
@@ -5,6 +5,7 @@
 //! VPC Subnets and their network interfaces
 
 use crate::external_api::params;
+use nexus_config::MIN_VPC_IPV4_SUBNET_PREFIX;
 use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db;
@@ -22,7 +23,6 @@ use omicron_common::api::external::ListResultVec;
 use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::NameOrId;
 use omicron_common::api::external::UpdateResult;
-use omicron_common::nexus_config::MIN_VPC_IPV4_SUBNET_PREFIX;
 use uuid::Uuid;
 
 impl super::Nexus {
diff --git a/nexus/src/bin/nexus.rs b/nexus/src/bin/nexus.rs
index 59a77e7ca4..452e033ce6 100644
--- a/nexus/src/bin/nexus.rs
+++ b/nexus/src/bin/nexus.rs
@@ -13,12 +13,12 @@
 use anyhow::anyhow;
 use camino::Utf8PathBuf;
 use clap::Parser;
+use nexus_config::NexusConfig;
 use omicron_common::cmd::fatal;
 use omicron_common::cmd::CmdError;
 use omicron_nexus::run_openapi_external;
 use omicron_nexus::run_openapi_internal;
 use omicron_nexus::run_server;
-use omicron_nexus::Config;
 
 #[derive(Debug, Parser)]
 #[clap(name = "nexus", about = "See README.adoc for more information")]
@@ -70,7 +70,7 @@ async fn do_run() -> Result<(), CmdError> {
                 ));
             }
         };
-        let config = Config::from_file(config_path)
+        let config = NexusConfig::from_file(config_path)
             .map_err(|e| CmdError::Failure(anyhow!(e)))?;
 
         run_server(&config).await.map_err(|err| CmdError::Failure(anyhow!(err)))
diff --git a/nexus/src/bin/schema-updater.rs b/nexus/src/bin/schema-updater.rs
index d016bd0421..6c7fe5adf3 100644
--- a/nexus/src/bin/schema-updater.rs
+++ b/nexus/src/bin/schema-updater.rs
@@ -8,12 +8,12 @@ use anyhow::{anyhow, bail};
 use camino::Utf8PathBuf;
 use clap::Parser;
 use clap::Subcommand;
+use nexus_config::PostgresConfigWithUrl;
+use nexus_config::SchemaConfig;
 use nexus_db_model::schema::SCHEMA_VERSION;
 use nexus_db_queries::db;
 use nexus_db_queries::db::DataStore;
 use omicron_common::api::external::SemverVersion;
-use omicron_common::nexus_config::SchemaConfig;
-use omicron_common::postgres_config::PostgresConfigWithUrl;
 use slog::Drain;
 use slog::Level;
 use slog::LevelFilter;
diff --git a/nexus/src/config.rs b/nexus/src/config.rs
deleted file mode 100644
index 51d745c959..0000000000
--- a/nexus/src/config.rs
+++ /dev/null
@@ -1,13 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//! Interfaces for parsing configuration files and working with a nexus server
-//! configuration
-
-// TODO: Use them directly? No need for this file
-
-pub use omicron_common::nexus_config::Config;
-pub(crate) use omicron_common::nexus_config::SchemeName;
-pub(crate) use omicron_common::nexus_config::Tunables;
-pub(crate) use omicron_common::nexus_config::UpdatesConfig;
diff --git a/nexus/src/context.rs b/nexus/src/context.rs
index d34a04437f..cf2b9d6f17 100644
--- a/nexus/src/context.rs
+++ b/nexus/src/context.rs
@@ -2,7 +2,6 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 //! Shared state used by API request handlers
-use super::config;
 use super::Nexus;
 use crate::saga_interface::SagaContext;
 use async_trait::async_trait;
@@ -13,14 +12,15 @@ use authn::external::HttpAuthnScheme;
 use camino::Utf8PathBuf;
 use chrono::Duration;
 use internal_dns::ServiceName;
+use nexus_config::NexusConfig;
+use nexus_config::PostgresConfigWithUrl;
+use nexus_config::SchemeName;
 use nexus_db_queries::authn::external::session_cookie::SessionStore;
 use nexus_db_queries::authn::ConsoleSessionWithSiloId;
 use nexus_db_queries::context::{OpContext, OpKind};
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_db_queries::{authn, authz, db};
 use omicron_common::address::{Ipv6Subnet, AZ_PREFIX};
-use omicron_common::nexus_config;
-use omicron_common::postgres_config::PostgresConfigWithUrl;
 use oximeter::types::ProducerRegistry;
 use oximeter_instruments::http::{HttpService, LatencyTracker};
 use slog::Logger;
@@ -69,7 +69,7 @@ impl ServerContext {
     pub async fn new(
         rack_id: Uuid,
         log: Logger,
-        config: &config::Config,
+        config: &NexusConfig,
     ) -> Result<Arc<ServerContext>, String> {
         let nexus_schemes = config
             .pkg
@@ -78,11 +78,11 @@ impl ServerContext {
             .iter()
             .map::<Box<dyn HttpAuthnScheme<ServerContext>>, _>(
                 |name| match name {
-                    config::SchemeName::Spoof => Box::new(HttpAuthnSpoof),
-                    config::SchemeName::SessionCookie => {
+                    SchemeName::Spoof => Box::new(HttpAuthnSpoof),
+                    SchemeName::SessionCookie => {
                         Box::new(HttpAuthnSessionCookie)
                     }
-                    config::SchemeName::AccessToken => Box::new(HttpAuthnToken),
+                    SchemeName::AccessToken => Box::new(HttpAuthnToken),
                 },
             )
             .collect();
diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs
index 1b80bc4c3c..771a78f0b1 100644
--- a/nexus/src/lib.rs
+++ b/nexus/src/lib.rs
@@ -14,7 +14,6 @@
 
 pub mod app; // Public for documentation examples
 mod cidata;
-mod config;
 mod context; // Public for documentation examples
 pub mod external_api; // Public for testing
 mod internal_api;
@@ -23,11 +22,11 @@ mod saga_interface;
 
 pub use app::test_interfaces::TestInterfaces;
 pub use app::Nexus;
-pub use config::Config;
 use context::ServerContext;
 use dropshot::ConfigDropshot;
 use external_api::http_entrypoints::external_api;
 use internal_api::http_entrypoints::internal_api;
+use nexus_config::NexusConfig;
 use nexus_types::internal_api::params::ServiceKind;
 use omicron_common::address::IpRange;
 use omicron_common::api::internal::shared::{
@@ -73,14 +72,14 @@ pub struct InternalServer {
     /// dropshot server for internal API
     http_server_internal: dropshot::HttpServer<Arc<ServerContext>>,
 
-    config: Config,
+    config: NexusConfig,
     log: Logger,
 }
 
 impl InternalServer {
     /// Start a nexus server.
     pub async fn start(
-        config: &Config,
+        config: &NexusConfig,
         log: &Logger,
     ) -> Result<InternalServer, String> {
         let log = log.new(o!("name" => config.deployment.id.to_string()));
@@ -219,7 +218,7 @@ impl nexus_test_interface::NexusServer for Server {
     type InternalServer = InternalServer;
 
     async fn start_internal(
-        config: &Config,
+        config: &NexusConfig,
         log: &Logger,
     ) -> (InternalServer, SocketAddr) {
         let internal_server =
@@ -231,7 +230,7 @@ impl nexus_test_interface::NexusServer for Server {
 
     async fn start(
         internal_server: InternalServer,
-        config: &Config,
+        config: &NexusConfig,
         services: Vec<nexus_types::internal_api::params::ServicePutRequest>,
         datasets: Vec<nexus_types::internal_api::params::DatasetCreateRequest>,
         internal_dns_zone_config: nexus_types::internal_api::params::DnsConfigParams,
@@ -343,7 +342,7 @@ impl nexus_test_interface::NexusServer for Server {
 }
 
 /// Run an instance of the Nexus server.
-pub async fn run_server(config: &Config) -> Result<(), String> {
+pub async fn run_server(config: &NexusConfig) -> Result<(), String> {
     use slog::Drain;
     let (drain, registration) =
         slog_dtrace::with_drain(
diff --git a/nexus/test-interface/Cargo.toml b/nexus/test-interface/Cargo.toml
index b96afa6dbf..004ce28545 100644
--- a/nexus/test-interface/Cargo.toml
+++ b/nexus/test-interface/Cargo.toml
@@ -6,6 +6,7 @@ license = "MPL-2.0"
 
 [dependencies]
 async-trait.workspace = true
+nexus-config.workspace = true
 nexus-types.workspace = true
 omicron-common.workspace = true
 slog.workspace = true
diff --git a/nexus/test-interface/src/lib.rs b/nexus/test-interface/src/lib.rs
index 23326a5ecb..0f53ac6445 100644
--- a/nexus/test-interface/src/lib.rs
+++ b/nexus/test-interface/src/lib.rs
@@ -32,7 +32,7 @@
 //! - integration tests -> nexus-test-utils
 
 use async_trait::async_trait;
-use omicron_common::nexus_config::Config;
+use nexus_config::NexusConfig;
 use slog::Logger;
 use std::net::{SocketAddr, SocketAddrV6};
 use uuid::Uuid;
@@ -42,14 +42,14 @@ pub trait NexusServer: Send + Sync + 'static {
     type InternalServer: Send + Sync + 'static;
 
     async fn start_internal(
-        config: &Config,
+        config: &NexusConfig,
         log: &Logger,
     ) -> (Self::InternalServer, SocketAddr);
 
     #[allow(clippy::too_many_arguments)]
     async fn start(
         internal_server: Self::InternalServer,
-        config: &Config,
+        config: &NexusConfig,
         services: Vec<nexus_types::internal_api::params::ServicePutRequest>,
         datasets: Vec<nexus_types::internal_api::params::DatasetCreateRequest>,
         internal_dns_config: nexus_types::internal_api::params::DnsConfigParams,
diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml
index 5605f33f75..e612547fa8 100644
--- a/nexus/test-utils/Cargo.toml
+++ b/nexus/test-utils/Cargo.toml
@@ -21,6 +21,7 @@ headers.workspace = true
 http.workspace = true
 hyper.workspace = true
 internal-dns.workspace = true
+nexus-config.workspace = true
 nexus-db-queries.workspace = true
 nexus-test-interface.workspace = true
 nexus-types.workspace = true
diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs
index 271025f7a7..4ef77b3352 100644
--- a/nexus/test-utils/src/lib.rs
+++ b/nexus/test-utils/src/lib.rs
@@ -17,6 +17,12 @@ use dropshot::HandlerTaskMode;
 use futures::future::BoxFuture;
 use futures::FutureExt;
 use gateway_test_utils::setup::GatewayTestContext;
+use nexus_config::Database;
+use nexus_config::DpdConfig;
+use nexus_config::InternalDns;
+use nexus_config::MgdConfig;
+use nexus_config::NexusConfig;
+use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use nexus_test_interface::NexusServer;
 use nexus_types::external_api::params::UserId;
 use nexus_types::internal_api::params::Certificate;
@@ -34,8 +40,6 @@ use omicron_common::api::external::{IdentityMetadata, Name};
 use omicron_common::api::internal::nexus::ProducerEndpoint;
 use omicron_common::api::internal::nexus::ProducerKind;
 use omicron_common::api::internal::shared::SwitchLocation;
-use omicron_common::nexus_config;
-use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use omicron_sled_agent::sim;
 use omicron_test_utils::dev;
 use oximeter_collector::Oximeter;
@@ -119,7 +123,7 @@ impl<N: NexusServer> ControlPlaneTestContext<N> {
     }
 }
 
-pub fn load_test_config() -> omicron_common::nexus_config::Config {
+pub fn load_test_config() -> NexusConfig {
     // We load as much configuration as we can from the test suite configuration
     // file.  In practice, TestContext requires that:
     //
@@ -136,9 +140,8 @@ pub fn load_test_config() -> omicron_common::nexus_config::Config {
     // configuration options, we expect many of those can be usefully configured
     // (and reconfigured) for the test suite.
     let config_file_path = Utf8Path::new("tests/config.test.toml");
-    let mut config =
-        omicron_common::nexus_config::Config::from_file(config_file_path)
-            .expect("failed to load config.test.toml");
+    let mut config = NexusConfig::from_file(config_file_path)
+        .expect("failed to load config.test.toml");
     config.deployment.id = Uuid::new_v4();
     config
 }
@@ -227,7 +230,7 @@ impl RackInitRequestBuilder {
 }
 
 pub struct ControlPlaneTestContextBuilder<'a, N: NexusServer> {
-    pub config: &'a mut omicron_common::nexus_config::Config,
+    pub config: &'a mut NexusConfig,
     test_name: &'a str,
     rack_init_builder: RackInitRequestBuilder,
 
@@ -269,10 +272,7 @@ type StepInitFn<'a, N> = Box<
 >;
 
 impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
-    pub fn new(
-        test_name: &'a str,
-        config: &'a mut omicron_common::nexus_config::Config,
-    ) -> Self {
+    pub fn new(test_name: &'a str, config: &'a mut NexusConfig) -> Self {
         let start_time = chrono::Utc::now();
         let logctx = LogContext::new(test_name, &config.pkg.log);
 
@@ -463,9 +463,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
         // Update the configuration options for Nexus, if it's launched later.
         //
         // NOTE: If dendrite is started after Nexus, this is ignored.
-        let config = omicron_common::nexus_config::DpdConfig {
-            address: std::net::SocketAddr::V6(address),
-        };
+        let config = DpdConfig { address: std::net::SocketAddr::V6(address) };
         self.config.pkg.dendrite.insert(switch_location, config);
 
         let sled_id = Uuid::parse_str(SLED_AGENT_UUID).unwrap();
@@ -489,9 +487,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
 
         debug!(log, "mgd port is {port}");
 
-        let config = omicron_common::nexus_config::MgdConfig {
-            address: std::net::SocketAddr::V6(address),
-        };
+        let config = MgdConfig { address: std::net::SocketAddr::V6(address) };
         self.config.pkg.mgd.insert(switch_location, config);
 
         let sled_id = Uuid::parse_str(SLED_AGENT_UUID).unwrap();
@@ -553,16 +549,15 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
         let log = &self.logctx.log;
         debug!(log, "Starting Nexus (internal API)");
 
-        self.config.deployment.internal_dns =
-            nexus_config::InternalDns::FromAddress {
-                address: self
-                    .internal_dns
-                    .as_ref()
-                    .expect("Must initialize internal DNS server first")
-                    .dns_server
-                    .local_address(),
-            };
-        self.config.deployment.database = nexus_config::Database::FromUrl {
+        self.config.deployment.internal_dns = InternalDns::FromAddress {
+            address: self
+                .internal_dns
+                .as_ref()
+                .expect("Must initialize internal DNS server first")
+                .dns_server
+                .local_address(),
+        };
+        self.config.deployment.database = Database::FromUrl {
             url: self
                 .database
                 .as_ref()
@@ -926,7 +921,7 @@ enum PopulateCrdb {
 /// should be done in the `crdb-seed` setup script.
 #[cfg(feature = "omicron-dev")]
 pub async fn omicron_dev_setup_with_config<N: NexusServer>(
-    config: &mut omicron_common::nexus_config::Config,
+    config: &mut NexusConfig,
 ) -> Result<ControlPlaneTestContext<N>> {
     let builder =
         ControlPlaneTestContextBuilder::<N>::new("omicron-dev", config);
@@ -959,7 +954,7 @@ pub async fn omicron_dev_setup_with_config<N: NexusServer>(
 /// Setup routine to use for tests.
 pub async fn test_setup_with_config<N: NexusServer>(
     test_name: &str,
-    config: &mut omicron_common::nexus_config::Config,
+    config: &mut NexusConfig,
     sim_mode: sim::SimMode,
     initial_cert: Option<Certificate>,
 ) -> ControlPlaneTestContext<N> {
diff --git a/nexus/tests/integration_tests/initialization.rs b/nexus/tests/integration_tests/initialization.rs
index 43a4ac8f2e..b77a121080 100644
--- a/nexus/tests/integration_tests/initialization.rs
+++ b/nexus/tests/integration_tests/initialization.rs
@@ -7,6 +7,8 @@ use std::net::{Ipv6Addr, SocketAddrV6};
 
 use gateway_messages::SpPort;
 use gateway_test_utils::setup as mgs_setup;
+use nexus_config::Database;
+use nexus_config::InternalDns;
 use nexus_test_interface::NexusServer;
 use nexus_test_utils::{load_test_config, ControlPlaneTestContextBuilder};
 use omicron_common::address::MGS_PORT;
@@ -39,17 +41,15 @@ async fn test_nexus_boots_before_cockroach() {
     // This call won't return successfully until we can...
     // 1. Contact the internal DNS system to find Cockroach
     // 2. Contact Cockroach to ensure the database has been populated
-    builder.config.deployment.database =
-        omicron_common::nexus_config::Database::FromDns;
-    builder.config.deployment.internal_dns =
-        omicron_common::nexus_config::InternalDns::FromAddress {
-            address: builder
-                .internal_dns
-                .as_ref()
-                .expect("Must start Internal DNS before acquiring an address")
-                .dns_server
-                .local_address(),
-        };
+    builder.config.deployment.database = Database::FromDns;
+    builder.config.deployment.internal_dns = InternalDns::FromAddress {
+        address: builder
+            .internal_dns
+            .as_ref()
+            .expect("Must start Internal DNS before acquiring an address")
+            .dns_server
+            .local_address(),
+    };
     let nexus_config = builder.config.clone();
     let nexus_log = log.clone();
     let nexus_handle = tokio::task::spawn(async move {
@@ -111,25 +111,23 @@ async fn test_nexus_boots_before_dendrite() {
     // This call won't return successfully until we can...
     // 1. Contact the internal DNS system to find Dendrite
     // 2. Contact Dendrite
-    builder.config.deployment.database =
-        omicron_common::nexus_config::Database::FromUrl {
-            url: builder
-                .database
-                .as_ref()
-                .expect("Must start CRDB first")
-                .pg_config()
-                .clone(),
-        };
+    builder.config.deployment.database = Database::FromUrl {
+        url: builder
+            .database
+            .as_ref()
+            .expect("Must start CRDB first")
+            .pg_config()
+            .clone(),
+    };
     builder.config.pkg.dendrite = HashMap::new();
-    builder.config.deployment.internal_dns =
-        omicron_common::nexus_config::InternalDns::FromAddress {
-            address: builder
-                .internal_dns
-                .as_ref()
-                .expect("Must start Internal DNS before acquiring an address")
-                .dns_server
-                .local_address(),
-        };
+    builder.config.deployment.internal_dns = InternalDns::FromAddress {
+        address: builder
+            .internal_dns
+            .as_ref()
+            .expect("Must start Internal DNS before acquiring an address")
+            .dns_server
+            .local_address(),
+    };
     let nexus_config = builder.config.clone();
     let nexus_log = log.clone();
     let nexus_handle = tokio::task::spawn(async move {
diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs
index 380ec1c975..f9bc4e1da7 100644
--- a/nexus/tests/integration_tests/schema.rs
+++ b/nexus/tests/integration_tests/schema.rs
@@ -6,6 +6,8 @@ use camino::Utf8PathBuf;
 use chrono::{DateTime, Utc};
 use dropshot::test_util::LogContext;
 use futures::future::BoxFuture;
+use nexus_config::NexusConfig;
+use nexus_config::SchemaConfig;
 use nexus_db_model::schema::SCHEMA_VERSION as LATEST_SCHEMA_VERSION;
 use nexus_db_queries::db::datastore::{
     all_sql_for_version_migration, EARLIEST_SUPPORTED_VERSION,
@@ -14,8 +16,6 @@ use nexus_db_queries::db::DISALLOW_FULL_TABLE_SCAN_SQL;
 use nexus_test_utils::{db, load_test_config, ControlPlaneTestContextBuilder};
 use omicron_common::api::external::SemverVersion;
 use omicron_common::api::internal::shared::SwitchLocation;
-use omicron_common::nexus_config::Config;
-use omicron_common::nexus_config::SchemaConfig;
 use omicron_test_utils::dev::db::{Client, CockroachInstance};
 use pretty_assertions::{assert_eq, assert_ne};
 use similar_asserts;
@@ -45,7 +45,7 @@ async fn test_setup_just_crdb<'a>(
 // Helper to ensure we perform the same setup for the positive and negative test
 // cases.
 async fn test_setup<'a>(
-    config: &'a mut Config,
+    config: &'a mut NexusConfig,
     name: &'static str,
 ) -> ControlPlaneTestContextBuilder<'a, omicron_nexus::Server> {
     let mut builder =
diff --git a/nexus/tests/integration_tests/subnet_allocation.rs b/nexus/tests/integration_tests/subnet_allocation.rs
index 3362d5a4ac..d9d015bf26 100644
--- a/nexus/tests/integration_tests/subnet_allocation.rs
+++ b/nexus/tests/integration_tests/subnet_allocation.rs
@@ -10,6 +10,7 @@ use dropshot::HttpErrorResponseBody;
 use http::method::Method;
 use http::StatusCode;
 use ipnetwork::Ipv4Network;
+use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use nexus_test_utils::http_testing::AuthnMode;
 use nexus_test_utils::http_testing::NexusRequest;
 use nexus_test_utils::http_testing::RequestBuilder;
@@ -23,7 +24,6 @@ use omicron_common::api::external::{
     ByteCount, IdentityMetadataCreateParams, InstanceCpuCount,
     InstanceNetworkInterface, Ipv4Net,
 };
-use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use std::net::Ipv4Addr;
 
 type ControlPlaneTestContext =
diff --git a/nexus/tests/integration_tests/updates.rs b/nexus/tests/integration_tests/updates.rs
index e830348103..9a50f9b4d5 100644
--- a/nexus/tests/integration_tests/updates.rs
+++ b/nexus/tests/integration_tests/updates.rs
@@ -13,6 +13,7 @@ use camino_tempfile::{Builder, Utf8TempDir, Utf8TempPath};
 use clap::Parser;
 use dropshot::test_util::LogContext;
 use http::{Method, StatusCode};
+use nexus_config::UpdatesConfig;
 use nexus_test_utils::http_testing::{AuthnMode, NexusRequest, RequestBuilder};
 use nexus_test_utils::{load_test_config, test_setup, test_setup_with_config};
 use omicron_common::api::external::{
@@ -20,7 +21,6 @@ use omicron_common::api::external::{
     TufRepoInsertStatus,
 };
 use omicron_common::api::internal::nexus::KnownArtifactKind;
-use omicron_common::nexus_config::UpdatesConfig;
 use omicron_sled_agent::sim;
 use pretty_assertions::assert_eq;
 use serde::Deserialize;
diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml
index a70b3b4638..d61f2d09e4 100644
--- a/sled-agent/Cargo.toml
+++ b/sled-agent/Cargo.toml
@@ -44,6 +44,7 @@ libc.workspace = true
 macaddr.workspace = true
 mg-admin-client.workspace = true
 nexus-client.workspace = true
+nexus-config.workspace = true
 omicron-common.workspace = true
 once_cell.workspace = true
 oximeter.workspace = true
diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs
index 77fd8a39de..144388a2de 100644
--- a/sled-agent/src/rack_setup/plan/service.rs
+++ b/sled-agent/src/rack_setup/plan/service.rs
@@ -857,11 +857,11 @@ struct ServicePortBuilder {
 
 impl ServicePortBuilder {
     fn new(config: &Config) -> Self {
+        use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
         use omicron_common::address::{
             DNS_OPTE_IPV4_SUBNET, DNS_OPTE_IPV6_SUBNET, NEXUS_OPTE_IPV4_SUBNET,
             NEXUS_OPTE_IPV6_SUBNET, NTP_OPTE_IPV4_SUBNET, NTP_OPTE_IPV6_SUBNET,
         };
-        use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 
         let external_dns_ips_set = config
             .external_dns_ips
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
index f3ddfdbf89..67e4a48901 100644
--- a/sled-agent/src/services.rs
+++ b/sled-agent/src/services.rs
@@ -62,6 +62,7 @@ use illumos_utils::zpool::ZpoolName;
 use illumos_utils::{execute, PFEXEC};
 use internal_dns::resolver::Resolver;
 use itertools::Itertools;
+use nexus_config::{ConfigDropshotWithTls, DeploymentConfig};
 use omicron_common::address::BOOTSTRAP_ARTIFACT_PORT;
 use omicron_common::address::CLICKHOUSE_KEEPER_PORT;
 use omicron_common::address::CLICKHOUSE_PORT;
@@ -86,9 +87,6 @@ use omicron_common::backoff::{
     retry_notify, retry_policy_internal_service_aggressive, BackoffError,
 };
 use omicron_common::ledger::{self, Ledger, Ledgerable};
-use omicron_common::nexus_config::{
-    self, ConfigDropshotWithTls, DeploymentConfig as NexusDeploymentConfig,
-};
 use once_cell::sync::OnceCell;
 use rand::prelude::SliceRandom;
 use sled_hardware::is_gimlet;
@@ -2090,7 +2088,7 @@ impl ServiceManager {
                         // Nexus takes a separate config file for parameters
                         // which cannot be known at packaging time.
                         let nexus_port = if *external_tls { 443 } else { 80 };
-                        let deployment_config = NexusDeploymentConfig {
+                        let deployment_config = DeploymentConfig {
                             id: zone_config.zone.id,
                             rack_id: sled_info.rack_id,
                             techport_external_server_port:
diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs
index 8854aee05c..f9db89135a 100644
--- a/sled-agent/src/sim/server.rs
+++ b/sled-agent/src/sim/server.rs
@@ -15,13 +15,13 @@ use crucible_agent_client::types::State as RegionState;
 use internal_dns::ServiceName;
 use nexus_client::types as NexusTypes;
 use nexus_client::types::{IpRange, Ipv4Range, Ipv6Range};
+use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use omicron_common::address::DNS_OPTE_IPV4_SUBNET;
 use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET;
 use omicron_common::api::external::MacAddr;
 use omicron_common::backoff::{
     retry_notify, retry_policy_internal_service_aggressive, BackoffError,
 };
-use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use omicron_common::FileKv;
 use slog::{info, Drain, Logger};
 use std::collections::HashMap;
diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml
index 48223e0291..0a607d5115 100644
--- a/test-utils/Cargo.toml
+++ b/test-utils/Cargo.toml
@@ -15,6 +15,7 @@ headers.workspace = true
 hex.workspace = true
 http.workspace = true
 libc.workspace = true
+nexus-config.workspace = true
 omicron-common.workspace = true
 pem.workspace = true
 ring.workspace = true
diff --git a/test-utils/src/dev/db.rs b/test-utils/src/dev/db.rs
index 3c1b46b4c2..c148a60e1c 100644
--- a/test-utils/src/dev/db.rs
+++ b/test-utils/src/dev/db.rs
@@ -8,7 +8,7 @@ use crate::dev::poll;
 use anyhow::anyhow;
 use anyhow::bail;
 use anyhow::Context;
-use omicron_common::postgres_config::PostgresConfigWithUrl;
+use nexus_config::PostgresConfigWithUrl;
 use std::collections::BTreeMap;
 use std::ffi::{OsStr, OsString};
 use std::fmt;

From c3f385ebe105ad7d2b281bfedfa20bf71ba5b434 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Karen=20C=C3=A1rcamo?= <karencfv@users.noreply.github.com>
Date: Thu, 7 Mar 2024 09:36:19 +1300
Subject: [PATCH 076/157] NTP self assembling zone (#5168)

---
 illumos-utils/src/running_zone.rs |  12 +-
 package-manifest.toml             |  13 +
 sled-agent/src/services.rs        | 462 ++++++++++++++++++------------
 sled-agent/src/sled_agent.rs      |   2 +-
 sled-agent/src/smf_helper.rs      |  15 -
 smf/ntp/manifest/manifest.xml     | 180 ++++++------
 6 files changed, 385 insertions(+), 299 deletions(-)

diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs
index 1c1df01980..d86a27e3f7 100644
--- a/illumos-utils/src/running_zone.rs
+++ b/illumos-utils/src/running_zone.rs
@@ -1139,7 +1139,7 @@ impl InstalledZone {
 
 #[derive(Clone)]
 pub struct FakeZoneBuilderConfig {
-    temp_dir: Arc<Utf8TempDir>,
+    temp_dir: Arc<Utf8PathBuf>,
 }
 
 #[derive(Clone, Default)]
@@ -1157,10 +1157,14 @@ pub struct ZoneBuilderFactory {
 
 impl ZoneBuilderFactory {
     /// For use in unit tests that don't require actual zone creation to occur.
-    pub fn fake() -> Self {
+    pub fn fake(temp_dir: Option<&String>) -> Self {
+        let temp_dir = match temp_dir {
+            Some(dir) => Utf8PathBuf::from(dir),
+            None => Utf8TempDir::new().unwrap().into_path(),
+        };
         Self {
             fake_cfg: Some(FakeZoneBuilderConfig {
-                temp_dir: Arc::new(Utf8TempDir::new().unwrap()),
+                temp_dir: Arc::new(temp_dir),
             }),
         }
     }
@@ -1280,7 +1284,7 @@ impl<'a> ZoneBuilder<'a> {
             .new_control(None)
             .map_err(move |err| InstallZoneError::CreateVnic { zone, err })?;
         let fake_cfg = self.fake_cfg.unwrap();
-        let temp_dir = fake_cfg.temp_dir.path().to_path_buf();
+        let temp_dir = fake_cfg.temp_dir;
         (|| {
             let full_zone_name = InstalledZone::get_zone_name(
                 self.zone_type?,
diff --git a/package-manifest.toml b/package-manifest.toml
index 065da4bf42..13123df9d0 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -273,12 +273,25 @@ output.type = "zone"
 [package.ntp]
 service_name = "ntp"
 only_for_targets.image = "standard"
+source.type = "composite"
+source.packages = [
+  "ntp-svc.tar.gz",
+  "opte-interface-setup.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz"
+]
+output.type = "zone"
+
+[package.ntp-svc]
+service_name = "ntp-svc"
+only_for_targets.image = "standard"
 source.type = "local"
 source.paths = [
 	{ from = "smf/ntp/manifest", to = "/var/svc/manifest/site/ntp" },
 	{ from = "smf/ntp/method", to = "/var/svc/method" },
 	{ from = "smf/ntp/etc", to = "/etc" },
 ]
+output.intermediate_only = true
 output.type = "zone"
 
 [package.omicron-gateway]
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
index 67e4a48901..41acf6c079 100644
--- a/sled-agent/src/services.rs
+++ b/sled-agent/src/services.rs
@@ -36,7 +36,6 @@ use crate::params::{
 };
 use crate::profile::*;
 use crate::services_migration::{AllZoneRequests, SERVICES_LEDGER_FILENAME};
-use crate::smf_helper::Service;
 use crate::smf_helper::SmfHelper;
 use crate::zone_bundle::BundleError;
 use crate::zone_bundle::ZoneBundler;
@@ -960,6 +959,7 @@ impl ServiceManager {
         self.ensure_all_omicron_zones(
             &mut existing_zones,
             omicron_zones_config,
+            None,
         )
         .await?;
         Ok(())
@@ -1335,64 +1335,10 @@ impl ServiceManager {
         needed
     }
 
-    async fn configure_dns_client(
-        &self,
-        running_zone: &RunningZone,
-        dns_servers: &[IpAddr],
-        domain: &Option<String>,
-    ) -> Result<(), Error> {
-        struct DnsClient {}
-
-        impl crate::smf_helper::Service for DnsClient {
-            fn service_name(&self) -> String {
-                "dns_client".to_string()
-            }
-            fn smf_name(&self) -> String {
-                "svc:/network/dns/client".to_string()
-            }
-            fn should_import(&self) -> bool {
-                false
-            }
-        }
-
-        let service = DnsClient {};
-        let smfh = SmfHelper::new(&running_zone, &service);
-
-        let etc = running_zone.root().join("etc");
-        let resolv_conf = etc.join("resolv.conf");
-        let nsswitch_conf = etc.join("nsswitch.conf");
-        let nsswitch_dns = etc.join("nsswitch.dns");
-
-        if dns_servers.is_empty() {
-            // Disable the dns/client service
-            smfh.disable()?;
-        } else {
-            debug!(self.inner.log, "enabling {:?}", service.service_name());
-            let mut config = String::new();
-            if let Some(d) = domain {
-                config.push_str(&format!("domain {d}\n"));
-            }
-            for s in dns_servers {
-                config.push_str(&format!("nameserver {s}\n"));
-            }
-
-            debug!(self.inner.log, "creating {resolv_conf}");
-            tokio::fs::write(&resolv_conf, config)
-                .await
-                .map_err(|err| Error::io_path(&resolv_conf, err))?;
-
-            tokio::fs::copy(&nsswitch_dns, &nsswitch_conf)
-                .await
-                .map_err(|err| Error::io_path(&nsswitch_dns, err))?;
-
-            smfh.refresh()?;
-            smfh.enable()?;
-        }
-        Ok(())
-    }
-
     async fn dns_install(
         info: &SledAgentInfo,
+        ip_addrs: Option<Vec<IpAddr>>,
+        domain: &Option<String>,
     ) -> Result<ServiceBuilder, Error> {
         // We want to configure the dns/install SMF service inside the
         // zone with the list of DNS nameservers.  This will cause
@@ -1405,18 +1351,45 @@ impl ServiceManager {
         // supplying values for an existing property group on the SMF
         // *service*.  We're not creating a new property group, nor are
         // we configuring a property group on the instance.
-        let all_nameservers = info
-            .resolver
-            .lookup_all_ipv6(internal_dns::ServiceName::InternalDns)
-            .await?;
+
+        // Users may decide to provide specific addresses to set as
+        // nameservers, or this information can be retrieved from
+        // from SledAgentInfo.
+        let nameservers = match ip_addrs {
+            None => {
+                let addrs = info
+                    .resolver
+                    .lookup_all_ipv6(internal_dns::ServiceName::InternalDns)
+                    .await?;
+
+                let mut servers: Vec<IpAddr> = vec![];
+                for a in addrs {
+                    let ip = IpAddr::V6(a);
+                    servers.push(ip);
+                }
+
+                servers
+            }
+            Some(servers) => servers,
+        };
+
         let mut dns_config_builder = PropertyGroupBuilder::new("install_props");
-        for ns_addr in &all_nameservers {
+        for ns_addr in &nameservers {
             dns_config_builder = dns_config_builder.add_property(
                 "nameserver",
                 "net_address",
                 &ns_addr.to_string(),
             );
         }
+
+        match domain {
+            Some(d) => {
+                dns_config_builder =
+                    dns_config_builder.add_property("domain", "astring", &d)
+            }
+            None => (),
+        }
+
         Ok(ServiceBuilder::new("network/dns/install")
             .add_property_group(dns_config_builder)
             // We do need to enable the default instance of the
@@ -1476,6 +1449,7 @@ impl ServiceManager {
         request: ZoneArgs<'_>,
         filesystems: &[zone::Fs],
         data_links: &[String],
+        fake_install_dir: Option<&String>,
     ) -> Result<RunningZone, Error> {
         let device_names = Self::devices_needed(&request)?;
         let (bootstrap_vnic, bootstrap_name_and_address) =
@@ -1541,7 +1515,11 @@ impl ServiceManager {
             ZoneArgs::Switch(_) => "switch".to_string(),
         };
 
-        let mut zone_builder = ZoneBuilderFactory::default().builder();
+        // We use the fake initialiser for testing
+        let mut zone_builder = match fake_install_dir {
+            None => ZoneBuilderFactory::default().builder(),
+            Some(dir) => ZoneBuilderFactory::fake(Some(dir)).builder(),
+        };
         if let Some(uuid) = unique_name {
             zone_builder = zone_builder.with_unique_name(uuid);
         }
@@ -1564,12 +1542,21 @@ impl ServiceManager {
             .install()
             .await?;
 
+        let disabled_ssh_service = ServiceBuilder::new("network/ssh")
+            .add_instance(ServiceInstanceBuilder::new("default").disable());
+
+        let disabled_dns_client_service =
+            ServiceBuilder::new("network/dns/client")
+                .add_instance(ServiceInstanceBuilder::new("default").disable());
+
+        let enabled_dns_client_service =
+            ServiceBuilder::new("network/dns/client")
+                .add_instance(ServiceInstanceBuilder::new("default"));
+
         // TODO(https://github.com/oxidecomputer/omicron/issues/1898):
         //
         // These zones are self-assembling -- after they boot, there should
         // be no "zlogin" necessary to initialize.
-        let disabled_ssh_service = ServiceBuilder::new("network/ssh")
-            .add_instance(ServiceInstanceBuilder::new("default").disable());
         match &request {
             ZoneArgs::Omicron(OmicronZoneConfigLocal {
                 zone:
@@ -1593,7 +1580,7 @@ impl ServiceManager {
                     listen_addr,
                 )?;
 
-                let dns_service = Self::dns_install(info).await?;
+                let dns_service = Self::dns_install(info, None, &None).await?;
 
                 let config = PropertyGroupBuilder::new("config")
                     .add_property("listen_addr", "astring", listen_addr)
@@ -1609,7 +1596,8 @@ impl ServiceManager {
                     .add_service(nw_setup_service)
                     .add_service(disabled_ssh_service)
                     .add_service(clickhouse_service)
-                    .add_service(dns_service);
+                    .add_service(dns_service)
+                    .add_service(enabled_dns_client_service);
                 profile
                     .add_to_zone(&self.inner.log, &installed_zone)
                     .await
@@ -1641,7 +1629,7 @@ impl ServiceManager {
                     listen_addr,
                 )?;
 
-                let dns_service = Self::dns_install(info).await?;
+                let dns_service = Self::dns_install(info, None, &None).await?;
 
                 let config = PropertyGroupBuilder::new("config")
                     .add_property("listen_addr", "astring", listen_addr)
@@ -1657,7 +1645,8 @@ impl ServiceManager {
                     .add_service(nw_setup_service)
                     .add_service(disabled_ssh_service)
                     .add_service(clickhouse_keeper_service)
-                    .add_service(dns_service);
+                    .add_service(dns_service)
+                    .add_service(enabled_dns_client_service);
                 profile
                     .add_to_zone(&self.inner.log, &installed_zone)
                     .await
@@ -1696,7 +1685,7 @@ impl ServiceManager {
                     listen_addr,
                 )?;
 
-                let dns_service = Self::dns_install(info).await?;
+                let dns_service = Self::dns_install(info, None, &None).await?;
 
                 // Configure the CockroachDB service.
                 let cockroachdb_config = PropertyGroupBuilder::new("config")
@@ -1713,7 +1702,8 @@ impl ServiceManager {
                     .add_service(nw_setup_service)
                     .add_service(disabled_ssh_service)
                     .add_service(cockroachdb_service)
-                    .add_service(dns_service);
+                    .add_service(dns_service)
+                    .add_service(enabled_dns_client_service);
                 profile
                     .add_to_zone(&self.inner.log, &installed_zone)
                     .await
@@ -1760,6 +1750,7 @@ impl ServiceManager {
                 let profile = ProfileBuilder::new("omicron")
                     .add_service(nw_setup_service)
                     .add_service(disabled_ssh_service)
+                    .add_service(disabled_dns_client_service)
                     .add_service(
                         ServiceBuilder::new("oxide/crucible/agent")
                             .add_instance(
@@ -1805,6 +1796,7 @@ impl ServiceManager {
                 let profile = ProfileBuilder::new("omicron")
                     .add_service(nw_setup_service)
                     .add_service(disabled_ssh_service)
+                    .add_service(disabled_dns_client_service)
                     .add_service(
                         ServiceBuilder::new("oxide/crucible/pantry")
                             .add_instance(
@@ -1859,7 +1851,8 @@ impl ServiceManager {
                 let profile = ProfileBuilder::new("omicron")
                     .add_service(nw_setup_service)
                     .add_service(disabled_ssh_service)
-                    .add_service(oximeter_service);
+                    .add_service(oximeter_service)
+                    .add_service(disabled_dns_client_service);
                 profile
                     .add_to_zone(&self.inner.log, &installed_zone)
                     .await
@@ -1926,7 +1919,8 @@ impl ServiceManager {
                     .add_service(nw_setup_service)
                     .add_service(opte_interface_setup)
                     .add_service(disabled_ssh_service)
-                    .add_service(external_dns_service);
+                    .add_service(external_dns_service)
+                    .add_service(disabled_dns_client_service);
                 profile
                     .add_to_zone(&self.inner.log, &installed_zone)
                     .await
@@ -1935,6 +1929,115 @@ impl ServiceManager {
                     })?;
                 return Ok(RunningZone::boot(installed_zone).await?);
             }
+            ZoneArgs::Omicron(OmicronZoneConfigLocal {
+                zone:
+                    OmicronZoneConfig {
+                        zone_type:
+                            OmicronZoneType::BoundaryNtp {
+                                dns_servers,
+                                ntp_servers,
+                                domain,
+                                ..
+                            },
+                        underlay_address,
+                        ..
+                    },
+                ..
+            })
+            | ZoneArgs::Omicron(OmicronZoneConfigLocal {
+                zone:
+                    OmicronZoneConfig {
+                        zone_type:
+                            OmicronZoneType::InternalNtp {
+                                dns_servers,
+                                ntp_servers,
+                                domain,
+                                ..
+                            },
+                        underlay_address,
+                        ..
+                    },
+                ..
+            }) => {
+                let Some(info) = self.inner.sled_info.get() else {
+                    return Err(Error::SledAgentNotReady);
+                };
+
+                let static_addr = underlay_address.to_string();
+
+                let nw_setup_service = Self::zone_network_setup_install(
+                    info,
+                    &installed_zone,
+                    &static_addr.clone(),
+                )?;
+
+                let is_boundary = matches!(
+                    // It's safe to unwrap here as we already know it's one of InternalNTP or BoundaryNtp.
+                    request.omicron_type().unwrap(),
+                    OmicronZoneType::BoundaryNtp { .. }
+                );
+
+                let rack_net =
+                    Ipv6Subnet::<RACK_PREFIX>::new(info.underlay_address)
+                        .net()
+                        .to_string();
+
+                let dns_install_service =
+                    Self::dns_install(info, Some(dns_servers.to_vec()), domain)
+                        .await?;
+
+                let mut ntp_config = PropertyGroupBuilder::new("config")
+                    .add_property("allow", "astring", &rack_net)
+                    .add_property(
+                        "boundary",
+                        "boolean",
+                        &is_boundary.to_string(),
+                    );
+
+                for s in ntp_servers {
+                    ntp_config = ntp_config.add_property(
+                        "server",
+                        "astring",
+                        &s.to_string(),
+                    );
+                }
+
+                let dns_client_service;
+                if dns_servers.is_empty() {
+                    dns_client_service = disabled_dns_client_service;
+                } else {
+                    dns_client_service = enabled_dns_client_service;
+                }
+
+                let ntp_service = ServiceBuilder::new("oxide/ntp")
+                    .add_instance(
+                        ServiceInstanceBuilder::new("default")
+                            .add_property_group(ntp_config),
+                    );
+
+                let mut profile = ProfileBuilder::new("omicron")
+                    .add_service(nw_setup_service)
+                    .add_service(disabled_ssh_service)
+                    .add_service(dns_install_service)
+                    .add_service(dns_client_service)
+                    .add_service(ntp_service);
+
+                // Only Boundary NTP needs an OPTE interface and port configured.
+                if is_boundary {
+                    let opte_interface_setup =
+                        Self::opte_interface_set_up_install(&installed_zone)?;
+                    profile = profile.add_service(opte_interface_setup)
+                }
+
+                profile
+                    .add_to_zone(&self.inner.log, &installed_zone)
+                    .await
+                    .map_err(|err| {
+                        Error::io("Failed to set up NTP profile", err)
+                    })?;
+
+                return Ok(RunningZone::boot(installed_zone).await?);
+            }
             _ => {}
         }
 
@@ -2249,73 +2352,14 @@ impl ServiceManager {
                         // service is enabled.
                         smfh.refresh()?;
                     }
-                    OmicronZoneType::BoundaryNtp {
-                        ntp_servers,
-                        dns_servers,
-                        domain,
-                        ..
-                    }
-                    | OmicronZoneType::InternalNtp {
-                        ntp_servers,
-                        dns_servers,
-                        domain,
-                        ..
-                    } => {
-                        let boundary = matches!(
-                            &zone_config.zone.zone_type,
-                            OmicronZoneType::BoundaryNtp { .. }
-                        );
-                        info!(
-                            self.inner.log,
-                            "Set up NTP service boundary={}, Servers={:?}",
-                            boundary,
-                            ntp_servers
-                        );
-
-                        let sled_info =
-                            if let Some(info) = self.inner.sled_info.get() {
-                                info
-                            } else {
-                                return Err(Error::SledAgentNotReady);
-                            };
-
-                        let rack_net = Ipv6Subnet::<RACK_PREFIX>::new(
-                            sled_info.underlay_address,
-                        )
-                        .net();
-
-                        smfh.setprop("config/allow", &format!("{}", rack_net))?;
-                        smfh.setprop(
-                            "config/boundary",
-                            if boundary { "true" } else { "false" },
-                        )?;
-
-                        if boundary {
-                            // Configure OPTE port for boundary NTP
-                            running_zone
-                                .ensure_address_for_port("public", 0)
-                                .await?;
-                        }
-
-                        smfh.delpropvalue("config/server", "*")?;
-                        for server in ntp_servers {
-                            smfh.addpropvalue("config/server", server)?;
-                        }
-                        self.configure_dns_client(
-                            &running_zone,
-                            &dns_servers,
-                            &domain,
-                        )
-                        .await?;
-
-                        smfh.refresh()?;
-                    }
-                    OmicronZoneType::Clickhouse { .. }
+                    OmicronZoneType::BoundaryNtp { .. }
+                    | OmicronZoneType::Clickhouse { .. }
                     | OmicronZoneType::ClickhouseKeeper { .. }
                     | OmicronZoneType::CockroachDb { .. }
                     | OmicronZoneType::Crucible { .. }
                     | OmicronZoneType::CruciblePantry { .. }
                     | OmicronZoneType::ExternalDns { .. }
+                    | OmicronZoneType::InternalNtp { .. }
                     | OmicronZoneType::Oximeter { .. } => {
                         panic!(
                             "{} is a service which exists as part of a \
@@ -2755,6 +2799,7 @@ impl ServiceManager {
         zone: &OmicronZoneConfig,
         time_is_synchronized: bool,
         all_u2_pools: &Vec<ZpoolName>,
+        fake_install_dir: Option<&String>,
     ) -> Result<OmicronZone, Error> {
         // Ensure the zone has been fully removed before we try to boot it.
         //
@@ -2781,6 +2826,7 @@ impl ServiceManager {
                 &[],
                 // data_links=
                 &[],
+                fake_install_dir,
             )
             .await?;
 
@@ -2800,6 +2846,7 @@ impl ServiceManager {
         requests: impl Iterator<Item = &OmicronZoneConfig> + Clone,
         time_is_synchronized: bool,
         all_u2_pools: &Vec<ZpoolName>,
+        fake_install_dir: Option<&String>,
     ) -> Result<StartZonesResult, Error> {
         if let Some(name) =
             requests.clone().map(|zone| zone.zone_name()).duplicates().next()
@@ -2811,9 +2858,14 @@ impl ServiceManager {
         }
 
         let futures = requests.map(|zone| async move {
-            self.start_omicron_zone(&zone, time_is_synchronized, all_u2_pools)
-                .await
-                .map_err(|err| (zone.zone_name().to_string(), err))
+            self.start_omicron_zone(
+                &zone,
+                time_is_synchronized,
+                all_u2_pools,
+                fake_install_dir,
+            )
+            .await
+            .map_err(|err| (zone.zone_name().to_string(), err))
         });
 
         let results = futures::future::join_all(futures).await;
@@ -2895,6 +2947,7 @@ impl ServiceManager {
     pub async fn ensure_all_omicron_zones_persistent(
         &self,
         mut request: OmicronZonesConfig,
+        fake_install_dir: Option<&String>,
     ) -> Result<(), Error> {
         let log = &self.inner.log;
 
@@ -2957,7 +3010,12 @@ impl ServiceManager {
 
         let omicron_generation = request.generation;
         let ledger_generation = ledger_zone_config.ledger_generation;
-        self.ensure_all_omicron_zones(&mut existing_zones, request).await?;
+        self.ensure_all_omicron_zones(
+            &mut existing_zones,
+            request,
+            fake_install_dir,
+        )
+        .await?;
         let zones = existing_zones
             .values()
             .map(|omicron_zone| omicron_zone.config.clone())
@@ -3003,6 +3061,7 @@ impl ServiceManager {
         // lock held when calling this function.
         existing_zones: &mut MutexGuard<'_, ZoneMap>,
         new_request: OmicronZonesConfig,
+        fake_install_dir: Option<&String>,
     ) -> Result<(), Error> {
         // Do some data-normalization to ensure we can compare the "requested
         // set" vs the "existing set" as HashSets.
@@ -3017,7 +3076,6 @@ impl ServiceManager {
             old_zone_configs.difference(&requested_zones_set);
         let zones_to_be_added =
             requested_zones_set.difference(&old_zone_configs);
-
         // Destroy zones that should not be running
         for zone in zones_to_be_removed {
             self.zone_bundle_and_try_remove(existing_zones, &zone).await;
@@ -3040,6 +3098,7 @@ impl ServiceManager {
                 zones_to_be_added,
                 time_is_synchronized,
                 &all_u2_pools,
+                fake_install_dir,
             )
             .await?;
 
@@ -3854,8 +3913,9 @@ impl ServiceManager {
         let zone_request =
             SwitchZoneConfigLocal { root, zone: request.clone() };
         let zone_args = ZoneArgs::Switch(&zone_request);
-        let zone =
-            self.initialize_zone(zone_args, filesystems, data_links).await?;
+        let zone = self
+            .initialize_zone(zone_args, filesystems, data_links, None)
+            .await?;
         *sled_zone = SledLocalZone::Running { request: request.clone(), zone };
         Ok(())
     }
@@ -4097,6 +4157,7 @@ mod test {
         mgr: &ServiceManager,
         id: Uuid,
         generation: Generation,
+        tmp_dir: String,
     ) {
         let address =
             SocketAddrV6::new(Ipv6Addr::LOCALHOST, EXPECTED_PORT, 0, 0);
@@ -4110,6 +4171,7 @@ mod test {
                 dns_servers: vec![],
                 domain: None,
             },
+            tmp_dir,
         )
         .await
         .expect("Could not create service");
@@ -4120,17 +4182,21 @@ mod test {
         id: Uuid,
         generation: Generation,
         zone_type: OmicronZoneType,
+        tmp_dir: String,
     ) -> Result<(), Error> {
         let zone_prefix = format!("oxz_{}", zone_type.zone_type_str());
         let _expectations = expect_new_service(&zone_prefix);
-        mgr.ensure_all_omicron_zones_persistent(OmicronZonesConfig {
-            generation,
-            zones: vec![OmicronZoneConfig {
-                id,
-                underlay_address: Ipv6Addr::LOCALHOST,
-                zone_type,
-            }],
-        })
+        mgr.ensure_all_omicron_zones_persistent(
+            OmicronZonesConfig {
+                generation,
+                zones: vec![OmicronZoneConfig {
+                    id,
+                    underlay_address: Ipv6Addr::LOCALHOST,
+                    zone_type,
+                }],
+            },
+            Some(&tmp_dir),
+        )
         .await
     }
 
@@ -4140,22 +4206,26 @@ mod test {
         mgr: &ServiceManager,
         id: Uuid,
         generation: Generation,
+        tmp_dir: String,
     ) {
         let address =
             SocketAddrV6::new(Ipv6Addr::LOCALHOST, EXPECTED_PORT, 0, 0);
-        mgr.ensure_all_omicron_zones_persistent(OmicronZonesConfig {
-            generation,
-            zones: vec![OmicronZoneConfig {
-                id,
-                underlay_address: Ipv6Addr::LOCALHOST,
-                zone_type: OmicronZoneType::InternalNtp {
-                    address,
-                    ntp_servers: vec![],
-                    dns_servers: vec![],
-                    domain: None,
-                },
-            }],
-        })
+        mgr.ensure_all_omicron_zones_persistent(
+            OmicronZonesConfig {
+                generation,
+                zones: vec![OmicronZoneConfig {
+                    id,
+                    underlay_address: Ipv6Addr::LOCALHOST,
+                    zone_type: OmicronZoneType::InternalNtp {
+                        address,
+                        ntp_servers: vec![],
+                        dns_servers: vec![],
+                        domain: None,
+                    },
+                }],
+            },
+            Some(&tmp_dir),
+        )
         .await
         .unwrap();
     }
@@ -4326,7 +4396,13 @@ mod test {
 
         let v2 = v1.next();
         let id = Uuid::new_v4();
-        ensure_new_service(&mgr, id, v2).await;
+        ensure_new_service(
+            &mgr,
+            id,
+            v2,
+            String::from(test_config.config_dir.path().as_str()),
+        )
+        .await;
 
         let found =
             mgr.omicron_zones_list().await.expect("failed to list zones");
@@ -4369,6 +4445,7 @@ mod test {
             id,
             v2,
             OmicronZoneType::Oximeter { address },
+            String::from(test_config.config_dir.path().as_str()),
         )
         .await;
 
@@ -4403,6 +4480,7 @@ mod test {
                 dns_servers: vec![],
                 domain: None,
             },
+            String::from(test_config.config_dir.path().as_str()),
         )
         .await
         .unwrap();
@@ -4424,9 +4502,10 @@ mod test {
 
         let v2 = Generation::new().next();
         let id = Uuid::new_v4();
-        ensure_new_service(&mgr, id, v2).await;
+        let dir = String::from(test_config.config_dir.path().as_str());
+        ensure_new_service(&mgr, id, v2, dir.clone()).await;
         let v3 = v2.next();
-        ensure_existing_service(&mgr, id, v3).await;
+        ensure_existing_service(&mgr, id, v3, dir).await;
         let found =
             mgr.omicron_zones_list().await.expect("failed to list zones");
         assert_eq!(found.generation, v3);
@@ -4454,7 +4533,13 @@ mod test {
 
         let v2 = Generation::new().next();
         let id = Uuid::new_v4();
-        ensure_new_service(&mgr, id, v2).await;
+        ensure_new_service(
+            &mgr,
+            id,
+            v2,
+            String::from(test_config.config_dir.path().as_str()),
+        )
+        .await;
         drop_service_manager(mgr);
 
         // Before we re-create the service manager - notably, using the same
@@ -4491,7 +4576,13 @@ mod test {
         let v1 = Generation::new();
         let v2 = v1.next();
         let id = Uuid::new_v4();
-        ensure_new_service(&mgr, id, v2).await;
+        ensure_new_service(
+            &mgr,
+            id,
+            v2,
+            String::from(test_config.config_dir.path().as_str()),
+        )
+        .await;
         drop_service_manager(mgr);
 
         // Next, delete the ledger. This means the zone we just created will not
@@ -4544,10 +4635,12 @@ mod test {
                 domain: None,
             },
         }];
-        mgr.ensure_all_omicron_zones_persistent(OmicronZonesConfig {
-            generation: v2,
-            zones: zones.clone(),
-        })
+
+        let tmp_dir = String::from(test_config.config_dir.path().as_str());
+        mgr.ensure_all_omicron_zones_persistent(
+            OmicronZonesConfig { generation: v2, zones: zones.clone() },
+            Some(&tmp_dir),
+        )
         .await
         .unwrap();
 
@@ -4573,11 +4666,12 @@ mod test {
 
         // Now try to apply that list with an older generation number.  This
         // shouldn't work and the reported state should be unchanged.
+        let tmp_dir = String::from(test_config.config_dir.path().as_str());
         let error = mgr
-            .ensure_all_omicron_zones_persistent(OmicronZonesConfig {
-                generation: v1,
-                zones: zones.clone(),
-            })
+            .ensure_all_omicron_zones_persistent(
+                OmicronZonesConfig { generation: v1, zones: zones.clone() },
+                Some(&tmp_dir),
+            )
             .await
             .expect_err("unexpectedly went backwards in zones generation");
         assert!(matches!(
@@ -4592,10 +4686,10 @@ mod test {
         // Now try to apply that list with the same generation number that we
         // used before.  This shouldn't work either.
         let error = mgr
-            .ensure_all_omicron_zones_persistent(OmicronZonesConfig {
-                generation: v2,
-                zones: zones.clone(),
-            })
+            .ensure_all_omicron_zones_persistent(
+                OmicronZonesConfig { generation: v2, zones: zones.clone() },
+                Some(&tmp_dir),
+            )
             .await
             .expect_err("unexpectedly changed a single zone generation");
         assert!(matches!(
@@ -4609,10 +4703,10 @@ mod test {
         // But we should be able to apply this new list of zones as long as we
         // advance the generation number.
         let v3 = v2.next();
-        mgr.ensure_all_omicron_zones_persistent(OmicronZonesConfig {
-            generation: v3,
-            zones: zones.clone(),
-        })
+        mgr.ensure_all_omicron_zones_persistent(
+            OmicronZonesConfig { generation: v3, zones: zones.clone() },
+            Some(&tmp_dir),
+        )
         .await
         .expect("failed to remove all zones in a new generation");
         let found4 =
diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs
index 07c4dbbf4e..39ba1045f7 100644
--- a/sled-agent/src/sled_agent.rs
+++ b/sled-agent/src/sled_agent.rs
@@ -908,7 +908,7 @@ impl SledAgent {
 
         self.inner
             .services
-            .ensure_all_omicron_zones_persistent(requested_zones)
+            .ensure_all_omicron_zones_persistent(requested_zones, None)
             .await?;
         Ok(())
     }
diff --git a/sled-agent/src/smf_helper.rs b/sled-agent/src/smf_helper.rs
index d9ec4f02d6..837aa59157 100644
--- a/sled-agent/src/smf_helper.rs
+++ b/sled-agent/src/smf_helper.rs
@@ -236,19 +236,4 @@ impl<'t> SmfHelper<'t> {
             })?;
         Ok(())
     }
-
-    pub fn disable(&self) -> Result<(), Error> {
-        self.running_zone
-            .run_cmd(&[
-                illumos_utils::zone::SVCADM,
-                "disable",
-                "-t",
-                &self.default_smf_name,
-            ])
-            .map_err(|err| Error::ZoneCommand {
-                intent: format!("Disable {} service", self.default_smf_name),
-                err,
-            })?;
-        Ok(())
-    }
 }
diff --git a/smf/ntp/manifest/manifest.xml b/smf/ntp/manifest/manifest.xml
index 4b3827e0e6..7783bbe76c 100644
--- a/smf/ntp/manifest/manifest.xml
+++ b/smf/ntp/manifest/manifest.xml
@@ -1,105 +1,95 @@
 <?xml version="1.0"?>
 <!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
+
 <!--
  This Source Code Form is subject to the terms of the Mozilla Public
  License, v. 2.0. If a copy of the MPL was not distributed with this
  file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
- Copyright 2023 Oxide Computer Company
+ Copyright 2024 Oxide Computer Company
 -->
-<service_bundle type="manifest"
-                name="ntp:default">
-
-<service name="oxide/ntp"
-             type="service"
-             version="1">
-
-        <create_default_instance enabled="false" />
-
-        <dependency name="network"
-                    grouping="require_any"
-                    restart_on="error"
-                    type="service">
-            <service_fmri value="svc:/network/service" />
-        </dependency>
-
-        <dependency name="filesystem"
-                    grouping="require_all"
-                    restart_on="error"
-                    type="service">
-            <service_fmri value="svc:/system/filesystem/minimal" />
-        </dependency>
-
-        <dependency name="name-services"
-                    grouping="optional_all"
-                    restart_on="none"
-                    type="service">
-            <service_fmri value="svc:/milestone/name-services" />
-        </dependency>
-
-        <dependency name="routing-setup"
-                    grouping="optional_all"
-                    restart_on="none"
-                    type="service">
-            <service_fmri value="svc:/network/routing-setup" />
-        </dependency>
-
-	<!--
-        The service is started as root, but with only the privileges it
-	    requires. The chrony daemon forks a privileged helper and then the
-	    main daemon drops everything it does not need, leaving it with
-	    completely minimal privileges (it even divests the privilege to
-        fork/exec).
-
-        The service also always starts the binary with ASLR enabled,
-	    regardless of whether it was linked with -zaslr
-	-->
-        <exec_method type="method"
-                     name="start"
-		     exec="/var/svc/method/svc-site-ntp %m"
-                     timeout_seconds="60">
-            <method_context security_flags="aslr">
-                <method_credential user="root"
-                                   group="root"
-                                   privileges="basic,!file_link_any,!proc_info,!proc_session,file_chown_self,file_dac_search,file_dac_write,net_privaddr,proc_lock_memory,proc_priocntl,proc_setid,sys_time" />
-            </method_context>
-        </exec_method>
-
-        <exec_method type="method"
-                     name="refresh"
-		     exec="/var/svc/method/svc-site-ntp %m %{restarter/contract}"
-                     timeout_seconds="60">
-        </exec_method>
-
-        <exec_method type="method"
-                     name="stop"
-		     exec="/var/svc/method/svc-site-ntp %m %{restarter/contract}"
-                     timeout_seconds="60" />
-
-        <property_group name="config"
-                        type="application">
-            <propval name="file"
-                     type="astring"
-                     value="/etc/inet/chrony.conf" />
-	    <propval name="boundary"
-                     type="boolean"
-		    value="false" />
-            <propval name="server"
-                     type="astring"
-                     value="" />
-            <propval name="allow"
-                     type="astring"
-                     value="" />
-        </property_group>
-
-        <stability value="Unstable" />
-
-        <template>
-            <common_name>
-                <loctext xml:lang="C">Network Time Protocol (NTP)</loctext>
-            </common_name>
-        </template>
-
-    </service>
+
+<service_bundle type="manifest" name="ntp:default">
+
+  <service name="oxide/ntp" type="service" version="1">
+  <create_default_instance enabled="true" />
+
+  <dependency name="network" grouping="require_any" restart_on="error"
+    type="service">
+  <service_fmri value="svc:/network/service" />
+  </dependency>
+
+  <dependency name="filesystem" grouping="require_all" restart_on="error"
+    type="service">
+  <service_fmri value="svc:/system/filesystem/minimal" />
+  </dependency>
+
+  <dependency name="name-services" grouping="require_all" restart_on="none"
+    type="service">
+  <service_fmri value="svc:/milestone/name-services" />
+  </dependency>
+
+  <dependency name="routing-setup" grouping="optional_all" restart_on="none"
+    type="service">
+  <service_fmri value="svc:/network/routing-setup" />
+  </dependency>
+
+  <dependency name='zone_network_setup' grouping='require_all' restart_on='none'
+    type='service'>
+  <service_fmri value='svc:/oxide/zone-network-setup:default' />
+  </dependency>
+
+  <!-- 
+    Only Boundary NTP needs the opte interface.
+  -->
+  <dependency name='opte_interface_setup' grouping='optional_all' restart_on='none'
+    type='service'>
+  <service_fmri value='svc:/oxide/opte-interface-setup:default' />
+  </dependency>
+
+  <!--
+    The service is started as root, but with only the privileges it
+    requires. The chrony daemon forks a privileged helper and then the
+    main daemon drops everything it does not need, leaving it with
+    completely minimal privileges (it even divests the privilege to
+    fork/exec).
+
+    The service also always starts the binary with ASLR enabled,
+    regardless of whether it was linked with -zaslr
+  -->
+  <exec_method type="method" name="start" exec="/var/svc/method/svc-site-ntp %m"
+    timeout_seconds="60">
+  <method_context security_flags="aslr">
+  <method_credential user="root" group="root"
+    privileges="basic,!file_link_any,!proc_info,!proc_session,file_chown_self,file_dac_search,file_dac_write,net_privaddr,proc_lock_memory,proc_priocntl,proc_setid,sys_time" />
+  </method_context>
+  </exec_method>
+
+  <exec_method type="method" name="refresh"
+    exec="/var/svc/method/svc-site-ntp %m %{restarter/contract}" timeout_seconds="60">
+  </exec_method>
+
+  <exec_method type="method" name="stop"
+    exec="/var/svc/method/svc-site-ntp %m %{restarter/contract}"
+    timeout_seconds="60" />
+
+  <property_group name="config" type="application">
+    <propval name="file" type="astring" value="/etc/inet/chrony.conf" />
+    <propval name="boundary" type="boolean" value="false" />
+    <propval name="server" type="astring" value="" />
+    <propval name="allow" type="astring" value="" />
+  </property_group>
+
+  <stability value="Unstable" />
+
+  <template>
+    <common_name>
+      <loctext xml:lang="C">Network Time Protocol (NTP)</loctext>
+    </common_name>
+    <description>
+      <loctext xml:lang='C'>Oxide's Network Time Protocol Service</loctext>
+    </description>
+  </template>
+  </service>
 
 </service_bundle>

From e103a5e433bf3cae5664e08db6aa7fbfc6c76a36 Mon Sep 17 00:00:00 2001
From: "oxide-reflector-bot[bot]"
 <130185838+oxide-reflector-bot[bot]@users.noreply.github.com>
Date: Wed, 6 Mar 2024 21:40:38 +0000
Subject: [PATCH 077/157] Update maghemite to ee0896f (#5209)

Updated maghemite to commit ee0896f.

Co-authored-by: reflector[bot] <130185838+reflector[bot]@users.noreply.github.com>
---
 package-manifest.toml               | 8 ++++----
 tools/maghemite_ddm_openapi_version | 2 +-
 tools/maghemite_mg_openapi_version  | 2 +-
 tools/maghemite_mgd_checksums       | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/package-manifest.toml b/package-manifest.toml
index 13123df9d0..5575efd285 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -497,7 +497,7 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "4b0e584eec455a43c36af08ae207086965cef833"
+source.commit = "ee0896f5cf9ba0799f155dd8c95f6e1dabaf80ba"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/maghemite.sha256.txt
 source.sha256 = "f1407cb9aac188d6493d2b0f948c75aad2c36668ddf4ae2a1ed80e9dd395b35d"
@@ -513,7 +513,7 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "4b0e584eec455a43c36af08ae207086965cef833"
+source.commit = "ee0896f5cf9ba0799f155dd8c95f6e1dabaf80ba"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mg-ddm.sha256.txt
 source.sha256 = "fae53cb39536dc92d97cb9610de65b0acbce285e685d7167b719ea6311844fec"
@@ -528,10 +528,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "4b0e584eec455a43c36af08ae207086965cef833"
+source.commit = "ee0896f5cf9ba0799f155dd8c95f6e1dabaf80ba"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mg-ddm.sha256.txt
-source.sha256 = "22996a6f3353296b848be729f14e78a42e7d3d6e62a4a918a5c2358ae011c8eb"
+source.sha256 = "151aeb26414989cad571b3886786efbeeafd91c41a93a747c784cdc654d5876d"
 output.type = "zone"
 output.intermediate_only = true
 
diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version
index d161091fa8..14ff749506 100644
--- a/tools/maghemite_ddm_openapi_version
+++ b/tools/maghemite_ddm_openapi_version
@@ -1,2 +1,2 @@
-COMMIT="4b0e584eec455a43c36af08ae207086965cef833"
+COMMIT="ee0896f5cf9ba0799f155dd8c95f6e1dabaf80ba"
 SHA2="0b0dbc2f8bbc5d2d9be92d64c4865f8f9335355aae62f7de9f67f81dfb3f1803"
diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version
index 475d273f4a..a2becb4efc 100644
--- a/tools/maghemite_mg_openapi_version
+++ b/tools/maghemite_mg_openapi_version
@@ -1,2 +1,2 @@
-COMMIT="4b0e584eec455a43c36af08ae207086965cef833"
+COMMIT="ee0896f5cf9ba0799f155dd8c95f6e1dabaf80ba"
 SHA2="0ac038bbaa54d0ae0ac5ccaeff48f03070618372cca26c9d09b716b909bf9355"
diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums
index ab84fafc01..2f58a88808 100644
--- a/tools/maghemite_mgd_checksums
+++ b/tools/maghemite_mgd_checksums
@@ -1,2 +1,2 @@
-CIDL_SHA256="22996a6f3353296b848be729f14e78a42e7d3d6e62a4a918a5c2358ae011c8eb"
+CIDL_SHA256="151aeb26414989cad571b3886786efbeeafd91c41a93a747c784cdc654d5876d"
 MGD_LINUX_SHA256="943b0a52d279bde55a419e2cdb24873acc32703bc97bd599376117ee0edc1511"
\ No newline at end of file

From 390b51376ac592cc4f68bb421a7f304c1f2c144a Mon Sep 17 00:00:00 2001
From: Rain <rain@oxide.computer>
Date: Wed, 6 Mar 2024 16:39:53 -0800
Subject: [PATCH 078/157] [reconfigurator] add logger to BlueprintBuilder
 (#5208)

I wanted to log a few debug statements and maybe errors, and this is pretty
straightforward.
---
 .../db-queries/src/db/datastore/deployment.rs |  4 ++
 .../planning/src/blueprint_builder.rs         | 56 +++++++++++++++++++
 nexus/reconfigurator/planning/src/planner.rs  |  1 +
 3 files changed, 61 insertions(+)

diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs
index 020916928d..a00318c9dc 100644
--- a/nexus/db-queries/src/db/datastore/deployment.rs
+++ b/nexus/db-queries/src/db/datastore/deployment.rs
@@ -1349,6 +1349,7 @@ mod tests {
         // different DNS version to test that that works.
         let new_dns_version = blueprint1.internal_dns_version.next();
         let mut builder = BlueprintBuilder::new_based_on(
+            &logctx.log,
             &blueprint1,
             new_dns_version,
             &policy,
@@ -1500,6 +1501,7 @@ mod tests {
         )
         .unwrap();
         let blueprint2 = BlueprintBuilder::new_based_on(
+            &logctx.log,
             &blueprint1,
             Generation::new(),
             &EMPTY_POLICY,
@@ -1508,6 +1510,7 @@ mod tests {
         .expect("failed to create builder")
         .build();
         let blueprint3 = BlueprintBuilder::new_based_on(
+            &logctx.log,
             &blueprint1,
             Generation::new(),
             &EMPTY_POLICY,
@@ -1604,6 +1607,7 @@ mod tests {
         // Create a child of blueprint3, and ensure when we set it as the target
         // with enabled=false, that status is serialized.
         let blueprint4 = BlueprintBuilder::new_based_on(
+            &logctx.log,
             &blueprint3,
             Generation::new(),
             &EMPTY_POLICY,
diff --git a/nexus/reconfigurator/planning/src/blueprint_builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder.rs
index d58d798770..ebb64c36f9 100644
--- a/nexus/reconfigurator/planning/src/blueprint_builder.rs
+++ b/nexus/reconfigurator/planning/src/blueprint_builder.rs
@@ -35,6 +35,8 @@ use omicron_common::api::external::Generation;
 use omicron_common::api::external::IpNet;
 use omicron_common::api::external::MacAddr;
 use omicron_common::api::external::Vni;
+use slog::o;
+use slog::Logger;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::collections::HashSet;
@@ -98,6 +100,9 @@ pub enum EnsureMultiple {
 ///    However, the new blueprint can only be made the system's target if its
 ///    parent is the current target.
 pub struct BlueprintBuilder<'a> {
+    #[allow(dead_code)]
+    log: Logger,
+
     /// previous blueprint, on which this one will be based
     parent_blueprint: &'a Blueprint,
     internal_dns_version: Generation,
@@ -186,11 +191,17 @@ impl<'a> BlueprintBuilder<'a> {
     /// Construct a new `BlueprintBuilder` based on a previous blueprint,
     /// starting with no changes from that state
     pub fn new_based_on(
+        log: &Logger,
         parent_blueprint: &'a Blueprint,
         internal_dns_version: Generation,
         policy: &'a Policy,
         creator: &str,
     ) -> anyhow::Result<BlueprintBuilder<'a>> {
+        let log = log.new(o!(
+            "component" => "BlueprintBuilder",
+            "parent_id" => parent_blueprint.id.to_string(),
+        ));
+
         // Scan through the parent blueprint and build several sets of "used
         // resources". When adding new control plane zones to a sled, we may
         // need to allocate new resources to that zone. However, allocation at
@@ -287,6 +298,7 @@ impl<'a> BlueprintBuilder<'a> {
         );
 
         Ok(BlueprintBuilder {
+            log,
             parent_blueprint,
             internal_dns_version,
             policy,
@@ -731,6 +743,7 @@ pub mod test {
     use omicron_common::address::Ipv6Subnet;
     use omicron_common::address::SLED_PREFIX;
     use omicron_common::api::external::ByteCount;
+    use omicron_test_utils::dev::test_setup_log;
     use sled_agent_client::types::{
         Baseboard, Inventory, OmicronZoneConfig, OmicronZoneDataset,
         OmicronZoneType, OmicronZonesConfig, SledRole,
@@ -945,6 +958,7 @@ pub mod test {
     fn test_initial() {
         // Test creating a blueprint from a collection and verifying that it
         // describes no changes.
+        let logctx = test_setup_log("blueprint_builder_test_initial");
         let (collection, policy) = example(DEFAULT_N_SLEDS);
         let blueprint_initial =
             BlueprintBuilder::build_initial_from_collection(
@@ -971,6 +985,7 @@ pub mod test {
 
         // Test a no-op blueprint.
         let builder = BlueprintBuilder::new_based_on(
+            &logctx.log,
             &blueprint_initial,
             Generation::new(),
             &policy,
@@ -987,10 +1002,13 @@ pub mod test {
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
         assert_eq!(diff.sleds_changed().count(), 0);
+
+        logctx.cleanup_successful();
     }
 
     #[test]
     fn test_basic() {
+        let logctx = test_setup_log("blueprint_builder_test_basic");
         let (collection, mut policy) = example(DEFAULT_N_SLEDS);
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
             &collection,
@@ -1002,6 +1020,7 @@ pub mod test {
         verify_blueprint(&blueprint1);
 
         let mut builder = BlueprintBuilder::new_based_on(
+            &logctx.log,
             &blueprint1,
             Generation::new(),
             &policy,
@@ -1036,6 +1055,7 @@ pub mod test {
         let new_sled_id = Uuid::new_v4();
         let _ = policy_add_sled(&mut policy, new_sled_id);
         let mut builder = BlueprintBuilder::new_based_on(
+            &logctx.log,
             &blueprint2,
             Generation::new(),
             &policy,
@@ -1106,10 +1126,15 @@ pub mod test {
             })
             .collect::<BTreeSet<_>>();
         assert_eq!(crucible_pool_names, new_sled_resources.zpools);
+
+        logctx.cleanup_successful();
     }
 
     #[test]
     fn test_add_nexus_with_no_existing_nexus_zones() {
+        let logctx = test_setup_log(
+            "blueprint_builder_test_add_nexus_with_no_existing_nexus_zones",
+        );
         let (mut collection, policy) = example(DEFAULT_N_SLEDS);
 
         // We don't care about the internal DNS version here.
@@ -1134,6 +1159,7 @@ pub mod test {
         .expect("failed to create initial blueprint");
 
         let mut builder = BlueprintBuilder::new_based_on(
+            &logctx.log,
             &parent,
             internal_dns_version,
             &policy,
@@ -1157,10 +1183,14 @@ pub mod test {
             matches!(err, Error::NoNexusZonesInParentBlueprint),
             "unexpected error {err}"
         );
+
+        logctx.cleanup_successful();
     }
 
     #[test]
     fn test_add_nexus_error_cases() {
+        let logctx =
+            test_setup_log("blueprint_builder_test_add_nexus_error_cases");
         let (mut collection, policy) = example(DEFAULT_N_SLEDS);
 
         // We don't care about the internal DNS version here.
@@ -1196,6 +1226,7 @@ pub mod test {
             // Attempting to add Nexus to the sled we removed it from (with no
             // other changes to the environment) should succeed.
             let mut builder = BlueprintBuilder::new_based_on(
+                &logctx.log,
                 &parent,
                 internal_dns_version,
                 &policy,
@@ -1214,6 +1245,7 @@ pub mod test {
             // from (with no other changes to the environment) should also
             // succeed.
             let mut builder = BlueprintBuilder::new_based_on(
+                &logctx.log,
                 &parent,
                 internal_dns_version,
                 &policy,
@@ -1246,6 +1278,7 @@ pub mod test {
             policy.service_ip_pool_ranges = used_ip_ranges;
 
             let mut builder = BlueprintBuilder::new_based_on(
+                &logctx.log,
                 &parent,
                 internal_dns_version,
                 &policy,
@@ -1269,10 +1302,16 @@ pub mod test {
         // `NEXUS_OPTE_*_SUBNET`. We could hack around that by creating the
         // `BlueprintBuilder` and mucking with its internals, but that doesn't
         // seem like a particularly useful test either.
+
+        logctx.cleanup_successful();
     }
 
     #[test]
     fn test_invalid_parent_blueprint_two_zones_with_same_external_ip() {
+        let logctx = test_setup_log(
+            "blueprint_builder_test_invalid_parent_blueprint_\
+             two_zones_with_same_external_ip",
+        );
         let (mut collection, policy) = example(DEFAULT_N_SLEDS);
 
         // We should fail if the parent blueprint claims to contain two
@@ -1309,6 +1348,7 @@ pub mod test {
         .unwrap();
 
         match BlueprintBuilder::new_based_on(
+            &logctx.log,
             &parent,
             Generation::new(),
             &policy,
@@ -1320,10 +1360,16 @@ pub mod test {
                 "unexpected error: {err:#}"
             ),
         };
+
+        logctx.cleanup_successful();
     }
 
     #[test]
     fn test_invalid_parent_blueprint_two_nexus_zones_with_same_nic_ip() {
+        let logctx = test_setup_log(
+            "blueprint_builder_test_invalid_parent_blueprint_\
+             two_nexus_zones_with_same_nic_ip",
+        );
         let (mut collection, policy) = example(DEFAULT_N_SLEDS);
 
         // We should fail if the parent blueprint claims to contain two
@@ -1358,6 +1404,7 @@ pub mod test {
         .unwrap();
 
         match BlueprintBuilder::new_based_on(
+            &logctx.log,
             &parent,
             Generation::new(),
             &policy,
@@ -1369,10 +1416,16 @@ pub mod test {
                 "unexpected error: {err:#}"
             ),
         };
+
+        logctx.cleanup_successful();
     }
 
     #[test]
     fn test_invalid_parent_blueprint_two_zones_with_same_vnic_mac() {
+        let logctx = test_setup_log(
+            "blueprint_builder_test_invalid_parent_blueprint_\
+             two_zones_with_same_vnic_mac",
+        );
         let (mut collection, policy) = example(DEFAULT_N_SLEDS);
 
         // We should fail if the parent blueprint claims to contain two
@@ -1407,6 +1460,7 @@ pub mod test {
         .unwrap();
 
         match BlueprintBuilder::new_based_on(
+            &logctx.log,
             &parent,
             Generation::new(),
             &policy,
@@ -1418,5 +1472,7 @@ pub mod test {
                 "unexpected error: {err:#}"
             ),
         };
+
+        logctx.cleanup_successful();
     }
 }
diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs
index 0773fec2bf..9f4b653507 100644
--- a/nexus/reconfigurator/planning/src/planner.rs
+++ b/nexus/reconfigurator/planning/src/planner.rs
@@ -48,6 +48,7 @@ impl<'a> Planner<'a> {
         inventory: &'a Collection,
     ) -> anyhow::Result<Planner<'a>> {
         let blueprint = BlueprintBuilder::new_based_on(
+            &log,
             parent_blueprint,
             internal_dns_version,
             policy,

From 9791b34127ccbcb16a170d22706cccd972532d62 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Thu, 7 Mar 2024 00:15:05 -0800
Subject: [PATCH 079/157] chore(deps): update rust crate rustls-pemfile to
 2.1.1 (#5185)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock | 10 +++++-----
 Cargo.toml |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 18c783037f..461aef7c90 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1985,7 +1985,7 @@ dependencies = [
  "percent-encoding",
  "proc-macro2",
  "rustls 0.22.2",
- "rustls-pemfile 2.1.0",
+ "rustls-pemfile 2.1.1",
  "schemars",
  "serde",
  "serde_json",
@@ -5099,7 +5099,7 @@ dependencies = [
  "reqwest",
  "ring 0.17.8",
  "rustls 0.22.2",
- "rustls-pemfile 2.1.0",
+ "rustls-pemfile 2.1.1",
  "samael",
  "schemars",
  "semver 1.0.22",
@@ -7449,7 +7449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792"
 dependencies = [
  "openssl-probe",
- "rustls-pemfile 2.1.0",
+ "rustls-pemfile 2.1.1",
  "rustls-pki-types",
  "schannel",
  "security-framework",
@@ -7466,9 +7466,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pemfile"
-version = "2.1.0"
+version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c333bb734fcdedcea57de1602543590f545f127dc8b533324318fd492c5c70b"
+checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab"
 dependencies = [
  "base64",
  "rustls-pki-types",
diff --git a/Cargo.toml b/Cargo.toml
index 474739a932..9969ee68fb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -336,7 +336,7 @@ rpassword = "7.3.1"
 rstest = "0.18.2"
 rustfmt-wrapper = "0.2"
 rustls = "0.22.2"
-rustls-pemfile = "2.1.0"
+rustls-pemfile = "2.1.1"
 rustyline = "13.0.0"
 samael = { version = "0.0.14", features = ["xmlsec"] }
 schemars = "0.8.16"

From 57ad1045a69fc260573a9418423a8dff6bfb729e Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Thu, 7 Mar 2024 00:18:17 -0800
Subject: [PATCH 080/157] chore(deps): update rust crate http to 0.2.12 (#5200)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock | 58 +++++++++++++++++++++++++++---------------------------
 Cargo.toml |  2 +-
 2 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 461aef7c90..00a8fb0074 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1856,7 +1856,7 @@ dependencies = [
  "dns-service-client",
  "dropshot",
  "expectorate",
- "http 0.2.11",
+ "http 0.2.12",
  "omicron-test-utils",
  "omicron-workspace-hack",
  "openapi-lint",
@@ -1889,7 +1889,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "chrono",
- "http 0.2.11",
+ "http 0.2.12",
  "omicron-workspace-hack",
  "progenitor",
  "reqwest",
@@ -1941,7 +1941,7 @@ dependencies = [
  "anyhow",
  "chrono",
  "futures",
- "http 0.2.11",
+ "http 0.2.12",
  "ipnetwork",
  "omicron-workspace-hack",
  "omicron-zone-package",
@@ -1976,7 +1976,7 @@ dependencies = [
  "form_urlencoded",
  "futures",
  "hostname",
- "http 0.2.11",
+ "http 0.2.12",
  "hyper 0.14.27",
  "indexmap 2.2.5",
  "multer",
@@ -2158,7 +2158,7 @@ dependencies = [
  "async-trait",
  "base64",
  "chrono",
- "http 0.2.11",
+ "http 0.2.12",
  "hyper 0.14.27",
  "omicron-sled-agent",
  "omicron-test-utils",
@@ -2812,7 +2812,7 @@ dependencies = [
  "futures-core",
  "futures-sink",
  "futures-util",
- "http 0.2.11",
+ "http 0.2.12",
  "indexmap 2.2.5",
  "slab",
  "tokio",
@@ -2882,7 +2882,7 @@ dependencies = [
  "base64",
  "bytes",
  "headers-core",
- "http 0.2.11",
+ "http 0.2.12",
  "httpdate",
  "mime",
  "sha1",
@@ -2894,7 +2894,7 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429"
 dependencies = [
- "http 0.2.11",
+ "http 0.2.12",
 ]
 
 [[package]]
@@ -3011,9 +3011,9 @@ dependencies = [
 
 [[package]]
 name = "http"
-version = "0.2.11"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb"
+checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
 dependencies = [
  "bytes",
  "fnv",
@@ -3038,7 +3038,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
 dependencies = [
  "bytes",
- "http 0.2.11",
+ "http 0.2.12",
  "pin-project-lite",
 ]
 
@@ -3081,7 +3081,7 @@ dependencies = [
  "crossbeam-channel",
  "form_urlencoded",
  "futures",
- "http 0.2.11",
+ "http 0.2.12",
  "hyper 0.14.27",
  "log",
  "once_cell",
@@ -3168,7 +3168,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "h2",
- "http 0.2.11",
+ "http 0.2.12",
  "http-body 0.4.5",
  "httparse",
  "httpdate",
@@ -3206,7 +3206,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
 dependencies = [
  "futures-util",
- "http 0.2.11",
+ "http 0.2.12",
  "hyper 0.14.27",
  "rustls 0.21.9",
  "tokio",
@@ -3239,7 +3239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "318ca89e4827e7fe4ddd2824f52337239796ae8ecc761a663324407dc3d8d7e7"
 dependencies = [
  "futures-util",
- "http 0.2.11",
+ "http 0.2.12",
  "http-range",
  "httpdate",
  "hyper 0.14.27",
@@ -3469,7 +3469,7 @@ dependencies = [
  "futures",
  "hex",
  "hex-literal",
- "http 0.2.11",
+ "http 0.2.12",
  "illumos-utils",
  "installinator-artifact-client",
  "installinator-common",
@@ -4364,7 +4364,7 @@ dependencies = [
  "futures",
  "gateway-client",
  "headers",
- "http 0.2.11",
+ "http 0.2.12",
  "hyper 0.14.27",
  "hyper-rustls 0.26.0",
  "illumos-utils",
@@ -4555,7 +4555,7 @@ dependencies = [
  "gateway-messages",
  "gateway-test-utils",
  "headers",
- "http 0.2.11",
+ "http 0.2.12",
  "hyper 0.14.27",
  "internal-dns",
  "nexus-config",
@@ -4907,7 +4907,7 @@ dependencies = [
  "expectorate",
  "futures",
  "hex",
- "http 0.2.11",
+ "http 0.2.12",
  "ipnetwork",
  "libc",
  "macaddr",
@@ -4984,7 +4984,7 @@ dependencies = [
  "gateway-sp-comms",
  "gateway-test-utils",
  "hex",
- "http 0.2.11",
+ "http 0.2.12",
  "hyper 0.14.27",
  "illumos-utils",
  "ipcc",
@@ -5044,7 +5044,7 @@ dependencies = [
  "gateway-test-utils",
  "headers",
  "hex",
- "http 0.2.11",
+ "http 0.2.12",
  "httptest",
  "hubtools",
  "hyper 0.14.27",
@@ -5273,7 +5273,7 @@ dependencies = [
  "glob",
  "guppy",
  "hex",
- "http 0.2.11",
+ "http 0.2.12",
  "hyper 0.14.27",
  "hyper-staticfile",
  "illumos-utils",
@@ -5347,7 +5347,7 @@ dependencies = [
  "filetime",
  "headers",
  "hex",
- "http 0.2.11",
+ "http 0.2.12",
  "libc",
  "nexus-config",
  "omicron-common",
@@ -5698,7 +5698,7 @@ dependencies = [
  "base64",
  "chrono",
  "futures",
- "http 0.2.11",
+ "http 0.2.12",
  "hyper 0.14.27",
  "omicron-workspace-hack",
  "progenitor",
@@ -5855,7 +5855,7 @@ dependencies = [
  "chrono",
  "dropshot",
  "futures",
- "http 0.2.11",
+ "http 0.2.12",
  "kstat-rs",
  "omicron-workspace-hack",
  "oximeter",
@@ -6599,7 +6599,7 @@ source = "git+https://github.com/oxidecomputer/progenitor?branch=main#08bbafc251
 dependencies = [
  "getopts",
  "heck 0.4.1",
- "http 0.2.11",
+ "http 0.2.12",
  "indexmap 2.2.5",
  "openapiv3",
  "proc-macro2",
@@ -7094,7 +7094,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "h2",
- "http 0.2.11",
+ "http 0.2.12",
  "http-body 0.4.5",
  "hyper 0.14.27",
  "hyper-rustls 0.24.2",
@@ -9675,7 +9675,7 @@ dependencies = [
  "byteorder",
  "bytes",
  "data-encoding",
- "http 0.2.11",
+ "http 0.2.12",
  "httparse",
  "log",
  "rand 0.8.5",
@@ -10401,7 +10401,7 @@ dependencies = [
  "gateway-messages",
  "gateway-test-utils",
  "hex",
- "http 0.2.11",
+ "http 0.2.12",
  "hubtools",
  "hyper 0.14.27",
  "illumos-utils",
diff --git a/Cargo.toml b/Cargo.toml
index 9969ee68fb..534c33e713 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -225,7 +225,7 @@ hex = "0.4.3"
 hex-literal = "0.4.1"
 highway = "1.1.0"
 hkdf = "0.12.4"
-http = "0.2.11"
+http = "0.2.12"
 httptest = "0.15.5"
 hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "main" }
 humantime = "2.1.0"

From 5620f9c900ebfd18dde3a9150c682f4ea3063132 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Karen=20C=C3=A1rcamo?= <karencfv@users.noreply.github.com>
Date: Thu, 7 Mar 2024 22:19:21 +1300
Subject: [PATCH 081/157] [sled-agent] Nexus self assembling zone (#5215)

Related: https://github.com/oxidecomputer/omicron/issues/1898
Closes: https://github.com/oxidecomputer/omicron/issues/2882
---
 .github/buildomat/jobs/deploy.sh              |   2 +-
 .github/buildomat/jobs/package.sh             |   6 +-
 .github/buildomat/jobs/tuf-repo.sh            |   2 +-
 illumos-utils/src/running_zone.rs             |  10 +-
 package-manifest.toml                         |  15 +-
 sled-agent/src/services.rs                    | 261 ++++++++++--------
 smf/nexus/manifest.xml                        |  17 +-
 tufaceous/README.adoc                         |   4 +-
 .../tests/integration-tests/command_tests.rs  |   6 +-
 9 files changed, 196 insertions(+), 127 deletions(-)

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index 5e43ff7f7c..c16d523b03 100755
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -161,7 +161,7 @@ cd /opt/oxide/work
 
 ptime -m tar xvzf /input/package/work/package.tar.gz
 cp /input/package/work/zones/* out/
-mv out/omicron-nexus-single-sled.tar.gz out/omicron-nexus.tar.gz
+mv out/nexus-single-sled.tar.gz out/nexus.tar.gz
 mkdir tests
 for p in /input/ci-tools/work/end-to-end-tests/*.gz; do
 	ptime -m gunzip < "$p" > "tests/$(basename "${p%.gz}")"
diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
index 13f374779c..dc89bc787b 100755
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -84,7 +84,7 @@ stamp_packages() {
 
 # Keep the single-sled Nexus zone around for the deploy job. (The global zone
 # build below overwrites the file.)
-mv out/omicron-nexus.tar.gz out/omicron-nexus-single-sled.tar.gz
+mv out/nexus.tar.gz out/nexus-single-sled.tar.gz
 
 # Build necessary for the global zone
 ptime -m cargo run --locked --release --bin omicron-package -- \
@@ -115,8 +115,8 @@ zones=(
   out/crucible-zone.tar.gz
   out/external-dns.tar.gz
   out/internal-dns.tar.gz
-  out/omicron-nexus.tar.gz
-  out/omicron-nexus-single-sled.tar.gz
+  out/nexus.tar.gz
+  out/nexus-single-sled.tar.gz
   out/oximeter.tar.gz
   out/propolis-server.tar.gz
   out/switch-*.tar.gz
diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh
index 14c2293f5b..aca43422d9 100644
--- a/.github/buildomat/jobs/tuf-repo.sh
+++ b/.github/buildomat/jobs/tuf-repo.sh
@@ -93,7 +93,7 @@ target/release/omicron-package -t default target create -i standard -m gimlet -s
 ln -s /input/package/work/zones/* out/
 rm out/switch-softnpu.tar.gz  # not used when target switch=asic
 rm out/omicron-gateway-softnpu.tar.gz  # not used when target switch=asic
-rm out/omicron-nexus-single-sled.tar.gz # only used for deploy tests
+rm out/nexus-single-sled.tar.gz # only used for deploy tests
 for zone in out/*.tar.gz; do
     target/release/omicron-package stamp "$(basename "${zone%.tar.gz}")" "$VERSION"
 done
diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs
index d86a27e3f7..02302347cd 100644
--- a/illumos-utils/src/running_zone.rs
+++ b/illumos-utils/src/running_zone.rs
@@ -404,7 +404,7 @@ impl RunningZone {
 
     /// Returns the filesystem path to the zone's root in the GZ.
     pub fn root(&self) -> Utf8PathBuf {
-        self.inner.zonepath.join(Self::ROOT_FS_PATH)
+        self.inner.root()
     }
 
     pub fn control_interface(&self) -> AddrObject {
@@ -1094,6 +1094,9 @@ pub struct InstalledZone {
 }
 
 impl InstalledZone {
+    /// The path to the zone's root filesystem (i.e., `/`), within zonepath.
+    pub const ROOT_FS_PATH: &'static str = "root";
+
     /// Returns the name of a zone, based on the base zone name plus any unique
     /// identifying info.
     ///
@@ -1135,6 +1138,11 @@ impl InstalledZone {
     pub fn opte_ports(&self) -> impl Iterator<Item = &Port> {
         self.opte_ports.iter().map(|(port, _)| port)
     }
+
+    /// Returns the filesystem path to the zone's root in the GZ.
+    pub fn root(&self) -> Utf8PathBuf {
+        self.zonepath.join(Self::ROOT_FS_PATH)
+    }
 }
 
 #[derive(Clone)]
diff --git a/package-manifest.toml b/package-manifest.toml
index 5575efd285..76f52ebc35 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -97,9 +97,21 @@ source.paths = [
 output.type = "zone"
 output.intermediate_only = true
 
-[package.omicron-nexus]
+[package.nexus]
 service_name = "nexus"
 only_for_targets.image = "standard"
+source.type = "composite"
+source.packages = [
+  "omicron-nexus.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz",
+  "opte-interface-setup.tar.gz"
+]
+output.type = "zone"
+
+[package.omicron-nexus]
+service_name = "omicron-nexus"
+only_for_targets.image = "standard"
 source.type = "local"
 source.rust.binary_names = ["nexus", "schema-updater"]
 source.rust.release = true
@@ -115,6 +127,7 @@ setup_hint = """
 - Run `./tools/ci_download_console` to download the web console assets
 - Run `pkg install library/postgresql-13` to download Postgres libraries
 """
+output.intermediate_only = true
 
 [package.oximeter]
 service_name = "oximeter"
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
index 41acf6c079..fed0114bc6 100644
--- a/sled-agent/src/services.rs
+++ b/sled-agent/src/services.rs
@@ -2038,6 +2038,153 @@ impl ServiceManager {
 
                 return Ok(RunningZone::boot(installed_zone).await?);
             }
+            ZoneArgs::Omicron(OmicronZoneConfigLocal {
+                zone:
+                    OmicronZoneConfig {
+                        zone_type:
+                            OmicronZoneType::Nexus {
+                                internal_address,
+                                external_tls,
+                                external_dns_servers,
+                                ..
+                            },
+                        underlay_address,
+                        id,
+                    },
+                ..
+            }) => {
+                let Some(info) = self.inner.sled_info.get() else {
+                    return Err(Error::SledAgentNotReady);
+                };
+
+                let static_addr = underlay_address.to_string();
+
+                let nw_setup_service = Self::zone_network_setup_install(
+                    &info,
+                    &installed_zone,
+                    &static_addr.clone(),
+                )?;
+
+                // While Nexus will be reachable via `external_ip`, it
+                // communicates atop an OPTE port which operates on a
+                // VPC private IP. OPTE will map the private IP to the
+                // external IP automatically.
+                let opte_interface_setup =
+                    Self::opte_interface_set_up_install(&installed_zone)?;
+
+                let port_idx = 0;
+                let port = installed_zone
+                    .opte_ports()
+                    .nth(port_idx)
+                    .ok_or_else(|| {
+                        Error::ZoneEnsureAddress(
+                            EnsureAddressError::MissingOptePort {
+                                zone: String::from(installed_zone.name()),
+                                port_idx,
+                            },
+                        )
+                    })?;
+                let opte_ip = port.ip();
+
+                // Nexus takes a separate config file for parameters
+                // which cannot be known at packaging time.
+                let nexus_port = if *external_tls { 443 } else { 80 };
+                let deployment_config = DeploymentConfig {
+                    id: *id,
+                    rack_id: info.rack_id,
+                    techport_external_server_port: NEXUS_TECHPORT_EXTERNAL_PORT,
+
+                    dropshot_external: ConfigDropshotWithTls {
+                        tls: *external_tls,
+                        dropshot: dropshot::ConfigDropshot {
+                            bind_address: SocketAddr::new(*opte_ip, nexus_port),
+                            // This has to be large enough to support:
+                            // - bulk writes to disks
+                            request_body_max_bytes: 8192 * 1024,
+                            default_handler_task_mode:
+                                HandlerTaskMode::Detached,
+                        },
+                    },
+                    dropshot_internal: dropshot::ConfigDropshot {
+                        bind_address: (*internal_address).into(),
+                        // This has to be large enough to support, among
+                        // other things, the initial list of TLS
+                        // certificates provided by the customer during
+                        // rack setup.
+                        request_body_max_bytes: 10 * 1024 * 1024,
+                        default_handler_task_mode: HandlerTaskMode::Detached,
+                    },
+                    internal_dns: nexus_config::InternalDns::FromSubnet {
+                        subnet: Ipv6Subnet::<RACK_PREFIX>::new(
+                            info.underlay_address,
+                        ),
+                    },
+                    database: nexus_config::Database::FromDns,
+                    external_dns_servers: external_dns_servers.clone(),
+                };
+
+                // Copy the partial config file to the expected
+                // location.
+                let config_dir = Utf8PathBuf::from(format!(
+                    "{}/var/svc/manifest/site/nexus",
+                    installed_zone.root()
+                ));
+                // The filename of a half-completed config, in need of
+                // parameters supplied at runtime.
+                const PARTIAL_LEDGER_FILENAME: &str = "config-partial.toml";
+                // The filename of a completed config, merging the
+                // partial config with additional appended parameters
+                // known at runtime.
+                const COMPLETE_LEDGER_FILENAME: &str = "config.toml";
+                let partial_config_path =
+                    config_dir.join(PARTIAL_LEDGER_FILENAME);
+                let config_path = config_dir.join(COMPLETE_LEDGER_FILENAME);
+                tokio::fs::copy(partial_config_path, &config_path)
+                    .await
+                    .map_err(|err| Error::io_path(&config_path, err))?;
+
+                // Serialize the configuration and append it into the
+                // file.
+                let serialized_cfg = toml::Value::try_from(&deployment_config)
+                    .expect("Cannot serialize config");
+                let mut map = toml::map::Map::new();
+                map.insert("deployment".to_string(), serialized_cfg);
+                let config_str = toml::to_string(&map).map_err(|err| {
+                    Error::TomlSerialize { path: config_path.clone(), err }
+                })?;
+                let mut file = tokio::fs::OpenOptions::new()
+                    .append(true)
+                    .open(&config_path)
+                    .await
+                    .map_err(|err| Error::io_path(&config_path, err))?;
+                file.write_all(b"\n\n")
+                    .await
+                    .map_err(|err| Error::io_path(&config_path, err))?;
+                file.write_all(config_str.as_bytes())
+                    .await
+                    .map_err(|err| Error::io_path(&config_path, err))?;
+
+                let nexus_config = PropertyGroupBuilder::new("config");
+                let nexus_service = ServiceBuilder::new("oxide/nexus")
+                    .add_instance(
+                        ServiceInstanceBuilder::new("default")
+                            .add_property_group(nexus_config),
+                    );
+
+                let profile = ProfileBuilder::new("omicron")
+                    .add_service(nw_setup_service)
+                    .add_service(opte_interface_setup)
+                    .add_service(disabled_ssh_service)
+                    .add_service(nexus_service)
+                    .add_service(disabled_dns_client_service);
+                profile
+                    .add_to_zone(&self.inner.log, &installed_zone)
+                    .await
+                    .map_err(|err| {
+                        Error::io("Failed to setup Nexus profile", err)
+                    })?;
+                return Ok(RunningZone::boot(installed_zone).await?);
+            }
             _ => {}
         }
 
@@ -2165,119 +2312,6 @@ impl ServiceManager {
                 smfh.import_manifest()?;
 
                 match &zone_config.zone.zone_type {
-                    OmicronZoneType::Nexus {
-                        internal_address,
-                        external_tls,
-                        external_dns_servers,
-                        ..
-                    } => {
-                        info!(self.inner.log, "Setting up Nexus service");
-
-                        let sled_info = self
-                            .inner
-                            .sled_info
-                            .get()
-                            .ok_or(Error::SledAgentNotReady)?;
-
-                        // While Nexus will be reachable via `external_ip`, it
-                        // communicates atop an OPTE port which operates on a
-                        // VPC private IP. OPTE will map the private IP to the
-                        // external IP automatically.
-                        let port_ip = running_zone
-                            .ensure_address_for_port("public", 0)
-                            .await?
-                            .ip();
-
-                        // Nexus takes a separate config file for parameters
-                        // which cannot be known at packaging time.
-                        let nexus_port = if *external_tls { 443 } else { 80 };
-                        let deployment_config = DeploymentConfig {
-                            id: zone_config.zone.id,
-                            rack_id: sled_info.rack_id,
-                            techport_external_server_port:
-                                NEXUS_TECHPORT_EXTERNAL_PORT,
-
-                            dropshot_external: ConfigDropshotWithTls {
-                                tls: *external_tls,
-                                dropshot: dropshot::ConfigDropshot {
-                                    bind_address: SocketAddr::new(
-                                        port_ip, nexus_port,
-                                    ),
-                                    // This has to be large enough to support:
-                                    // - bulk writes to disks
-                                    request_body_max_bytes: 8192 * 1024,
-                                    default_handler_task_mode:
-                                        HandlerTaskMode::Detached,
-                                },
-                            },
-                            dropshot_internal: dropshot::ConfigDropshot {
-                                bind_address: (*internal_address).into(),
-                                // This has to be large enough to support, among
-                                // other things, the initial list of TLS
-                                // certificates provided by the customer during
-                                // rack setup.
-                                request_body_max_bytes: 10 * 1024 * 1024,
-                                default_handler_task_mode:
-                                    HandlerTaskMode::Detached,
-                            },
-                            internal_dns:
-                                nexus_config::InternalDns::FromSubnet {
-                                    subnet: Ipv6Subnet::<RACK_PREFIX>::new(
-                                        sled_info.underlay_address,
-                                    ),
-                                },
-                            database: nexus_config::Database::FromDns,
-                            external_dns_servers: external_dns_servers.clone(),
-                        };
-
-                        // Copy the partial config file to the expected
-                        // location.
-                        let config_dir = Utf8PathBuf::from(format!(
-                            "{}/var/svc/manifest/site/nexus",
-                            running_zone.root()
-                        ));
-                        // The filename of a half-completed config, in need of
-                        // parameters supplied at runtime.
-                        const PARTIAL_LEDGER_FILENAME: &str =
-                            "config-partial.toml";
-                        // The filename of a completed config, merging the
-                        // partial config with additional appended parameters
-                        // known at runtime.
-                        const COMPLETE_LEDGER_FILENAME: &str = "config.toml";
-                        let partial_config_path =
-                            config_dir.join(PARTIAL_LEDGER_FILENAME);
-                        let config_path =
-                            config_dir.join(COMPLETE_LEDGER_FILENAME);
-                        tokio::fs::copy(partial_config_path, &config_path)
-                            .await
-                            .map_err(|err| Error::io_path(&config_path, err))?;
-
-                        // Serialize the configuration and append it into the
-                        // file.
-                        let serialized_cfg =
-                            toml::Value::try_from(&deployment_config)
-                                .expect("Cannot serialize config");
-                        let mut map = toml::map::Map::new();
-                        map.insert("deployment".to_string(), serialized_cfg);
-                        let config_str =
-                            toml::to_string(&map).map_err(|err| {
-                                Error::TomlSerialize {
-                                    path: config_path.clone(),
-                                    err,
-                                }
-                            })?;
-                        let mut file = tokio::fs::OpenOptions::new()
-                            .append(true)
-                            .open(&config_path)
-                            .await
-                            .map_err(|err| Error::io_path(&config_path, err))?;
-                        file.write_all(b"\n\n")
-                            .await
-                            .map_err(|err| Error::io_path(&config_path, err))?;
-                        file.write_all(config_str.as_bytes())
-                            .await
-                            .map_err(|err| Error::io_path(&config_path, err))?;
-                    }
                     OmicronZoneType::InternalDns {
                         http_address,
                         dns_address,
@@ -2360,6 +2394,7 @@ impl ServiceManager {
                     | OmicronZoneType::CruciblePantry { .. }
                     | OmicronZoneType::ExternalDns { .. }
                     | OmicronZoneType::InternalNtp { .. }
+                    | OmicronZoneType::Nexus { .. }
                     | OmicronZoneType::Oximeter { .. } => {
                         panic!(
                             "{} is a service which exists as part of a \
diff --git a/smf/nexus/manifest.xml b/smf/nexus/manifest.xml
index 5a72df8a22..7f1a3e76ff 100644
--- a/smf/nexus/manifest.xml
+++ b/smf/nexus/manifest.xml
@@ -4,28 +4,41 @@
 <service_bundle type='manifest' name='nexus'>
 
 <service name='oxide/nexus' type='service' version='1'>
-  <create_default_instance enabled='false' />
+  <create_default_instance enabled='true' />
 
   <dependency name='multi_user' grouping='require_all' restart_on='none'
     type='service'>
   <service_fmri value='svc:/milestone/multi-user:default' />
   </dependency>
+
   <dependency name='network' grouping='require_all' restart_on='none'
     type='service'>
   <service_fmri value='svc:/milestone/network:default' />
+
   </dependency>
+
   <dependency name='routing' grouping='require_all' restart_on='none'
     type='service'>
   <service_fmri value='svc:/network/routing/ndp:default' />
   </dependency>
 
+  <dependency name='zone_network_setup' grouping='require_all' restart_on='none'
+    type='service'>
+  <service_fmri value='svc:/oxide/zone-network-setup:default' />
+  </dependency>
+
+  <dependency name='opte_interface_setup' grouping='require_all' restart_on='none'
+    type='service'>
+  <service_fmri value='svc:/oxide/opte-interface-setup:default' />
+  </dependency>
+
   <method_context>
     <method_environment>
       <envvar name="LD_LIBRARY_PATH" value="/opt/ooce/pgsql-13/lib/amd64" />
     </method_environment>
   </method_context>
   <exec_method type='method' name='start'
-    exec='ctrun -l child -o noorphan,regent /opt/oxide/nexus/bin/nexus /var/svc/manifest/site/nexus/config.toml &amp;'
+    exec='ctrun -l child -o noorphan,regent /opt/oxide/omicron-nexus/bin/nexus /var/svc/manifest/site/nexus/config.toml &amp;'
     timeout_seconds='0' />
   <exec_method type='method' name='stop' exec=':kill' timeout_seconds='0' />
 
diff --git a/tufaceous/README.adoc b/tufaceous/README.adoc
index 973a37aa90..86e4cfc4e5 100644
--- a/tufaceous/README.adoc
+++ b/tufaceous/README.adoc
@@ -31,6 +31,6 @@ tuftool [-r PATH/TO/REPO] add-zone [--name NAME] ZONE_TAR_GZ VERSION
 Example:
 
 ----
-$ tuftool add-zone out/omicron-nexus.tar.gz 0.0.0
-added zone omicron-nexus, version 0.0.0
+$ tuftool add-zone out/nexus.tar.gz 0.0.0
+added zone nexus, version 0.0.0
 ----
diff --git a/tufaceous/tests/integration-tests/command_tests.rs b/tufaceous/tests/integration-tests/command_tests.rs
index 72c3a1a13a..20c6a06d66 100644
--- a/tufaceous/tests/integration-tests/command_tests.rs
+++ b/tufaceous/tests/integration-tests/command_tests.rs
@@ -25,7 +25,7 @@ async fn test_init_and_add() -> Result<()> {
     cmd.assert().success();
 
     // Create a couple of stub files on disk.
-    let nexus_path = tempdir.path().join("omicron-nexus.tar.gz");
+    let nexus_path = tempdir.path().join("nexus.tar.gz");
     fs_err::write(&nexus_path, "test")?;
     let unknown_path = tempdir.path().join("my-unknown-kind.tar.gz");
     fs_err::write(&unknown_path, "unknown test")?;
@@ -65,7 +65,7 @@ async fn test_init_and_add() -> Result<()> {
 
     let mut artifacts_iter = artifacts.artifacts.into_iter();
     let artifact = artifacts_iter.next().unwrap();
-    assert_eq!(artifact.name, "omicron-nexus", "artifact name");
+    assert_eq!(artifact.name, "nexus", "artifact name");
     assert_eq!(artifact.version, "42.0.0".parse().unwrap(), "artifact version");
     assert_eq!(
         artifact.kind,
@@ -73,7 +73,7 @@ async fn test_init_and_add() -> Result<()> {
         "artifact kind"
     );
     assert_eq!(
-        artifact.target, "gimlet_sp-omicron-nexus-42.0.0.tar.gz",
+        artifact.target, "gimlet_sp-nexus-42.0.0.tar.gz",
         "artifact target"
     );
 

From bcad9a301d74d3bf1ad13130a58776bfdc708ba5 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Thu, 7 Mar 2024 09:57:16 +0000
Subject: [PATCH 082/157] chore(deps): update taiki-e/install-action digest to
 4978b3c (#5216)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [taiki-e/install-action](https://togithub.com/taiki-e/install-action)
| action | digest | [`dabb9c1` ->
`4978b3c`](https://togithub.com/taiki-e/install-action/compare/dabb9c1...4978b3c)
|

---

### Configuration

📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone
America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone
America/Los_Angeles.

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update
again.

---

- [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check
this box

---

This PR has been generated by [Renovate
Bot](https://togithub.com/renovatebot/renovate).

<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNy4yMzAuMCIsInVwZGF0ZWRJblZlciI6IjM3LjIzMC4wIiwidGFyZ2V0QnJhbmNoIjoibWFpbiJ9-->

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index cac987b277..557cfedbd3 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@dabb9c1ee51c21c545764a0d517f069ff52e6477 # v2
+        uses: taiki-e/install-action@4978b3c8549425a60eb2d68446d0b58cef63ad0a # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From d91a7b2faf7be43609a1f9c81f895678a5cbf912 Mon Sep 17 00:00:00 2001
From: David Crespo <david-crespo@users.noreply.github.com>
Date: Thu, 7 Mar 2024 04:06:36 -0600
Subject: [PATCH 083/157] Bump web console (floating IP edit, fix button
 styles) (#5206)

https://github.com/oxidecomputer/console/compare/269c2c82...784e8aac

* [784e8aac](https://github.com/oxidecomputer/console/commit/784e8aac)
oxidecomputer/console#2035
* [da07ce01](https://github.com/oxidecomputer/console/commit/da07ce01)
oxidecomputer/console#2039
* [04b56b02](https://github.com/oxidecomputer/console/commit/04b56b02)
oxidecomputer/console#2037
* [55a72c86](https://github.com/oxidecomputer/console/commit/55a72c86)
oxidecomputer/console#2036
* [f0cf2be5](https://github.com/oxidecomputer/console/commit/f0cf2be5)
oxidecomputer/console#2033
* [3ad46af4](https://github.com/oxidecomputer/console/commit/3ad46af4)
oxidecomputer/console#2032
* [cd1bff5c](https://github.com/oxidecomputer/console/commit/cd1bff5c)
bump playwright and react query and stuff
* [08bd3fe5](https://github.com/oxidecomputer/console/commit/08bd3fe5)
oxidecomputer/console#2030
* [02701738](https://github.com/oxidecomputer/console/commit/02701738)
forgot to delete app/ui/index.ts
---
 tools/console_version | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/console_version b/tools/console_version
index 24d16b17eb..b3ccaf80b8 100644
--- a/tools/console_version
+++ b/tools/console_version
@@ -1,2 +1,2 @@
-COMMIT="269c2c82018d09d442a932474d84366a54837068"
-SHA2="7a6a04f43c6189384065675ebb7c429de2b8369ffbb7c1989a13aafe779be2d6"
+COMMIT="784e8aac273dc282b6218994d09042aa5928a198"
+SHA2="b5afd1243e40e86d3a779f089fe61ed9cb3fbbd6d59e547e522a59c1b0e4b7d8"

From e2e70722c23f5866b5516c94fa70b2fa4ccedc53 Mon Sep 17 00:00:00 2001
From: Kyle Simpson <kyle@oxide.computer>
Date: Thu, 7 Mar 2024 10:08:50 +0000
Subject: [PATCH 084/157] Chore: Bump OPTE to `b85995f` (#5205)

Includes fixes for panics on bad packets, and reduces the performance
impact of handling non-OPTE underlay packets.
---
 .github/buildomat/jobs/deploy.sh | 13 ------------
 Cargo.lock                       | 35 ++++++++++++++++++++++++++------
 Cargo.toml                       |  4 ++--
 tools/opte_version               |  2 +-
 4 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index c16d523b03..a2957d35b4 100755
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -102,19 +102,6 @@ z_swadm () {
 	pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm $@
 }
 
-# XXX remove. This is just to test against a development branch of OPTE in CI.
-set +x
-OPTE_COMMIT="73d4669ea213d0b7aca35c4babb6fd09ed51d29e"
-curl  -sSfOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/module/$OPTE_COMMIT/xde
-pfexec rem_drv xde || true
-pfexec mv xde /kernel/drv/amd64/xde
-pfexec add_drv xde || true
-curl -sSfOL https://buildomat.eng.oxide.computer/wg/0/artefact/01HM09S4M15WNXB2B2MX8R1GBT/yLalJU5vT4S4IEpwSeY4hPuspxw3JcINokZmlfNU14npHkzG/01HM09SJ2RQSFGW7MVKC9JKZ8D/01HM0A58D888AJ7YP6N1Q6T6ZD/opteadm
-chmod +x opteadm
-cp opteadm /tmp/opteadm
-pfexec mv opteadm /opt/oxide/opte/bin/opteadm
-set -x
-
 #
 # XXX work around 14537 (UFS should not allow directories to be unlinked) which
 # is probably not yet fixed in xde branch?  Once the xde branch merges from
diff --git a/Cargo.lock b/Cargo.lock
index 00a8fb0074..7471aa2dc4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1708,6 +1708,17 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "derror-macro"
+version = "0.1.0"
+source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
+dependencies = [
+ "darling 0.20.3",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.51",
+]
+
 [[package]]
 name = "diesel"
 version = "2.1.4"
@@ -3347,7 +3358,7 @@ dependencies = [
 [[package]]
 name = "illumos-sys-hdrs"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=1d29ef60a18179babfb44f0f7a3c2fe71034a2c1#1d29ef60a18179babfb44f0f7a3c2fe71034a2c1"
+source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
 
 [[package]]
 name = "illumos-utils"
@@ -3736,7 +3747,7 @@ dependencies = [
 [[package]]
 name = "kstat-macro"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=1d29ef60a18179babfb44f0f7a3c2fe71034a2c1#1d29ef60a18179babfb44f0f7a3c2fe71034a2c1"
+source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
 dependencies = [
  "quote",
  "syn 2.0.51",
@@ -5614,9 +5625,10 @@ dependencies = [
 [[package]]
 name = "opte"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=1d29ef60a18179babfb44f0f7a3c2fe71034a2c1#1d29ef60a18179babfb44f0f7a3c2fe71034a2c1"
+source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
 dependencies = [
  "cfg-if",
+ "derror-macro",
  "dyn-clone",
  "illumos-sys-hdrs",
  "kstat-macro",
@@ -5624,13 +5636,14 @@ dependencies = [
  "postcard",
  "serde",
  "smoltcp 0.11.0",
+ "tabwriter",
  "version_check",
 ]
 
 [[package]]
 name = "opte-api"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=1d29ef60a18179babfb44f0f7a3c2fe71034a2c1#1d29ef60a18179babfb44f0f7a3c2fe71034a2c1"
+source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
 dependencies = [
  "illumos-sys-hdrs",
  "ipnetwork",
@@ -5642,7 +5655,7 @@ dependencies = [
 [[package]]
 name = "opte-ioctl"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=1d29ef60a18179babfb44f0f7a3c2fe71034a2c1#1d29ef60a18179babfb44f0f7a3c2fe71034a2c1"
+source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
 dependencies = [
  "libc",
  "libnet",
@@ -5716,7 +5729,7 @@ dependencies = [
 [[package]]
 name = "oxide-vpc"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=1d29ef60a18179babfb44f0f7a3c2fe71034a2c1#1d29ef60a18179babfb44f0f7a3c2fe71034a2c1"
+source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
 dependencies = [
  "cfg-if",
  "illumos-sys-hdrs",
@@ -5724,6 +5737,7 @@ dependencies = [
  "poptrie",
  "serde",
  "smoltcp 0.11.0",
+ "tabwriter",
  "zerocopy 0.7.32",
 ]
 
@@ -8862,6 +8876,15 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "tabwriter"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a327282c4f64f6dc37e3bba4c2b6842cc3a992f204fa58d917696a89f691e5f6"
+dependencies = [
+ "unicode-width",
+]
+
 [[package]]
 name = "take_mut"
 version = "0.2.2"
diff --git a/Cargo.toml b/Cargo.toml
index 534c33e713..9cd22cbbcc 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -283,7 +283,7 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.11.0"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "1d29ef60a18179babfb44f0f7a3c2fe71034a2c1", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "b85995f92ae94cdc78b97b0a610c69e103e00423", features = [ "api", "std" ] }
 once_cell = "1.19.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "2.0.0"
@@ -291,7 +291,7 @@ openapiv3 = "2.0.0"
 openssl = "0.10"
 openssl-sys = "0.9"
 openssl-probe = "0.1.5"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "1d29ef60a18179babfb44f0f7a3c2fe71034a2c1" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "b85995f92ae94cdc78b97b0a610c69e103e00423" }
 oso = "0.27"
 owo-colors = "4.0.0"
 oximeter = { path = "oximeter/oximeter" }
diff --git a/tools/opte_version b/tools/opte_version
index 0a04873e11..23d9da8fa3 100644
--- a/tools/opte_version
+++ b/tools/opte_version
@@ -1 +1 @@
-0.28.215
+0.28.227

From dcdf9bb25112bd2740bfda31922bfe62db8b9f72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Karen=20C=C3=A1rcamo?= <karencfv@users.noreply.github.com>
Date: Fri, 8 Mar 2024 00:09:57 +1300
Subject: [PATCH 085/157] [sled-agent] Internal DNS self assembling zone
 (#5213)

Closes: https://github.com/oxidecomputer/omicron/issues/2880
Related: https://github.com/oxidecomputer/omicron/issues/1898
---
 package-manifest.toml         |   7 +-
 sled-agent/src/services.rs    | 189 +++++++++++++++++-----------------
 smf/internal-dns/manifest.xml |   7 +-
 smf/nexus/manifest.xml        |   1 -
 4 files changed, 104 insertions(+), 100 deletions(-)

diff --git a/package-manifest.toml b/package-manifest.toml
index 76f52ebc35..c6f39d2ecd 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -241,7 +241,12 @@ output.intermediate_only = true
 service_name = "internal_dns"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "dns-server.tar.gz", "internal-dns-customizations.tar.gz" ]
+source.packages = [
+  "dns-server.tar.gz",
+  "internal-dns-customizations.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz"
+]
 output.type = "zone"
 
 [package.external-dns]
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
index fed0114bc6..9273e0ef3b 100644
--- a/sled-agent/src/services.rs
+++ b/sled-agent/src/services.rs
@@ -1400,12 +1400,12 @@ impl ServiceManager {
     }
 
     fn zone_network_setup_install(
-        info: &SledAgentInfo,
+        gw_addr: &Ipv6Addr,
         zone: &InstalledZone,
         static_addr: &String,
     ) -> Result<ServiceBuilder, Error> {
         let datalink = zone.get_control_vnic_name();
-        let gateway = &info.underlay_address.to_string();
+        let gateway = &gw_addr.to_string();
 
         let mut config_builder = PropertyGroupBuilder::new("config");
         config_builder = config_builder
@@ -1575,7 +1575,7 @@ impl ServiceManager {
                 let listen_port = &CLICKHOUSE_PORT.to_string();
 
                 let nw_setup_service = Self::zone_network_setup_install(
-                    info,
+                    &info.underlay_address,
                     &installed_zone,
                     listen_addr,
                 )?;
@@ -1624,7 +1624,7 @@ impl ServiceManager {
                 let listen_port = &CLICKHOUSE_KEEPER_PORT.to_string();
 
                 let nw_setup_service = Self::zone_network_setup_install(
-                    info,
+                    &info.underlay_address,
                     &installed_zone,
                     listen_addr,
                 )?;
@@ -1680,7 +1680,7 @@ impl ServiceManager {
                 let listen_port = &address.port().to_string();
 
                 let nw_setup_service = Self::zone_network_setup_install(
-                    info,
+                    &info.underlay_address,
                     &installed_zone,
                     listen_addr,
                 )?;
@@ -1729,7 +1729,7 @@ impl ServiceManager {
                 let listen_port = &CRUCIBLE_PORT.to_string();
 
                 let nw_setup_service = Self::zone_network_setup_install(
-                    info,
+                    &info.underlay_address,
                     &installed_zone,
                     listen_addr,
                 )?;
@@ -1784,7 +1784,7 @@ impl ServiceManager {
                 let listen_port = &CRUCIBLE_PANTRY_PORT.to_string();
 
                 let nw_setup_service = Self::zone_network_setup_install(
-                    info,
+                    &info.underlay_address,
                     &installed_zone,
                     listen_addr,
                 )?;
@@ -1834,7 +1834,7 @@ impl ServiceManager {
                 let listen_addr = &address.ip().to_string();
 
                 let nw_setup_service = Self::zone_network_setup_install(
-                    info,
+                    &info.underlay_address,
                     &installed_zone,
                     listen_addr,
                 )?;
@@ -1877,7 +1877,7 @@ impl ServiceManager {
                 let static_addr = underlay_address.to_string();
 
                 let nw_setup_service = Self::zone_network_setup_install(
-                    info,
+                    &info.underlay_address,
                     &installed_zone,
                     &static_addr.clone(),
                 )?;
@@ -1966,7 +1966,7 @@ impl ServiceManager {
                 let static_addr = underlay_address.to_string();
 
                 let nw_setup_service = Self::zone_network_setup_install(
-                    info,
+                    &info.underlay_address,
                     &installed_zone,
                     &static_addr.clone(),
                 )?;
@@ -2038,6 +2038,86 @@ impl ServiceManager {
 
                 return Ok(RunningZone::boot(installed_zone).await?);
             }
+            ZoneArgs::Omicron(OmicronZoneConfigLocal {
+                zone:
+                    OmicronZoneConfig {
+                        zone_type:
+                            OmicronZoneType::InternalDns {
+                                http_address,
+                                dns_address,
+                                gz_address,
+                                gz_address_index,
+                                ..
+                            },
+                        underlay_address,
+                        ..
+                    },
+                ..
+            }) => {
+                let nw_setup_service = Self::zone_network_setup_install(
+                    gz_address,
+                    &installed_zone,
+                    &underlay_address.to_string(),
+                )?;
+
+                // Internal DNS zones require a special route through
+                // the global zone, since they are not on the same part
+                // of the underlay as most other services on this sled
+                // (the sled's subnet).
+                //
+                // We create an IP address in the dedicated portion of
+                // the underlay used for internal DNS servers, but we
+                // *also* add a number ("which DNS server is this") to
+                // ensure these addresses are given unique names. In the
+                // unlikely case that two internal DNS servers end up on
+                // the same machine (which is effectively a
+                // developer-only environment -- we wouldn't want this
+                // in prod!), they need to be given distinct names.
+                let addr_name = format!("internaldns{gz_address_index}");
+                Zones::ensure_has_global_zone_v6_address(
+                    self.inner.underlay_vnic.clone(),
+                    *gz_address,
+                    &addr_name,
+                )
+                .map_err(|err| Error::GzAddress {
+                    message: format!(
+                        "Failed to create address {} for Internal DNS zone",
+                        addr_name
+                    ),
+                    err,
+                })?;
+
+                // If this address is in a new ipv6 prefix, notify
+                // maghemite so it can advertise it to other sleds.
+                self.advertise_prefix_of_address(*gz_address).await;
+
+                let http_addr =
+                    format!("[{}]:{}", http_address.ip(), http_address.port());
+                let dns_addr =
+                    format!("[{}]:{}", dns_address.ip(), dns_address.port());
+
+                let internal_dns_config = PropertyGroupBuilder::new("config")
+                    .add_property("http_address", "astring", &http_addr)
+                    .add_property("dns_address", "astring", &dns_addr);
+                let internal_dns_service =
+                    ServiceBuilder::new("oxide/internal_dns").add_instance(
+                        ServiceInstanceBuilder::new("default")
+                            .add_property_group(internal_dns_config),
+                    );
+
+                let profile = ProfileBuilder::new("omicron")
+                    .add_service(nw_setup_service)
+                    .add_service(disabled_ssh_service)
+                    .add_service(internal_dns_service)
+                    .add_service(disabled_dns_client_service);
+                profile
+                    .add_to_zone(&self.inner.log, &installed_zone)
+                    .await
+                    .map_err(|err| {
+                        Error::io("Failed to setup Internal DNS profile", err)
+                    })?;
+                return Ok(RunningZone::boot(installed_zone).await?);
+            }
             ZoneArgs::Omicron(OmicronZoneConfigLocal {
                 zone:
                     OmicronZoneConfig {
@@ -2060,7 +2140,7 @@ impl ServiceManager {
                 let static_addr = underlay_address.to_string();
 
                 let nw_setup_service = Self::zone_network_setup_install(
-                    &info,
+                    &info.underlay_address,
                     &installed_zone,
                     &static_addr.clone(),
                 )?;
@@ -2302,90 +2382,7 @@ impl ServiceManager {
 
         match &request {
             ZoneArgs::Omicron(zone_config) => {
-                // TODO: Related to
-                // https://github.com/oxidecomputer/omicron/pull/1124 , should we
-                // avoid importing this manifest?
-                debug!(self.inner.log, "importing manifest");
-
-                let smfh =
-                    SmfHelper::new(&running_zone, &zone_config.zone.zone_type);
-                smfh.import_manifest()?;
-
                 match &zone_config.zone.zone_type {
-                    OmicronZoneType::InternalDns {
-                        http_address,
-                        dns_address,
-                        gz_address,
-                        gz_address_index,
-                        ..
-                    } => {
-                        info!(
-                            self.inner.log,
-                            "Setting up internal-dns service"
-                        );
-
-                        // Internal DNS zones require a special route through
-                        // the global zone, since they are not on the same part
-                        // of the underlay as most other services on this sled
-                        // (the sled's subnet).
-                        //
-                        // We create an IP address in the dedicated portion of
-                        // the underlay used for internal DNS servers, but we
-                        // *also* add a number ("which DNS server is this") to
-                        // ensure these addresses are given unique names. In the
-                        // unlikely case that two internal DNS servers end up on
-                        // the same machine (which is effectively a
-                        // developer-only environment -- we wouldn't want this
-                        // in prod!), they need to be given distinct names.
-                        let addr_name =
-                            format!("internaldns{gz_address_index}");
-                        Zones::ensure_has_global_zone_v6_address(
-                            self.inner.underlay_vnic.clone(),
-                            *gz_address,
-                            &addr_name,
-                        )
-                        .map_err(|err| {
-                            Error::GzAddress {
-                                message: format!(
-                            "Failed to create address {} for Internal DNS zone",
-                            addr_name
-                        ),
-                                err,
-                            }
-                        })?;
-                        // If this address is in a new ipv6 prefix, notify
-                        // maghemite so it can advertise it to other sleds.
-                        self.advertise_prefix_of_address(*gz_address).await;
-
-                        running_zone.add_default_route(*gz_address).map_err(
-                            |err| Error::ZoneCommand {
-                                intent: "Adding Route".to_string(),
-                                err,
-                            },
-                        )?;
-
-                        smfh.setprop(
-                            "config/http_address",
-                            format!(
-                                "[{}]:{}",
-                                http_address.ip(),
-                                http_address.port(),
-                            ),
-                        )?;
-                        smfh.setprop(
-                            "config/dns_address",
-                            &format!(
-                                "[{}]:{}",
-                                dns_address.ip(),
-                                dns_address.port(),
-                            ),
-                        )?;
-
-                        // Refresh the manifest with the new properties we set,
-                        // so they become "effective" properties when the
-                        // service is enabled.
-                        smfh.refresh()?;
-                    }
                     OmicronZoneType::BoundaryNtp { .. }
                     | OmicronZoneType::Clickhouse { .. }
                     | OmicronZoneType::ClickhouseKeeper { .. }
@@ -2393,6 +2390,7 @@ impl ServiceManager {
                     | OmicronZoneType::Crucible { .. }
                     | OmicronZoneType::CruciblePantry { .. }
                     | OmicronZoneType::ExternalDns { .. }
+                    | OmicronZoneType::InternalDns { .. }
                     | OmicronZoneType::InternalNtp { .. }
                     | OmicronZoneType::Nexus { .. }
                     | OmicronZoneType::Oximeter { .. } => {
@@ -2403,9 +2401,6 @@ impl ServiceManager {
                         )
                     }
                 };
-
-                debug!(self.inner.log, "enabling service");
-                smfh.enable()?;
             }
             ZoneArgs::Switch(request) => {
                 for service in &request.zone.services {
diff --git a/smf/internal-dns/manifest.xml b/smf/internal-dns/manifest.xml
index 213f861b43..07cedc4ad5 100644
--- a/smf/internal-dns/manifest.xml
+++ b/smf/internal-dns/manifest.xml
@@ -4,13 +4,18 @@
 <service_bundle type='manifest' name='internal_dns'>
 
   <service name='oxide/internal_dns' type='service' version='1'>
-  <create_default_instance enabled='false' />
+  <create_default_instance enabled='true' />
 
   <dependency name='multi_user' grouping='require_all' restart_on='none'
     type='service'>
   <service_fmri value='svc:/milestone/multi-user:default' />
   </dependency>
 
+  <dependency name='zone_network_setup' grouping='require_all' restart_on='none'
+    type='service'>
+  <service_fmri value='svc:/oxide/zone-network-setup:default' />
+  </dependency>
+
   <exec_method type='method' name='start'
       exec='ctrun -l child -o noorphan,regent /opt/oxide/dns-server/bin/dns-server --config-file /var/svc/manifest/site/internal_dns/config.toml --http-address %{config/http_address} --dns-address %{config/dns_address} &amp;'
     timeout_seconds='0' />
diff --git a/smf/nexus/manifest.xml b/smf/nexus/manifest.xml
index 7f1a3e76ff..0f43079cb9 100644
--- a/smf/nexus/manifest.xml
+++ b/smf/nexus/manifest.xml
@@ -14,7 +14,6 @@
   <dependency name='network' grouping='require_all' restart_on='none'
     type='service'>
   <service_fmri value='svc:/milestone/network:default' />
-
   </dependency>
 
   <dependency name='routing' grouping='require_all' restart_on='none'

From 6a0d1af4686e2dfbf2090c2cb177fe7fe25bd153 Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Thu, 7 Mar 2024 11:20:43 -0500
Subject: [PATCH 086/157] Expose uninitialized sleds and "add sled" via
 internal API and omdb (#5204)

This will let us drive the add sled process from a support context.
(Prior to this change adding a sled required interaction with the
external API.)

I want to test this before merging, but it's small enough that if
there's anything that needs tweaking it's unlikely to affect any review
comments. One change is that we had two `Baseboard` structs that were
identical except for a minor different in field names (`{serial,part}`
vs `{serial_number,part_number}`). I removed the internal one and
replaced it with the external one to avoid unnecessary churn in the
public API, but if I'm missing something intentional that led to having
both, please say so.

Closes #5133.
---
 dev-tools/omdb/src/bin/omdb/main.rs        |  13 +++
 dev-tools/omdb/src/bin/omdb/nexus.rs       | 111 ++++++++++++++++++-
 dev-tools/omdb/tests/usage_errors.out      |   7 +-
 nexus/src/app/sled.rs                      |   4 +-
 nexus/src/app/switch.rs                    |   4 +-
 nexus/src/internal_api/http_entrypoints.rs |  47 ++++++++
 nexus/test-utils/src/resource_helpers.rs   |   6 +-
 nexus/tests/integration_tests/rack.rs      |   7 +-
 nexus/types/src/internal_api/params.rs     |  10 +-
 openapi/nexus-internal.json                | 121 ++++++++++++++++++++-
 sled-agent/src/nexus.rs                    |   4 +-
 sled-agent/src/sim/server.rs               |   4 +-
 12 files changed, 302 insertions(+), 36 deletions(-)

diff --git a/dev-tools/omdb/src/bin/omdb/main.rs b/dev-tools/omdb/src/bin/omdb/main.rs
index 32141d2809..0fb9ba0121 100644
--- a/dev-tools/omdb/src/bin/omdb/main.rs
+++ b/dev-tools/omdb/src/bin/omdb/main.rs
@@ -84,11 +84,24 @@ struct Omdb {
     #[arg(env = "OMDB_DNS_SERVER", long)]
     dns_server: Option<SocketAddr>,
 
+    /// allow potentially-destructive subcommands
+    #[arg(short = 'w', long = "destructive")]
+    allow_destructive: bool,
+
     #[command(subcommand)]
     command: OmdbCommands,
 }
 
 impl Omdb {
+    fn check_allow_destructive(&self) -> anyhow::Result<()> {
+        anyhow::ensure!(
+            self.allow_destructive,
+            "This command is potentially destructive. \
+             Pass the `-w` / `--destructive` flag to allow it."
+        );
+        Ok(())
+    }
+
     async fn dns_lookup_all(
         &self,
         log: slog::Logger,
diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs
index aed7d86ba0..03d6fd6e80 100644
--- a/dev-tools/omdb/src/bin/omdb/nexus.rs
+++ b/dev-tools/omdb/src/bin/omdb/nexus.rs
@@ -16,6 +16,7 @@ use nexus_client::types::ActivationReason;
 use nexus_client::types::BackgroundTask;
 use nexus_client::types::CurrentStatus;
 use nexus_client::types::LastResult;
+use nexus_client::types::UninitializedSledId;
 use serde::Deserialize;
 use slog_error_chain::InlineErrorChain;
 use std::collections::BTreeMap;
@@ -38,8 +39,10 @@ pub struct NexusArgs {
 enum NexusCommands {
     /// print information about background tasks
     BackgroundTasks(BackgroundTasksArgs),
-    /// print information about blueprints
+    /// interact with blueprints
     Blueprints(BlueprintsArgs),
+    /// interact with sleds
+    Sleds(SledsArgs),
 }
 
 #[derive(Debug, Args)]
@@ -116,6 +119,28 @@ enum BlueprintTargetCommands {
     Set(BlueprintIdArgs),
 }
 
+#[derive(Debug, Args)]
+struct SledsArgs {
+    #[command(subcommand)]
+    command: SledsCommands,
+}
+
+#[derive(Debug, Subcommand)]
+enum SledsCommands {
+    /// List all uninitialized sleds
+    ListUninitialized,
+    /// Add an uninitialized sled
+    Add(SledAddArgs),
+}
+
+#[derive(Debug, Args)]
+struct SledAddArgs {
+    /// sled's serial number
+    serial: String,
+    /// sled's part number
+    part: String,
+}
+
 impl NexusArgs {
     /// Run a `omdb nexus` subcommand.
     pub(crate) async fn run_cmd(
@@ -167,7 +192,10 @@ impl NexusArgs {
             }) => cmd_nexus_blueprints_diff(&client, args).await,
             NexusCommands::Blueprints(BlueprintsArgs {
                 command: BlueprintsCommands::Delete(args),
-            }) => cmd_nexus_blueprints_delete(&client, args).await,
+            }) => {
+                omdb.check_allow_destructive()?;
+                cmd_nexus_blueprints_delete(&client, args).await
+            }
             NexusCommands::Blueprints(BlueprintsArgs {
                 command:
                     BlueprintsCommands::Target(BlueprintsTargetArgs {
@@ -179,16 +207,33 @@ impl NexusArgs {
                     BlueprintsCommands::Target(BlueprintsTargetArgs {
                         command: BlueprintTargetCommands::Set(args),
                     }),
-            }) => cmd_nexus_blueprints_target_set(&client, args).await,
+            }) => {
+                omdb.check_allow_destructive()?;
+                cmd_nexus_blueprints_target_set(&client, args).await
+            }
             NexusCommands::Blueprints(BlueprintsArgs {
                 command: BlueprintsCommands::Regenerate,
-            }) => cmd_nexus_blueprints_regenerate(&client).await,
+            }) => {
+                omdb.check_allow_destructive()?;
+                cmd_nexus_blueprints_regenerate(&client).await
+            }
             NexusCommands::Blueprints(BlueprintsArgs {
                 command: BlueprintsCommands::GenerateFromCollection(args),
             }) => {
+                omdb.check_allow_destructive()?;
                 cmd_nexus_blueprints_generate_from_collection(&client, args)
                     .await
             }
+
+            NexusCommands::Sleds(SledsArgs {
+                command: SledsCommands::ListUninitialized,
+            }) => cmd_nexus_sleds_list_uninitialized(&client).await,
+            NexusCommands::Sleds(SledsArgs {
+                command: SledsCommands::Add(args),
+            }) => {
+                omdb.check_allow_destructive()?;
+                cmd_nexus_sled_add(&client, args).await
+            }
         }
     }
 }
@@ -946,3 +991,61 @@ async fn cmd_nexus_blueprints_regenerate(
     eprintln!("generated new blueprint {}", blueprint.id);
     Ok(())
 }
+
+/// Runs `omdb nexus sleds list-uninitialized`
+async fn cmd_nexus_sleds_list_uninitialized(
+    client: &nexus_client::Client,
+) -> Result<(), anyhow::Error> {
+    let response = client
+        .sled_list_uninitialized()
+        .await
+        .context("listing uninitialized sleds")?;
+    let sleds = response.into_inner();
+    if sleds.next_page.is_some() {
+        eprintln!(
+            "warning: response includes next_page token; \
+             pagination not implemented"
+        );
+    }
+    let mut sleds = sleds.items;
+    sleds.sort_by_key(|sled| sled.cubby);
+
+    #[derive(Tabled)]
+    #[tabled(rename_all = "SCREAMING_SNAKE_CASE")]
+    struct UninitializedSledRow {
+        rack_id: Uuid,
+        cubby: u16,
+        serial: String,
+        part: String,
+        revision: i64,
+    }
+    let rows = sleds.into_iter().map(|sled| UninitializedSledRow {
+        rack_id: sled.rack_id,
+        cubby: sled.cubby,
+        serial: sled.baseboard.serial,
+        part: sled.baseboard.part,
+        revision: sled.baseboard.revision,
+    });
+    let table = tabled::Table::new(rows)
+        .with(tabled::settings::Style::empty())
+        .with(tabled::settings::Padding::new(0, 1, 0, 0))
+        .to_string();
+    println!("{}", table);
+    Ok(())
+}
+
+/// Runs `omdb nexus sleds add`
+async fn cmd_nexus_sled_add(
+    client: &nexus_client::Client,
+    args: &SledAddArgs,
+) -> Result<(), anyhow::Error> {
+    client
+        .sled_add(&UninitializedSledId {
+            part: args.part.clone(),
+            serial: args.serial.clone(),
+        })
+        .await
+        .context("adding sled")?;
+    eprintln!("added sled {} ({})", args.serial, args.part);
+    Ok(())
+}
diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out
index c10f95e23d..2f9001671d 100644
--- a/dev-tools/omdb/tests/usage_errors.out
+++ b/dev-tools/omdb/tests/usage_errors.out
@@ -19,6 +19,7 @@ Commands:
 Options:
       --log-level <LOG_LEVEL>    log level filter [env: LOG_LEVEL=] [default: warn]
       --dns-server <DNS_SERVER>  [env: OMDB_DNS_SERVER=]
+  -w, --destructive              allow potentially-destructive subcommands
   -h, --help                     Print help (see more with '--help')
 =============================================
 EXECUTING COMMAND: omdb ["--help"]
@@ -50,6 +51,9 @@ Options:
       --dns-server <DNS_SERVER>
           [env: OMDB_DNS_SERVER=]
 
+  -w, --destructive
+          allow potentially-destructive subcommands
+
   -h, --help
           Print help (see a summary with '-h')
 ---------------------------------------------
@@ -294,7 +298,8 @@ Usage: omdb nexus [OPTIONS] <COMMAND>
 
 Commands:
   background-tasks  print information about background tasks
-  blueprints        print information about blueprints
+  blueprints        interact with blueprints
+  sleds             interact with sleds
   help              Print this message or the help of the given subcommand(s)
 
 Options:
diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs
index 88955d78e9..a9341472e0 100644
--- a/nexus/src/app/sled.rs
+++ b/nexus/src/app/sled.rs
@@ -58,8 +58,8 @@ impl super::Nexus {
             id,
             info.sa_address,
             db::model::SledBaseboard {
-                serial_number: info.baseboard.serial_number,
-                part_number: info.baseboard.part_number,
+                serial_number: info.baseboard.serial,
+                part_number: info.baseboard.part,
                 revision: info.baseboard.revision,
             },
             db::model::SledSystemHardware {
diff --git a/nexus/src/app/switch.rs b/nexus/src/app/switch.rs
index 362d56553a..cea16ba7af 100644
--- a/nexus/src/app/switch.rs
+++ b/nexus/src/app/switch.rs
@@ -32,8 +32,8 @@ impl super::Nexus {
     ) -> Result<Switch, Error> {
         let switch = db::model::Switch::new(
             id,
-            request.baseboard.serial_number,
-            request.baseboard.part_number,
+            request.baseboard.serial,
+            request.baseboard.part,
             request.baseboard.revision,
             request.rack_id,
         );
diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs
index eddc834a2a..5702539a4b 100644
--- a/nexus/src/internal_api/http_entrypoints.rs
+++ b/nexus/src/internal_api/http_entrypoints.rs
@@ -29,6 +29,8 @@ use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::BlueprintMetadata;
 use nexus_types::deployment::BlueprintTarget;
 use nexus_types::deployment::BlueprintTargetSet;
+use nexus_types::external_api::params::UninitializedSledId;
+use nexus_types::external_api::shared::UninitializedSled;
 use nexus_types::internal_api::params::SwitchPutRequest;
 use nexus_types::internal_api::params::SwitchPutResponse;
 use nexus_types::internal_api::views::to_list;
@@ -88,6 +90,9 @@ pub(crate) fn internal_api() -> NexusApiDescription {
         api.register(blueprint_generate_from_collection)?;
         api.register(blueprint_regenerate)?;
 
+        api.register(sled_list_uninitialized)?;
+        api.register(sled_add)?;
+
         Ok(())
     }
 
@@ -795,3 +800,45 @@ async fn blueprint_regenerate(
     };
     apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
 }
+
+/// List uninitialized sleds
+#[endpoint {
+    method = GET,
+    path = "/sleds/uninitialized",
+}]
+async fn sled_list_uninitialized(
+    rqctx: RequestContext<Arc<ServerContext>>,
+) -> Result<HttpResponseOk<ResultsPage<UninitializedSled>>, HttpError> {
+    let apictx = rqctx.context();
+    let handler = async {
+        let nexus = &apictx.nexus;
+        let opctx = crate::context::op_context_for_internal_api(&rqctx).await;
+        let sleds = nexus.sled_list_uninitialized(&opctx).await?;
+        Ok(HttpResponseOk(ResultsPage { items: sleds, next_page: None }))
+    };
+    apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
+/// Add sled to initialized rack
+//
+// TODO: In the future this should really be a PUT request, once we resolve
+// https://github.com/oxidecomputer/omicron/issues/4494. It should also
+// explicitly be tied to a rack via a `rack_id` path param. For now we assume
+// we are only operating on single rack systems.
+#[endpoint {
+    method = POST,
+    path = "/sleds/add",
+}]
+async fn sled_add(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    sled: TypedBody<UninitializedSledId>,
+) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+    let apictx = rqctx.context();
+    let nexus = &apictx.nexus;
+    let handler = async {
+        let opctx = crate::context::op_context_for_internal_api(&rqctx).await;
+        nexus.sled_add(&opctx, sled.into_inner()).await?;
+        Ok(HttpResponseUpdatedNoContent())
+    };
+    apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs
index 25e58d093d..7331843000 100644
--- a/nexus/test-utils/src/resource_helpers.rs
+++ b/nexus/test-utils/src/resource_helpers.rs
@@ -18,6 +18,7 @@ use nexus_types::external_api::params;
 use nexus_types::external_api::params::PhysicalDiskKind;
 use nexus_types::external_api::params::UserId;
 use nexus_types::external_api::shared;
+use nexus_types::external_api::shared::Baseboard;
 use nexus_types::external_api::shared::IdentityType;
 use nexus_types::external_api::shared::IpRange;
 use nexus_types::external_api::views;
@@ -29,7 +30,6 @@ use nexus_types::external_api::views::User;
 use nexus_types::external_api::views::{Project, Silo, Vpc, VpcRouter};
 use nexus_types::identity::Resource;
 use nexus_types::internal_api::params as internal_params;
-use nexus_types::internal_api::params::Baseboard;
 use omicron_common::api::external::ByteCount;
 use omicron_common::api::external::Disk;
 use omicron_common::api::external::IdentityMetadataCreateParams;
@@ -325,8 +325,8 @@ pub async fn create_switch(
         "/switches",
         &internal_params::SwitchPutRequest {
             baseboard: Baseboard {
-                serial_number: serial.to_string(),
-                part_number: part.to_string(),
+                serial: serial.to_string(),
+                part: part.to_string(),
                 revision,
             },
             rack_id,
diff --git a/nexus/tests/integration_tests/rack.rs b/nexus/tests/integration_tests/rack.rs
index a58871ee71..2098ff660d 100644
--- a/nexus/tests/integration_tests/rack.rs
+++ b/nexus/tests/integration_tests/rack.rs
@@ -13,7 +13,6 @@ use nexus_test_utils_macros::nexus_test;
 use nexus_types::external_api::params;
 use nexus_types::external_api::shared::UninitializedSled;
 use nexus_types::external_api::views::Rack;
-use nexus_types::internal_api::params::Baseboard;
 use nexus_types::internal_api::params::SledAgentStartupInfo;
 use nexus_types::internal_api::params::SledRole;
 use omicron_common::api::external::ByteCount;
@@ -112,11 +111,7 @@ async fn test_sled_list_uninitialized(cptestctx: &ControlPlaneTestContext) {
     let sa = SledAgentStartupInfo {
         sa_address: "[fd00:1122:3344:0100::1]:8080".parse().unwrap(),
         role: SledRole::Gimlet,
-        baseboard: Baseboard {
-            serial_number: baseboard.serial,
-            part_number: baseboard.part,
-            revision: baseboard.revision,
-        },
+        baseboard,
         usable_hardware_threads: 32,
         usable_physical_ram: ByteCount::from_gibibytes_u32(100),
         reservoir_size: ByteCount::from_mebibytes_u32(100),
diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs
index 987efbf6b7..31ea86907a 100644
--- a/nexus/types/src/internal_api/params.rs
+++ b/nexus/types/src/internal_api/params.rs
@@ -6,6 +6,7 @@
 
 use crate::external_api::params::PhysicalDiskKind;
 use crate::external_api::params::UserId;
+use crate::external_api::shared::Baseboard;
 use crate::external_api::shared::IpRange;
 use omicron_common::api::external::ByteCount;
 use omicron_common::api::external::MacAddr;
@@ -35,15 +36,6 @@ pub enum SledRole {
     Scrimlet,
 }
 
-// TODO: We need a unified representation of these hardware identifiers
-/// Describes properties that should uniquely identify Oxide manufactured hardware
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
-pub struct Baseboard {
-    pub serial_number: String,
-    pub part_number: String,
-    pub revision: i64,
-}
-
 /// Sent by a sled agent on startup to Nexus to request further instruction
 #[derive(Serialize, Deserialize, Debug, JsonSchema)]
 pub struct SledAgentStartupInfo {
diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json
index 53a53fb219..65c167f177 100644
--- a/openapi/nexus-internal.json
+++ b/openapi/nexus-internal.json
@@ -918,6 +918,57 @@
         }
       }
     },
+    "/sleds/add": {
+      "post": {
+        "summary": "Add sled to initialized rack",
+        "operationId": "sled_add",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/UninitializedSledId"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "204": {
+            "description": "resource updated"
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
+    "/sleds/uninitialized": {
+      "get": {
+        "summary": "List uninitialized sleds",
+        "operationId": "sled_list_uninitialized",
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/UninitializedSledResultsPage"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/switch/{switch_id}": {
       "put": {
         "operationId": "switch_put",
@@ -1049,24 +1100,24 @@
         ]
       },
       "Baseboard": {
-        "description": "Describes properties that should uniquely identify Oxide manufactured hardware",
+        "description": "Properties that uniquely identify an Oxide hardware component",
         "type": "object",
         "properties": {
-          "part_number": {
+          "part": {
             "type": "string"
           },
           "revision": {
             "type": "integer",
             "format": "int64"
           },
-          "serial_number": {
+          "serial": {
             "type": "string"
           }
         },
         "required": [
-          "part_number",
+          "part",
           "revision",
-          "serial_number"
+          "serial"
         ]
       },
       "BgpConfig": {
@@ -6546,6 +6597,66 @@
       "SwitchPutResponse": {
         "type": "object"
       },
+      "UninitializedSled": {
+        "description": "A sled that has not been added to an initialized rack yet",
+        "type": "object",
+        "properties": {
+          "baseboard": {
+            "$ref": "#/components/schemas/Baseboard"
+          },
+          "cubby": {
+            "type": "integer",
+            "format": "uint16",
+            "minimum": 0
+          },
+          "rack_id": {
+            "type": "string",
+            "format": "uuid"
+          }
+        },
+        "required": [
+          "baseboard",
+          "cubby",
+          "rack_id"
+        ]
+      },
+      "UninitializedSledId": {
+        "description": "The unique hardware ID for a sled",
+        "type": "object",
+        "properties": {
+          "part": {
+            "type": "string"
+          },
+          "serial": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "part",
+          "serial"
+        ]
+      },
+      "UninitializedSledResultsPage": {
+        "description": "A single page of results",
+        "type": "object",
+        "properties": {
+          "items": {
+            "description": "list of items on this page of results",
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/UninitializedSled"
+            }
+          },
+          "next_page": {
+            "nullable": true,
+            "description": "token used to fetch the next page of results (if any)",
+            "type": "string"
+          }
+        },
+        "required": [
+          "items"
+        ]
+      },
       "UserId": {
         "title": "A name unique within the parent collection",
         "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.",
diff --git a/sled-agent/src/nexus.rs b/sled-agent/src/nexus.rs
index cc715f4010..4cd97d9ba8 100644
--- a/sled-agent/src/nexus.rs
+++ b/sled-agent/src/nexus.rs
@@ -178,8 +178,8 @@ impl ConvertInto<nexus_client::types::PhysicalDiskKind>
 impl ConvertInto<nexus_client::types::Baseboard> for sled_hardware::Baseboard {
     fn convert(self) -> nexus_client::types::Baseboard {
         nexus_client::types::Baseboard {
-            serial_number: self.identifier().to_string(),
-            part_number: self.model().to_string(),
+            serial: self.identifier().to_string(),
+            part: self.model().to_string(),
             revision: self.revision(),
         }
     }
diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs
index f9db89135a..784e3f4938 100644
--- a/sled-agent/src/sim/server.rs
+++ b/sled-agent/src/sim/server.rs
@@ -105,11 +105,11 @@ impl Server {
                             sa_address: sa_address.to_string(),
                             role: NexusTypes::SledRole::Scrimlet,
                             baseboard: NexusTypes::Baseboard {
-                                serial_number: format!(
+                                serial: format!(
                                     "sim-{}",
                                     &config.id.to_string()[0..8]
                                 ),
-                                part_number: String::from("Unknown"),
+                                part: String::from("Unknown"),
                                 revision: 0,
                             },
                             usable_hardware_threads: config

From 6120b6fcec70cb8a4b8e9b4d04898843f6b7e78f Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Thu, 7 Mar 2024 11:00:21 -0800
Subject: [PATCH 087/157] chore(deps): update rust crate reedline to 0.30.0
 (#5218)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock | 4 ++--
 Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7471aa2dc4..605c1c0466 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6986,9 +6986,9 @@ dependencies = [
 
 [[package]]
 name = "reedline"
-version = "0.29.0"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e01ebfbdb1a88963121d3c928c97be7f10fec7795bec8b918c8cda1db7c29e6"
+checksum = "413a9fa6a5d8c937d3ae1e975bfb6a918bb0b6cdfae6a10416218c837a31b8fc"
 dependencies = [
  "chrono",
  "crossterm",
diff --git a/Cargo.toml b/Cargo.toml
index 9cd22cbbcc..deda5a8dd8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -326,7 +326,7 @@ rand = "0.8.5"
 ratatui = "0.26.1"
 rayon = "1.9"
 rcgen = "0.12.1"
-reedline = "0.29.0"
+reedline = "0.30.0"
 ref-cast = "1.0"
 regex = "1.10.3"
 regress = "0.8.0"

From 81447ec02274245256a393bd38cf48c8e8a29766 Mon Sep 17 00:00:00 2001
From: bnaecker <ben@oxide.computer>
Date: Thu, 7 Mar 2024 12:38:42 -0800
Subject: [PATCH 088/157] Groundwork for oximeter instance vCPU metrics (#4900)

- Adds the silo and project IDs to the instance-ensure request from
Nexus to the sled-agent. These are used as fields on the
instance-related statistics. This metadata is currently unused, but will
be forwarded to Propolis in the instance-ensure request once the server
is updated to accept it.
- Defines a `VirtualMachine` oximeter target and `VcpuUsage` metric. The
latter has a `state` field which corresponds to the named kstats
published by the hypervisor that accumulate the time spent in a number
of vCPU microstates. The combination of these should allow us to
aggregate or break down vCPU usage by silo, project, instance, vCPU ID,
and CPU state. Adds some simple mocks and tests for these.
- Adds more fine-grained feature flags to the `oximeter-instruments`
crate.
---
 Cargo.lock                                |   4 +-
 nexus/src/app/instance.rs                 |  20 ++++
 openapi/sled-agent.json                   |  27 ++++++
 oximeter/instruments/Cargo.toml           |  43 ++++++---
 oximeter/instruments/src/kstat/mod.rs     |   9 +-
 oximeter/instruments/src/kstat/sampler.rs |   4 +-
 oximeter/instruments/src/lib.rs           |   2 +-
 oximeter/producer/src/lib.rs              |  22 +++--
 sled-agent/src/http_entrypoints.rs        |   1 +
 sled-agent/src/instance.rs                |  11 ++-
 sled-agent/src/instance_manager.rs        |  14 ++-
 sled-agent/src/metrics.rs                 | 106 +++++++++++++---------
 sled-agent/src/params.rs                  |  14 +++
 sled-agent/src/sim/http_entrypoints.rs    |   1 +
 sled-agent/src/sim/sled_agent.rs          |   5 +-
 sled-agent/src/sled_agent.rs              |  95 ++++++++++---------
 16 files changed, 261 insertions(+), 117 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 605c1c0466..5d773a9abc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1974,7 +1974,7 @@ dependencies = [
 [[package]]
 name = "dropshot"
 version = "0.9.1-dev"
-source = "git+https://github.com/oxidecomputer/dropshot?branch=main#711a7490d81416731cfe0f9fef366ed5f266a0ee"
+source = "git+https://github.com/oxidecomputer/dropshot?branch=main#29ae98d1f909c6832661408a4c03f929e8afa6e9"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -2020,7 +2020,7 @@ dependencies = [
 [[package]]
 name = "dropshot_endpoint"
 version = "0.9.1-dev"
-source = "git+https://github.com/oxidecomputer/dropshot?branch=main#711a7490d81416731cfe0f9fef366ed5f266a0ee"
+source = "git+https://github.com/oxidecomputer/dropshot?branch=main#29ae98d1f909c6832661408a4c03f929e8afa6e9"
 dependencies = [
  "proc-macro2",
  "quote",
diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs
index f6cf90718f..2d09078e18 100644
--- a/nexus/src/app/instance.rs
+++ b/nexus/src/app/instance.rs
@@ -1183,6 +1183,25 @@ impl super::Nexus {
         let ssh_keys: Vec<String> =
             ssh_keys.map(|ssh_key| ssh_key.public_key).collect();
 
+        // Construct instance metadata used to track its statistics.
+        //
+        // This requires another fetch on the silo and project, to extract their
+        // IDs.
+        let (.., db_project) = self
+            .project_lookup(
+                opctx,
+                params::ProjectSelector {
+                    project: NameOrId::Id(db_instance.project_id),
+                },
+            )?
+            .fetch()
+            .await?;
+        let (_, db_silo) = self.current_silo_lookup(opctx)?.fetch().await?;
+        let metadata = sled_agent_client::types::InstanceMetadata {
+            silo_id: db_silo.id(),
+            project_id: db_project.id(),
+        };
+
         // Ask the sled agent to begin the state change.  Then update the
         // database to reflect the new intermediate state.  If this update is
         // not the newest one, that's fine.  That might just mean the sled agent
@@ -1226,6 +1245,7 @@ impl super::Nexus {
                         initial_vmm.propolis_port.into(),
                     )
                     .to_string(),
+                    metadata,
                 },
             )
             .await
diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json
index 99156fffd4..238b5832ca 100644
--- a/openapi/sled-agent.json
+++ b/openapi/sled-agent.json
@@ -4708,6 +4708,14 @@
               }
             ]
           },
+          "metadata": {
+            "description": "Metadata used to track instance statistics.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/InstanceMetadata"
+              }
+            ]
+          },
           "propolis_addr": {
             "description": "The address at which this VMM should serve a Propolis server API.",
             "type": "string"
@@ -4729,6 +4737,7 @@
         "required": [
           "hardware",
           "instance_runtime",
+          "metadata",
           "propolis_addr",
           "propolis_id",
           "vmm_runtime"
@@ -4860,6 +4869,24 @@
           "snapshot_id"
         ]
       },
+      "InstanceMetadata": {
+        "description": "Metadata used to track statistics about an instance.",
+        "type": "object",
+        "properties": {
+          "project_id": {
+            "type": "string",
+            "format": "uuid"
+          },
+          "silo_id": {
+            "type": "string",
+            "format": "uuid"
+          }
+        },
+        "required": [
+          "project_id",
+          "silo_id"
+        ]
+      },
       "InstanceMigrationSourceParams": {
         "description": "Instance runtime state to update for a migration.",
         "type": "object",
diff --git a/oximeter/instruments/Cargo.toml b/oximeter/instruments/Cargo.toml
index 8372b7c560..c49d3b976e 100644
--- a/oximeter/instruments/Cargo.toml
+++ b/oximeter/instruments/Cargo.toml
@@ -5,27 +5,46 @@ edition = "2021"
 license = "MPL-2.0"
 
 [dependencies]
-cfg-if.workspace = true
-chrono.workspace = true
-dropshot.workspace = true
-futures.workspace = true
+cfg-if = { workspace = true, optional = true }
+chrono = { workspace = true, optional = true }
+dropshot = { workspace = true, optional = true }
+futures = { workspace = true, optional = true }
 http = { workspace = true, optional = true }
-oximeter.workspace = true
-slog.workspace = true
-tokio.workspace = true
-thiserror.workspace = true
-uuid.workspace = true
+oximeter = { workspace = true, optional = true }
+slog = { workspace = true, optional = true }
+tokio = { workspace = true, optional = true }
+thiserror = { workspace = true, optional = true }
+uuid = { workspace = true, optional = true }
 omicron-workspace-hack.workspace = true
 
 [features]
-default = ["http-instruments", "kstat"]
-http-instruments = ["http"]
-kstat = ["kstat-rs"]
+default = ["http-instruments", "datalink"]
+http-instruments = [
+    "dep:chrono",
+    "dep:dropshot",
+    "dep:futures",
+    "dep:http",
+    "dep:oximeter",
+    "dep:uuid"
+]
+kstat = [
+    "dep:cfg-if",
+    "dep:chrono",
+    "dep:futures",
+    "dep:kstat-rs",
+    "dep:oximeter",
+    "dep:slog",
+    "dep:tokio",
+    "dep:thiserror",
+    "dep:uuid"
+]
+datalink = ["kstat"]
 
 [dev-dependencies]
 rand.workspace = true
 slog-async.workspace = true
 slog-term.workspace = true
+oximeter.workspace = true
 
 [target.'cfg(target_os = "illumos")'.dependencies]
 kstat-rs = { workspace = true, optional = true }
diff --git a/oximeter/instruments/src/kstat/mod.rs b/oximeter/instruments/src/kstat/mod.rs
index 90f34acae8..c792a51408 100644
--- a/oximeter/instruments/src/kstat/mod.rs
+++ b/oximeter/instruments/src/kstat/mod.rs
@@ -2,7 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-// Copyright 2023 Oxide Computer Company
+// Copyright 2024 Oxide Computer Company
 
 //! Types for publishing kernel statistics via oximeter.
 //!
@@ -87,6 +87,7 @@ use std::cmp::Ordering;
 use std::collections::BTreeMap;
 use std::time::Duration;
 
+#[cfg(any(feature = "datalink", test))]
 pub mod link;
 mod sampler;
 
@@ -206,9 +207,9 @@ pub fn hrtime_to_utc(hrtime: i64) -> Result<DateTime<Utc>, Error> {
     }
 }
 
-// Helper trait for converting a `NamedData` item into a specific contained data
-// type, if possible.
-pub(crate) trait ConvertNamedData {
+/// Helper trait for converting a `NamedData` item into a specific contained data
+/// type, if possible.
+pub trait ConvertNamedData {
     fn as_i32(&self) -> Result<i32, Error>;
     fn as_u32(&self) -> Result<u32, Error>;
     fn as_i64(&self) -> Result<i64, Error>;
diff --git a/oximeter/instruments/src/kstat/sampler.rs b/oximeter/instruments/src/kstat/sampler.rs
index bab8ad0ba5..af1b3ba7cf 100644
--- a/oximeter/instruments/src/kstat/sampler.rs
+++ b/oximeter/instruments/src/kstat/sampler.rs
@@ -671,7 +671,7 @@ impl KstatSamplerWorker {
         let n_current_samples = current_samples.len();
         let n_total_samples = n_new_samples + n_current_samples;
         let n_overflow_samples =
-            n_total_samples.checked_sub(self.sample_limit).unwrap_or(0);
+            n_total_samples.saturating_sub(self.sample_limit);
         if n_overflow_samples > 0 {
             warn!(
                 self.log,
@@ -788,7 +788,7 @@ impl KstatSamplerWorker {
                         // or the time we added the kstat if not.
                         let start = sampled_kstat
                             .time_of_last_collection
-                            .unwrap_or_else(|| sampled_kstat.time_added);
+                            .unwrap_or(sampled_kstat.time_added);
                         let expire_at = start + duration;
                         cfg_if::cfg_if! {
                             if #[cfg(test)] {
diff --git a/oximeter/instruments/src/lib.rs b/oximeter/instruments/src/lib.rs
index d003e71739..c1f839c85d 100644
--- a/oximeter/instruments/src/lib.rs
+++ b/oximeter/instruments/src/lib.rs
@@ -4,7 +4,7 @@
 
 //! General-purpose types for instrumenting code to producer oximeter metrics.
 
-// Copyright 2023 Oxide Computer Company
+// Copyright 2024 Oxide Computer Company
 
 #[cfg(feature = "http-instruments")]
 pub mod http;
diff --git a/oximeter/producer/src/lib.rs b/oximeter/producer/src/lib.rs
index 2354f9c217..3fecaadf4f 100644
--- a/oximeter/producer/src/lib.rs
+++ b/oximeter/producer/src/lib.rs
@@ -38,8 +38,8 @@ pub enum Error {
     #[error("Error running producer HTTP server: {0}")]
     Server(String),
 
-    #[error("Error registering as metric producer: {0}")]
-    RegistrationError(String),
+    #[error("Error registering as metric producer: {msg}")]
+    RegistrationError { retryable: bool, msg: String },
 
     #[error("Producer registry and config UUIDs do not match")]
     UuidMismatch,
@@ -251,11 +251,19 @@ pub async fn register(
 ) -> Result<(), Error> {
     let client =
         nexus_client::Client::new(&format!("http://{}", address), log.clone());
-    client
-        .cpapi_producers_post(&server_info.into())
-        .await
-        .map(|_| ())
-        .map_err(|msg| Error::RegistrationError(msg.to_string()))
+    client.cpapi_producers_post(&server_info.into()).await.map(|_| ()).map_err(
+        |err| {
+            let retryable = match &err {
+                nexus_client::Error::CommunicationError(..) => true,
+                nexus_client::Error::ErrorResponse(resp) => {
+                    resp.status().is_server_error()
+                }
+                _ => false,
+            };
+            let msg = err.to_string();
+            Error::RegistrationError { retryable, msg }
+        },
+    )
 }
 
 /// Handle a request to pull available metric data from a [`ProducerRegistry`].
diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs
index 5f888504db..6ce9d59b1c 100644
--- a/sled-agent/src/http_entrypoints.rs
+++ b/sled-agent/src/http_entrypoints.rs
@@ -413,6 +413,7 @@ async fn instance_register(
             body_args.instance_runtime,
             body_args.vmm_runtime,
             body_args.propolis_addr,
+            body_args.metadata,
         )
         .await?,
     ))
diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs
index 7a6033b4bb..5ddca90403 100644
--- a/sled-agent/src/instance.rs
+++ b/sled-agent/src/instance.rs
@@ -15,7 +15,7 @@ use crate::nexus::NexusClientWithResolver;
 use crate::params::ZoneBundleMetadata;
 use crate::params::{InstanceExternalIpBody, ZoneBundleCause};
 use crate::params::{
-    InstanceHardware, InstanceMigrationSourceParams,
+    InstanceHardware, InstanceMetadata, InstanceMigrationSourceParams,
     InstanceMigrationTargetParams, InstancePutStateResponse,
     InstanceStateRequested, InstanceUnregisterResponse, VpcFirewallRule,
 };
@@ -311,6 +311,11 @@ struct InstanceRunner {
     // Properties visible to Propolis
     properties: propolis_client::types::InstanceProperties,
 
+    // This is currently unused, but will be sent to Propolis as part of the
+    // work tracked in https://github.com/oxidecomputer/omicron/issues/4851. It
+    // will be included in the InstanceProperties above, most likely.
+    _metadata: InstanceMetadata,
+
     // The ID of the Propolis server (and zone) running this instance
     propolis_id: Uuid,
 
@@ -928,6 +933,7 @@ impl Instance {
         ticket: InstanceTicket,
         state: InstanceInitialState,
         services: InstanceManagerServices,
+        metadata: InstanceMetadata,
     ) -> Result<Self, Error> {
         info!(log, "initializing new Instance";
               "instance_id" => %id,
@@ -1005,6 +1011,9 @@ impl Instance {
                 // InstanceCpuCount here, to avoid any casting...
                 vcpus: hardware.properties.ncpus.0 as u8,
             },
+            // This will be used in a follow up, tracked under
+            // https://github.com/oxidecomputer/omicron/issues/4851.
+            _metadata: metadata,
             propolis_id,
             propolis_addr,
             vnic_allocator,
diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs
index 80c62be234..666d970538 100644
--- a/sled-agent/src/instance_manager.rs
+++ b/sled-agent/src/instance_manager.rs
@@ -8,6 +8,7 @@ use crate::instance::propolis_zone_name;
 use crate::instance::Instance;
 use crate::nexus::NexusClientWithResolver;
 use crate::params::InstanceExternalIpBody;
+use crate::params::InstanceMetadata;
 use crate::params::ZoneBundleMetadata;
 use crate::params::{
     InstanceHardware, InstanceMigrationSourceParams, InstancePutStateResponse,
@@ -227,6 +228,7 @@ impl InstanceManager {
         *self.inner.reservoir_size.lock().unwrap()
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub async fn ensure_registered(
         &self,
         instance_id: Uuid,
@@ -235,6 +237,7 @@ impl InstanceManager {
         instance_runtime: InstanceRuntimeState,
         vmm_runtime: VmmRuntimeState,
         propolis_addr: SocketAddr,
+        metadata: InstanceMetadata,
     ) -> Result<SledInstanceState, Error> {
         let (tx, rx) = oneshot::channel();
         self.inner
@@ -246,6 +249,7 @@ impl InstanceManager {
                 instance_runtime,
                 vmm_runtime,
                 propolis_addr,
+                metadata,
                 tx,
             })
             .await
@@ -396,6 +400,7 @@ enum InstanceManagerRequest {
         instance_runtime: InstanceRuntimeState,
         vmm_runtime: VmmRuntimeState,
         propolis_addr: SocketAddr,
+        metadata: InstanceMetadata,
         tx: oneshot::Sender<Result<SledInstanceState, Error>>,
     },
     EnsureUnregistered {
@@ -509,9 +514,10 @@ impl InstanceManagerRunner {
                             instance_runtime,
                             vmm_runtime,
                             propolis_addr,
+                            metadata,
                             tx,
                         }) => {
-                            tx.send(self.ensure_registered(instance_id, propolis_id, hardware, instance_runtime, vmm_runtime, propolis_addr).await).map_err(|_| Error::FailedSendClientClosed)
+                            tx.send(self.ensure_registered(instance_id, propolis_id, hardware, instance_runtime, vmm_runtime, propolis_addr, metadata).await).map_err(|_| Error::FailedSendClientClosed)
                         },
                         Some(EnsureUnregistered { instance_id, tx }) => {
                             self.ensure_unregistered(tx, instance_id).await
@@ -574,7 +580,8 @@ impl InstanceManagerRunner {
     /// (instance ID, Propolis ID) pair multiple times, but will fail if the
     /// instance is registered with a Propolis ID different from the one the
     /// caller supplied.
-    async fn ensure_registered(
+    #[allow(clippy::too_many_arguments)]
+    pub async fn ensure_registered(
         &mut self,
         instance_id: Uuid,
         propolis_id: Uuid,
@@ -582,6 +589,7 @@ impl InstanceManagerRunner {
         instance_runtime: InstanceRuntimeState,
         vmm_runtime: VmmRuntimeState,
         propolis_addr: SocketAddr,
+        metadata: InstanceMetadata,
     ) -> Result<SledInstanceState, Error> {
         info!(
             &self.log,
@@ -592,6 +600,7 @@ impl InstanceManagerRunner {
             "instance_runtime" => ?instance_runtime,
             "vmm_runtime" => ?vmm_runtime,
             "propolis_addr" => ?propolis_addr,
+            "metadata" => ?metadata,
         );
 
         let instance = {
@@ -646,6 +655,7 @@ impl InstanceManagerRunner {
                     ticket,
                     state,
                     services,
+                    metadata,
                 )?;
                 let _old =
                     self.instances.insert(instance_id, (propolis_id, instance));
diff --git a/sled-agent/src/metrics.rs b/sled-agent/src/metrics.rs
index 6c3383c88f..a9d5acfff8 100644
--- a/sled-agent/src/metrics.rs
+++ b/sled-agent/src/metrics.rs
@@ -8,6 +8,7 @@ use oximeter::types::MetricsError;
 use oximeter::types::ProducerRegistry;
 use sled_hardware::Baseboard;
 use slog::Logger;
+use std::sync::Arc;
 use std::time::Duration;
 use uuid::Uuid;
 
@@ -19,6 +20,7 @@ cfg_if::cfg_if! {
         use oximeter_instruments::kstat::KstatSampler;
         use oximeter_instruments::kstat::TargetId;
         use std::collections::BTreeMap;
+        use std::sync::Mutex;
     } else {
         use anyhow::anyhow;
     }
@@ -46,6 +48,21 @@ pub enum Error {
 
     #[error("Failed to fetch hostname")]
     Hostname(#[source] std::io::Error),
+
+    #[error("Non-UTF8 hostname")]
+    NonUtf8Hostname,
+
+    #[error("Missing NULL byte in hostname")]
+    HostnameMissingNull,
+}
+
+// Basic metadata about the sled agent used when publishing metrics.
+#[derive(Clone, Debug)]
+#[cfg_attr(not(target_os = "illumos"), allow(dead_code))]
+struct SledIdentifiers {
+    sled_id: Uuid,
+    rack_id: Uuid,
+    baseboard: Baseboard,
 }
 
 /// Type managing all oximeter metrics produced by the sled-agent.
@@ -61,10 +78,7 @@ pub enum Error {
 // the name of fields that are not yet used.
 #[cfg_attr(not(target_os = "illumos"), allow(dead_code))]
 pub struct MetricsManager {
-    sled_id: Uuid,
-    rack_id: Uuid,
-    baseboard: Baseboard,
-    hostname: Option<String>,
+    metadata: Arc<SledIdentifiers>,
     _log: Logger,
     #[cfg(target_os = "illumos")]
     kstat_sampler: KstatSampler,
@@ -78,7 +92,7 @@ pub struct MetricsManager {
     // namespace them internally, e.g., `"datalink:{link_name}"` would be the
     // real key.
     #[cfg(target_os = "illumos")]
-    tracked_links: BTreeMap<String, TargetId>,
+    tracked_links: Arc<Mutex<BTreeMap<String, TargetId>>>,
     registry: ProducerRegistry,
 }
 
@@ -101,14 +115,11 @@ impl MetricsManager {
                 registry
                     .register_producer(kstat_sampler.clone())
                     .map_err(Error::Registry)?;
-                let tracked_links = BTreeMap::new();
+                let tracked_links = Arc::new(Mutex::new(BTreeMap::new()));
             }
         }
         Ok(Self {
-            sled_id,
-            rack_id,
-            baseboard,
-            hostname: None,
+            metadata: Arc::new(SledIdentifiers { sled_id, rack_id, baseboard }),
             _log: log,
             #[cfg(target_os = "illumos")]
             kstat_sampler,
@@ -128,14 +139,14 @@ impl MetricsManager {
 impl MetricsManager {
     /// Track metrics for a physical datalink.
     pub async fn track_physical_link(
-        &mut self,
+        &self,
         link_name: impl AsRef<str>,
         interval: Duration,
     ) -> Result<(), Error> {
-        let hostname = self.hostname().await?;
+        let hostname = hostname()?;
         let link = link::PhysicalDataLink {
-            rack_id: self.rack_id,
-            sled_id: self.sled_id,
+            rack_id: self.metadata.rack_id,
+            sled_id: self.metadata.sled_id,
             serial: self.serial_number(),
             hostname,
             link_name: link_name.as_ref().to_string(),
@@ -146,7 +157,10 @@ impl MetricsManager {
             .add_target(link, details)
             .await
             .map_err(Error::Kstat)?;
-        self.tracked_links.insert(link_name.as_ref().to_string(), id);
+        self.tracked_links
+            .lock()
+            .unwrap()
+            .insert(link_name.as_ref().to_string(), id);
         Ok(())
     }
 
@@ -155,10 +169,12 @@ impl MetricsManager {
     /// This works for both physical and virtual links.
     #[allow(dead_code)]
     pub async fn stop_tracking_link(
-        &mut self,
+        &self,
         link_name: impl AsRef<str>,
     ) -> Result<(), Error> {
-        if let Some(id) = self.tracked_links.remove(link_name.as_ref()) {
+        let maybe_id =
+            self.tracked_links.lock().unwrap().remove(link_name.as_ref());
+        if let Some(id) = maybe_id {
             self.kstat_sampler.remove_target(id).await.map_err(Error::Kstat)
         } else {
             Ok(())
@@ -174,8 +190,8 @@ impl MetricsManager {
         interval: Duration,
     ) -> Result<(), Error> {
         let link = link::VirtualDataLink {
-            rack_id: self.rack_id,
-            sled_id: self.sled_id,
+            rack_id: self.metadata.rack_id,
+            sled_id: self.metadata.sled_id,
             serial: self.serial_number(),
             hostname: hostname.as_ref().to_string(),
             link_name: link_name.as_ref().to_string(),
@@ -190,40 +206,19 @@ impl MetricsManager {
 
     // Return the serial number out of the baseboard, if one exists.
     fn serial_number(&self) -> String {
-        match &self.baseboard {
+        match &self.metadata.baseboard {
             Baseboard::Gimlet { identifier, .. } => identifier.clone(),
             Baseboard::Unknown => String::from("unknown"),
             Baseboard::Pc { identifier, .. } => identifier.clone(),
         }
     }
-
-    // Return the system's hostname.
-    //
-    // If we've failed to get it previously, we try again. If _that_ fails,
-    // return an error.
-    //
-    // TODO-cleanup: This will become much simpler once
-    // `OnceCell::get_or_try_init` is stabilized.
-    async fn hostname(&mut self) -> Result<String, Error> {
-        if let Some(hn) = &self.hostname {
-            return Ok(hn.clone());
-        }
-        let hn = tokio::process::Command::new("hostname")
-            .env_clear()
-            .output()
-            .await
-            .map(|out| String::from_utf8_lossy(&out.stdout).trim().to_string())
-            .map_err(Error::Hostname)?;
-        self.hostname.replace(hn.clone());
-        Ok(hn)
-    }
 }
 
 #[cfg(not(target_os = "illumos"))]
 impl MetricsManager {
     /// Track metrics for a physical datalink.
     pub async fn track_physical_link(
-        &mut self,
+        &self,
         _link_name: impl AsRef<str>,
         _interval: Duration,
     ) -> Result<(), Error> {
@@ -237,7 +232,7 @@ impl MetricsManager {
     /// This works for both physical and virtual links.
     #[allow(dead_code)]
     pub async fn stop_tracking_link(
-        &mut self,
+        &self,
         _link_name: impl AsRef<str>,
     ) -> Result<(), Error> {
         Err(Error::Kstat(anyhow!(
@@ -258,3 +253,28 @@ impl MetricsManager {
         )))
     }
 }
+
+// Return the current hostname if possible.
+#[cfg(target_os = "illumos")]
+fn hostname() -> Result<String, Error> {
+    // See netdb.h
+    const MAX_LEN: usize = 256;
+    let mut out = vec![0u8; MAX_LEN + 1];
+    if unsafe {
+        libc::gethostname(out.as_mut_ptr() as *mut libc::c_char, MAX_LEN)
+    } == 0
+    {
+        // Split into subslices by NULL bytes.
+        //
+        // We should have a NULL byte, since we've asked for no more than 255
+        // bytes in a 256 byte buffer, but you never know.
+        let Some(chunk) = out.split(|x| *x == 0).next() else {
+            return Err(Error::HostnameMissingNull);
+        };
+        let s = std::ffi::CString::new(chunk)
+            .map_err(|_| Error::NonUtf8Hostname)?;
+        s.into_string().map_err(|_| Error::NonUtf8Hostname)
+    } else {
+        Err(std::io::Error::last_os_error()).map_err(|_| Error::NonUtf8Hostname)
+    }
+}
diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs
index fda952ca87..d192f745f6 100644
--- a/sled-agent/src/params.rs
+++ b/sled-agent/src/params.rs
@@ -82,6 +82,17 @@ pub struct InstanceHardware {
     pub cloud_init_bytes: Option<String>,
 }
 
+/// Metadata used to track statistics about an instance.
+///
+// NOTE: The instance ID is not here, since it's already provided in other
+// pieces of the instance-related requests. It is pulled from there when
+// publishing metrics for the instance.
+#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]
+pub struct InstanceMetadata {
+    pub silo_id: Uuid,
+    pub project_id: Uuid,
+}
+
 /// The body of a request to ensure that a instance and VMM are known to a sled
 /// agent.
 #[derive(Serialize, Deserialize, JsonSchema)]
@@ -103,6 +114,9 @@ pub struct InstanceEnsureBody {
 
     /// The address at which this VMM should serve a Propolis server API.
     pub propolis_addr: SocketAddr,
+
+    /// Metadata used to track instance statistics.
+    pub metadata: InstanceMetadata,
 }
 
 /// The body of a request to move a previously-ensured instance into a specific
diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs
index 09ffdf5dc4..494db58828 100644
--- a/sled-agent/src/sim/http_entrypoints.rs
+++ b/sled-agent/src/sim/http_entrypoints.rs
@@ -98,6 +98,7 @@ async fn instance_register(
             body_args.hardware,
             body_args.instance_runtime,
             body_args.vmm_runtime,
+            body_args.metadata,
         )
         .await?,
     ))
diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs
index 27a06e4617..483b2d6aa8 100644
--- a/sled-agent/src/sim/sled_agent.rs
+++ b/sled-agent/src/sim/sled_agent.rs
@@ -13,7 +13,7 @@ use super::storage::Storage;
 use crate::nexus::NexusClient;
 use crate::params::{
     DiskStateRequested, InstanceExternalIpBody, InstanceHardware,
-    InstanceMigrationSourceParams, InstancePutStateResponse,
+    InstanceMetadata, InstanceMigrationSourceParams, InstancePutStateResponse,
     InstanceStateRequested, InstanceUnregisterResponse, Inventory,
     OmicronZonesConfig, SledRole,
 };
@@ -239,6 +239,9 @@ impl SledAgent {
         hardware: InstanceHardware,
         instance_runtime: InstanceRuntimeState,
         vmm_runtime: VmmRuntimeState,
+        // This is currently unused, but will be included as part of work
+        // tracked in https://github.com/oxidecomputer/omicron/issues/4851.
+        _metadata: InstanceMetadata,
     ) -> Result<SledInstanceState, Error> {
         // respond with a fake 500 level failure if asked to ensure an instance
         // with more than 16 CPUs.
diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs
index 39ba1045f7..5381337b8c 100644
--- a/sled-agent/src/sled_agent.rs
+++ b/sled-agent/src/sled_agent.rs
@@ -17,7 +17,7 @@ use crate::metrics::MetricsManager;
 use crate::nexus::{ConvertInto, NexusClientWithResolver, NexusRequestQueue};
 use crate::params::{
     DiskStateRequested, InstanceExternalIpBody, InstanceHardware,
-    InstanceMigrationSourceParams, InstancePutStateResponse,
+    InstanceMetadata, InstanceMigrationSourceParams, InstancePutStateResponse,
     InstanceStateRequested, InstanceUnregisterResponse, Inventory,
     OmicronZonesConfig, SledRole, TimeSync, VpcFirewallRule,
     ZoneBundleMetadata, Zpool,
@@ -401,6 +401,55 @@ impl SledAgent {
         let underlay_nics = underlay::find_nics(&config.data_links)?;
         illumos_utils::opte::initialize_xde_driver(&log, &underlay_nics)?;
 
+        // Start collecting metric data.
+        //
+        // First, we're creating a shareable type for managing the metrics
+        // themselves early on, so that we can pass it to other components of
+        // the sled agent that need it.
+        //
+        // Then we'll start tracking physical links and register as a producer
+        // with Nexus in the background.
+        let metrics_manager = MetricsManager::new(
+            request.body.id,
+            request.body.rack_id,
+            long_running_task_handles.hardware_manager.baseboard(),
+            log.new(o!("component" => "MetricsManager")),
+        )?;
+
+        // Start tracking the underlay physical links.
+        for nic in underlay::find_nics(&config.data_links)? {
+            let link_name = nic.interface();
+            if let Err(e) = metrics_manager
+                .track_physical_link(
+                    link_name,
+                    crate::metrics::LINK_SAMPLE_INTERVAL,
+                )
+                .await
+            {
+                error!(
+                    log,
+                    "failed to start tracking physical link metrics";
+                    "link_name" => link_name,
+                    "error" => ?e,
+                );
+            }
+        }
+
+        // Spawn a task in the background to register our metric producer with
+        // Nexus. This should not block progress here.
+        let endpoint = ProducerEndpoint {
+            id: request.body.id,
+            kind: ProducerKind::SledAgent,
+            address: sled_address.into(),
+            base_route: String::from("/metrics/collect"),
+            interval: crate::metrics::METRIC_COLLECTION_INTERVAL,
+        };
+        tokio::task::spawn(register_metric_producer_with_nexus(
+            log.clone(),
+            nexus_client.clone(),
+            endpoint,
+        ));
+
         // Create the PortManager to manage all the OPTE ports on the sled.
         let port_manager = PortManager::new(
             parent_log.new(o!("component" => "PortManager")),
@@ -521,47 +570,6 @@ impl SledAgent {
             rack_network_config.clone(),
         )?;
 
-        let mut metrics_manager = MetricsManager::new(
-            request.body.id,
-            request.body.rack_id,
-            long_running_task_handles.hardware_manager.baseboard(),
-            log.new(o!("component" => "MetricsManager")),
-        )?;
-
-        // Start tracking the underlay physical links.
-        for nic in underlay::find_nics(&config.data_links)? {
-            let link_name = nic.interface();
-            if let Err(e) = metrics_manager
-                .track_physical_link(
-                    link_name,
-                    crate::metrics::LINK_SAMPLE_INTERVAL,
-                )
-                .await
-            {
-                error!(
-                    log,
-                    "failed to start tracking physical link metrics";
-                    "link_name" => link_name,
-                    "error" => ?e,
-                );
-            }
-        }
-
-        // Spawn a task in the background to register our metric producer with
-        // Nexus. This should not block progress here.
-        let endpoint = ProducerEndpoint {
-            id: request.body.id,
-            kind: ProducerKind::SledAgent,
-            address: sled_address.into(),
-            base_route: String::from("/metrics/collect"),
-            interval: crate::metrics::METRIC_COLLECTION_INTERVAL,
-        };
-        tokio::task::spawn(register_metric_producer_with_nexus(
-            log.clone(),
-            nexus_client.clone(),
-            endpoint,
-        ));
-
         let sled_agent = SledAgent {
             inner: Arc::new(SledAgentInner {
                 id: request.body.id,
@@ -945,6 +953,7 @@ impl SledAgent {
     /// Idempotently ensures that a given instance is registered with this sled,
     /// i.e., that it can be addressed by future calls to
     /// [`Self::instance_ensure_state`].
+    #[allow(clippy::too_many_arguments)]
     pub async fn instance_ensure_registered(
         &self,
         instance_id: Uuid,
@@ -953,6 +962,7 @@ impl SledAgent {
         instance_runtime: InstanceRuntimeState,
         vmm_runtime: VmmRuntimeState,
         propolis_addr: SocketAddr,
+        metadata: InstanceMetadata,
     ) -> Result<SledInstanceState, Error> {
         self.inner
             .instances
@@ -963,6 +973,7 @@ impl SledAgent {
                 instance_runtime,
                 vmm_runtime,
                 propolis_addr,
+                metadata,
             )
             .await
             .map_err(|e| Error::Instance(e))

From cd0effee306ed1ec563bd328ed749840f512ed60 Mon Sep 17 00:00:00 2001
From: David Pacheco <dap@oxidecomputer.com>
Date: Thu, 7 Mar 2024 13:07:19 -0800
Subject: [PATCH 089/157] add reconfigurator-cli and facilities for exploring,
 testing the planner (#5197)

---
 Cargo.lock                                    |    49 +-
 Cargo.toml                                    |     7 +-
 dev-tools/omdb/Cargo.toml                     |     4 +-
 dev-tools/omdb/src/bin/omdb/db.rs             |   136 +
 dev-tools/omdb/src/bin/omdb/nexus.rs          |    34 +-
 dev-tools/omdb/tests/successes.out            |    13 +
 dev-tools/omdb/tests/test_all_output.rs       |   122 +-
 dev-tools/omdb/tests/usage_errors.out         |    46 +-
 dev-tools/reconfigurator-cli/Cargo.toml       |    40 +
 dev-tools/reconfigurator-cli/src/main.rs      |   768 +
 .../reconfigurator-cli/tests/input/cmds.txt   |    14 +
 .../tests/input/complex.json                  | 11558 ++++++++++++++++
 .../tests/output/cmd-complex-stdout           |   385 +
 .../tests/output/cmd-stderr                   |     0
 .../tests/output/cmd-stdout                   |    58 +
 .../reconfigurator-cli/tests/test_basic.rs    |   115 +
 nexus/Cargo.toml                              |     1 +
 .../db-queries/src/db/datastore/deployment.rs |    11 +-
 nexus/db-queries/src/db/datastore/ip_pool.rs  |    29 +
 nexus/db-queries/src/db/datastore/zpool.rs    |    32 +-
 nexus/inventory/src/builder.rs                |     7 +-
 nexus/reconfigurator/planning/Cargo.toml      |     3 +
 .../planning/src/blueprint_builder.rs         |   357 +-
 nexus/reconfigurator/planning/src/lib.rs      |     1 +
 nexus/reconfigurator/planning/src/planner.rs  |    31 +-
 nexus/reconfigurator/planning/src/system.rs   |   578 +
 nexus/reconfigurator/preparation/Cargo.toml   |    12 +
 nexus/reconfigurator/preparation/src/lib.rs   |    74 +
 nexus/src/app/deployment.rs                   |   102 +-
 nexus/types/Cargo.toml                        |     2 +
 nexus/types/src/deployment.rs                 |    82 +-
 nexus/types/src/inventory.rs                  |    67 +-
 test-utils/Cargo.toml                         |     2 +-
 test-utils/src/dev/test_cmds.rs               |    70 +
 34 files changed, 14330 insertions(+), 480 deletions(-)
 create mode 100644 dev-tools/reconfigurator-cli/Cargo.toml
 create mode 100644 dev-tools/reconfigurator-cli/src/main.rs
 create mode 100644 dev-tools/reconfigurator-cli/tests/input/cmds.txt
 create mode 100644 dev-tools/reconfigurator-cli/tests/input/complex.json
 create mode 100644 dev-tools/reconfigurator-cli/tests/output/cmd-complex-stdout
 create mode 100644 dev-tools/reconfigurator-cli/tests/output/cmd-stderr
 create mode 100644 dev-tools/reconfigurator-cli/tests/output/cmd-stdout
 create mode 100644 dev-tools/reconfigurator-cli/tests/test_basic.rs
 create mode 100644 nexus/reconfigurator/planning/src/system.rs
 create mode 100644 nexus/reconfigurator/preparation/Cargo.toml
 create mode 100644 nexus/reconfigurator/preparation/src/lib.rs

diff --git a/Cargo.lock b/Cargo.lock
index 5d773a9abc..fd9155bb8e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4521,6 +4521,8 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "chrono",
+ "gateway-client",
+ "indexmap 2.2.5",
  "internal-dns",
  "ipnet",
  "ipnetwork",
@@ -4536,6 +4538,17 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "nexus-reconfigurator-preparation"
+version = "0.1.0"
+dependencies = [
+ "illumos-utils",
+ "nexus-db-model",
+ "nexus-types",
+ "omicron-common",
+ "omicron-workspace-hack",
+]
+
 [[package]]
 name = "nexus-test-interface"
 version = "0.1.0"
@@ -4611,6 +4624,7 @@ dependencies = [
  "dns-service-client",
  "futures",
  "gateway-client",
+ "humantime",
  "omicron-common",
  "omicron-passwords",
  "omicron-uuid-kinds",
@@ -4620,6 +4634,7 @@ dependencies = [
  "schemars",
  "serde",
  "serde_json",
+ "serde_with",
  "sled-agent-client",
  "steno",
  "strum 0.26.1",
@@ -5074,6 +5089,7 @@ dependencies = [
  "nexus-inventory",
  "nexus-reconfigurator-execution",
  "nexus-reconfigurator-planning",
+ "nexus-reconfigurator-preparation",
  "nexus-test-interface",
  "nexus-test-utils",
  "nexus-test-utils-macros",
@@ -5148,6 +5164,8 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-bb8-diesel",
+ "camino",
+ "camino-tempfile",
  "chrono",
  "clap 4.5.0",
  "crossterm",
@@ -5169,6 +5187,7 @@ dependencies = [
  "nexus-config",
  "nexus-db-model",
  "nexus-db-queries",
+ "nexus-reconfigurator-preparation",
  "nexus-test-utils",
  "nexus-test-utils-macros",
  "nexus-types",
@@ -5180,7 +5199,6 @@ dependencies = [
  "oximeter-client",
  "pq-sys",
  "ratatui",
- "regex",
  "serde",
  "serde_json",
  "sled-agent-client",
@@ -6946,6 +6964,35 @@ dependencies = [
  "rand_core 0.3.1",
 ]
 
+[[package]]
+name = "reconfigurator-cli"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "camino",
+ "camino-tempfile",
+ "clap 4.5.0",
+ "dropshot",
+ "expectorate",
+ "humantime",
+ "indexmap 2.2.5",
+ "nexus-reconfigurator-planning",
+ "nexus-types",
+ "omicron-common",
+ "omicron-rpaths",
+ "omicron-test-utils",
+ "omicron-workspace-hack",
+ "reedline",
+ "regex",
+ "serde_json",
+ "slog",
+ "slog-error-chain",
+ "subprocess",
+ "swrite",
+ "tabled",
+ "uuid",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"
diff --git a/Cargo.toml b/Cargo.toml
index deda5a8dd8..299d715d67 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -21,6 +21,7 @@ members = [
     "dev-tools/omdb",
     "dev-tools/omicron-dev",
     "dev-tools/oxlog",
+    "dev-tools/reconfigurator-cli",
     "dev-tools/xtask",
     "dns-server",
     "end-to-end-tests",
@@ -46,6 +47,7 @@ members = [
     "nexus/macros-common",
     "nexus/reconfigurator/execution",
     "nexus/reconfigurator/planning",
+    "nexus/reconfigurator/preparation",
     "nexus/test-interface",
     "nexus/test-utils-macros",
     "nexus/test-utils",
@@ -98,6 +100,7 @@ default-members = [
     "dev-tools/omdb",
     "dev-tools/omicron-dev",
     "dev-tools/oxlog",
+    "dev-tools/reconfigurator-cli",
     # Do not include xtask in the list of default members, because this causes
     # hakari to not work as well and build times to be longer.
     # See omicron#4392.
@@ -124,6 +127,7 @@ default-members = [
     "nexus/inventory",
     "nexus/reconfigurator/execution",
     "nexus/reconfigurator/planning",
+    "nexus/reconfigurator/preparation",
     "nexus/types",
     "oximeter/collector",
     "oximeter/db",
@@ -264,6 +268,7 @@ nexus-inventory = { path = "nexus/inventory" }
 nexus-macros-common = { path = "nexus/macros-common" }
 nexus-reconfigurator-execution = { path = "nexus/reconfigurator/execution" }
 nexus-reconfigurator-planning = { path = "nexus/reconfigurator/planning" }
+nexus-reconfigurator-preparation = { path = "nexus/reconfigurator/preparation" }
 omicron-certificates = { path = "certificates" }
 omicron-passwords = { path = "passwords" }
 omicron-workspace-hack = "0.1.0"
@@ -342,7 +347,7 @@ samael = { version = "0.0.14", features = ["xmlsec"] }
 schemars = "0.8.16"
 secrecy = "0.8.0"
 semver = { version = "1.0.22", features = ["std", "serde"] }
-serde = { version = "1.0", default-features = false, features = [ "derive" ] }
+serde = { version = "1.0", default-features = false, features = [ "derive", "rc" ] }
 serde_derive = "1.0"
 serde_human_bytes = { git = "http://github.com/oxidecomputer/serde_human_bytes", branch = "main" }
 serde_json = "1.0.114"
diff --git a/dev-tools/omdb/Cargo.toml b/dev-tools/omdb/Cargo.toml
index df56fda571..2b0480be2d 100644
--- a/dev-tools/omdb/Cargo.toml
+++ b/dev-tools/omdb/Cargo.toml
@@ -10,6 +10,7 @@ omicron-rpaths.workspace = true
 [dependencies]
 anyhow.workspace = true
 async-bb8-diesel.workspace = true
+camino.workspace = true
 chrono.workspace = true
 clap.workspace = true
 crossterm.workspace = true
@@ -28,6 +29,7 @@ nexus-client.workspace = true
 nexus-config.workspace = true
 nexus-db-model.workspace = true
 nexus-db-queries.workspace = true
+nexus-reconfigurator-preparation.workspace = true
 nexus-types.workspace = true
 omicron-common.workspace = true
 oximeter-client.workspace = true
@@ -55,8 +57,8 @@ expectorate.workspace = true
 nexus-test-utils-macros.workspace = true
 omicron-nexus.workspace = true
 omicron-test-utils.workspace = true
-regex.workspace = true
 subprocess.workspace = true
+camino-tempfile.workspace = true
 
 # Disable doc builds by default for our binaries to work around issue
 # rust-lang/cargo#8373.  These docs would not be very useful anyway.
diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs
index c7e9022fc2..02b68144df 100644
--- a/dev-tools/omdb/src/bin/omdb/db.rs
+++ b/dev-tools/omdb/src/bin/omdb/db.rs
@@ -22,6 +22,7 @@ use anyhow::Context;
 use async_bb8_diesel::AsyncConnection;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use async_bb8_diesel::AsyncSimpleConnection;
+use camino::Utf8PathBuf;
 use chrono::SecondsFormat;
 use clap::Args;
 use clap::Subcommand;
@@ -34,6 +35,8 @@ use diesel::JoinOnDsl;
 use diesel::NullableExpressionMethods;
 use diesel::OptionalExtension;
 use diesel::TextExpressionMethods;
+use dropshot::PaginationOrder;
+use futures::StreamExt;
 use gateway_client::types::SpType;
 use ipnetwork::IpNetwork;
 use nexus_config::PostgresConfigWithUrl;
@@ -68,20 +71,26 @@ use nexus_db_queries::db;
 use nexus_db_queries::db::datastore::read_only_resources_associated_with_volume;
 use nexus_db_queries::db::datastore::CrucibleTargets;
 use nexus_db_queries::db::datastore::DataStoreConnection;
+use nexus_db_queries::db::datastore::DataStoreInventoryTest;
 use nexus_db_queries::db::datastore::InstanceAndActiveVmm;
 use nexus_db_queries::db::identity::Asset;
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_db_queries::db::model::ServiceKind;
 use nexus_db_queries::db::DataStore;
+use nexus_reconfigurator_preparation::policy_from_db;
 use nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL;
+use nexus_types::deployment::Blueprint;
+use nexus_types::deployment::UnstableReconfiguratorState;
 use nexus_types::identity::Resource;
 use nexus_types::internal_api::params::DnsRecord;
 use nexus_types::internal_api::params::Srv;
 use nexus_types::inventory::CabooseWhich;
 use nexus_types::inventory::Collection;
 use nexus_types::inventory::RotPageWhich;
+use omicron_common::address::NEXUS_REDUNDANCY;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::Generation;
+use omicron_common::api::external::LookupType;
 use omicron_common::api::external::MacAddr;
 use sled_agent_client::types::VolumeConstructionRequest;
 use std::borrow::Cow;
@@ -175,6 +184,8 @@ enum DbCommands {
     Dns(DnsArgs),
     /// Print information about collected hardware/software inventory
     Inventory(InventoryArgs),
+    /// Save the current Reconfigurator inputs to a file
+    ReconfiguratorSave(ReconfiguratorSaveArgs),
     /// Print information about control plane services
     Services(ServicesArgs),
     /// Print information about sleds
@@ -310,6 +321,12 @@ struct CollectionsShowArgs {
     show_long_strings: bool,
 }
 
+#[derive(Debug, Args)]
+struct ReconfiguratorSaveArgs {
+    /// where to save the output
+    output_file: Utf8PathBuf,
+}
+
 #[derive(Debug, Args)]
 struct ServicesArgs {
     #[command(subcommand)]
@@ -464,6 +481,15 @@ impl DbArgs {
                 )
                 .await
             }
+            DbCommands::ReconfiguratorSave(reconfig_save_args) => {
+                cmd_db_reconfigurator_save(
+                    &opctx,
+                    &datastore,
+                    &self.fetch_opts,
+                    reconfig_save_args,
+                )
+                .await
+            }
             DbCommands::Services(ServicesArgs {
                 command: ServicesCommands::ListInstances,
             }) => {
@@ -3104,3 +3130,113 @@ impl LongStringFormatter {
         s.into()
     }
 }
+
+// Reconfigurator
+
+/// Packages up database state that's used as input to the Reconfigurator
+/// planner into a file so that it can be loaded into `reconfigurator-cli`
+async fn cmd_db_reconfigurator_save(
+    opctx: &OpContext,
+    datastore: &DataStore,
+    fetch_opts: &DbFetchOptions,
+    reconfig_save_args: &ReconfiguratorSaveArgs,
+) -> Result<(), anyhow::Error> {
+    // See Nexus::blueprint_planning_context().
+    eprint!("assembling policy ... ");
+    let sled_rows = datastore
+        .sled_list_all_batched(opctx)
+        .await
+        .context("listing sleds")?;
+    let zpool_rows = datastore
+        .zpool_list_all_external_batched(opctx)
+        .await
+        .context("listing zpools")?;
+    let ip_pool_range_rows = {
+        let (authz_service_ip_pool, _) = datastore
+            .ip_pools_service_lookup(opctx)
+            .await
+            .context("fetching IP services pool")?;
+        datastore
+            .ip_pool_list_ranges_batched(opctx, &authz_service_ip_pool)
+            .await
+            .context("listing services IP pool ranges")?
+    };
+
+    let policy = policy_from_db(
+        &sled_rows,
+        &zpool_rows,
+        &ip_pool_range_rows,
+        NEXUS_REDUNDANCY,
+    )
+    .context("assembling policy")?;
+    eprintln!("done.");
+
+    eprint!("loading inventory collections ... ");
+    let collection_ids = datastore
+        .inventory_collections()
+        .await
+        .context("listing collections")?;
+    let collections = futures::stream::iter(collection_ids)
+        .filter_map(|id| async move {
+            let read = datastore
+                .inventory_collection_read(opctx, id)
+                .await
+                .with_context(|| format!("reading collection {}", id));
+            if let Err(error) = &read {
+                eprintln!("warning: {}", error);
+            }
+            read.ok()
+        })
+        .collect::<Vec<Collection>>()
+        .await;
+    eprintln!("done.");
+
+    eprint!("loading blueprints ... ");
+    let limit = fetch_opts.fetch_limit;
+    let pagparams = DataPageParams {
+        marker: None,
+        direction: PaginationOrder::Ascending,
+        limit,
+    };
+    let blueprint_ids = datastore
+        .blueprints_list(opctx, &pagparams)
+        .await
+        .context("listing blueprints")?;
+    check_limit(&blueprint_ids, limit, || "listing blueprint ids");
+    let blueprints = futures::stream::iter(blueprint_ids)
+        .filter_map(|bpm| async move {
+            let blueprint_id = bpm.id;
+            let read = datastore
+                .blueprint_read(
+                    opctx,
+                    &nexus_db_queries::authz::Blueprint::new(
+                        nexus_db_queries::authz::FLEET,
+                        blueprint_id,
+                        LookupType::ById(blueprint_id),
+                    ),
+                )
+                .await
+                .with_context(|| format!("reading blueprint {}", blueprint_id));
+            if let Err(error) = &read {
+                eprintln!("warning: {}", error);
+            }
+            read.ok()
+        })
+        .collect::<Vec<Blueprint>>()
+        .await;
+    eprintln!("done.");
+
+    let state =
+        UnstableReconfiguratorState { policy: policy, collections, blueprints };
+
+    let output_path = &reconfig_save_args.output_file;
+    let file = std::fs::OpenOptions::new()
+        .create_new(true)
+        .write(true)
+        .open(&output_path)
+        .with_context(|| format!("open {:?}", output_path))?;
+    serde_json::to_writer_pretty(&file, &state)
+        .with_context(|| format!("write {:?}", output_path))?;
+    eprintln!("wrote {}", output_path);
+    Ok(())
+}
diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs
index 03d6fd6e80..692931db51 100644
--- a/dev-tools/omdb/src/bin/omdb/nexus.rs
+++ b/dev-tools/omdb/src/bin/omdb/nexus.rs
@@ -869,39 +869,7 @@ async fn cmd_nexus_blueprints_show(
         .blueprint_view(&args.blueprint_id)
         .await
         .with_context(|| format!("fetching blueprint {}", args.blueprint_id))?;
-    println!("blueprint  {}", blueprint.id);
-    println!(
-        "parent:    {}",
-        blueprint
-            .parent_blueprint_id
-            .map(|u| u.to_string())
-            .unwrap_or_else(|| String::from("<none>"))
-    );
-    println!(
-        "created by {}{}",
-        blueprint.creator,
-        if blueprint.creator.parse::<Uuid>().is_ok() {
-            " (likely a Nexus instance)"
-        } else {
-            ""
-        }
-    );
-    println!(
-        "created at {}",
-        humantime::format_rfc3339_millis(blueprint.time_created.into(),)
-    );
-    println!("comment: {}", blueprint.comment);
-    println!("zones:\n");
-    for (sled_id, sled_zones) in &blueprint.omicron_zones {
-        println!(
-            "  sled {}: Omicron zones at generation {}",
-            sled_id, sled_zones.generation
-        );
-        for z in &sled_zones.zones {
-            println!("    {} {}", z.id, z.zone_type.label());
-        }
-    }
-
+    println!("{:?}", blueprint);
     Ok(())
 }
 
diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out
index 3086c98f32..fe590acf55 100644
--- a/dev-tools/omdb/tests/successes.out
+++ b/dev-tools/omdb/tests/successes.out
@@ -58,6 +58,19 @@ stderr:
 note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable
 note: database schema version matches expected (<redacted database version>)
 =============================================
+EXECUTING COMMAND: omdb ["db", "reconfigurator-save", "<REDACTED>"]
+termination: Exited(0)
+---------------------------------------------
+stdout:
+---------------------------------------------
+stderr:
+note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable
+note: database schema version matches expected (<redacted database version>)
+assembling policy ... done.
+loading inventory collections ... done.
+loading blueprints ... done.
+wrote <REDACTED>
+=============================================
 EXECUTING COMMAND: omdb ["db", "services", "list-instances"]
 termination: Exited(0)
 ---------------------------------------------
diff --git a/dev-tools/omdb/tests/test_all_output.rs b/dev-tools/omdb/tests/test_all_output.rs
index dc681712eb..4a9802eee6 100644
--- a/dev-tools/omdb/tests/test_all_output.rs
+++ b/dev-tools/omdb/tests/test_all_output.rs
@@ -9,8 +9,11 @@
 
 use expectorate::assert_contents;
 use nexus_test_utils_macros::nexus_test;
+use nexus_types::deployment::UnstableReconfiguratorState;
 use omicron_test_utils::dev::test_cmds::path_to_executable;
+use omicron_test_utils::dev::test_cmds::redact_variable;
 use omicron_test_utils::dev::test_cmds::run_command;
+use slog_error_chain::InlineErrorChain;
 use std::fmt::Write;
 use std::path::Path;
 use subprocess::Exec;
@@ -53,7 +56,7 @@ async fn test_omdb_usage_errors() {
     ];
 
     for args in invocations {
-        do_run(&mut output, |exec| exec, &cmd_path, args).await;
+        do_run(&mut output, |exec| exec, &cmd_path, args, &[]).await;
     }
 
     assert_contents("tests/usage_errors.out", &output);
@@ -71,13 +74,17 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) {
     let nexus_internal_url =
         format!("http://{}/", cptestctx.internal_client.bind_address);
     let mgs_url = format!("http://{}/", gwtestctx.client.bind_address);
+    let tmpdir = camino_tempfile::tempdir()
+        .expect("failed to create temporary directory");
+    let tmppath = tmpdir.path().join("reconfigurator-save.out");
     let mut output = String::new();
-    let invocations: &[&[&'static str]] = &[
+    let invocations: &[&[&str]] = &[
         &["db", "disks", "list"],
         &["db", "dns", "show"],
         &["db", "dns", "diff", "external", "2"],
         &["db", "dns", "names", "external", "2"],
         &["db", "instances"],
+        &["db", "reconfigurator-save", tmppath.as_str()],
         &["db", "services", "list-instances"],
         &["db", "services", "list-by-sled"],
         &["db", "sleds"],
@@ -103,11 +110,35 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) {
             },
             &cmd_path,
             args,
+            &[tmppath.as_str()],
         )
         .await;
     }
 
     assert_contents("tests/successes.out", &output);
+
+    // The `reconfigurator-save` output is not easy to compare as a string.  But
+    // let's make sure we can at least parse it and that it looks broadly like
+    // what we'd expect.
+    let generated = std::fs::read_to_string(&tmppath).unwrap_or_else(|error| {
+        panic!(
+            "failed to read temporary file containing reconfigurator-save \
+            output: {:?}: {}",
+            tmppath,
+            InlineErrorChain::new(&error),
+        )
+    });
+    let parsed: UnstableReconfiguratorState = serde_json::from_str(&generated)
+        .unwrap_or_else(|error| {
+            panic!(
+                "failed to parse reconfigurator-save output (path {}): {}",
+                tmppath,
+                InlineErrorChain::new(&error),
+            )
+        });
+    assert!(parsed.policy.sleds.len() > 0);
+    assert!(parsed.collections.len() > 0);
+
     gwtestctx.teardown().await;
 }
 
@@ -132,7 +163,7 @@ async fn test_omdb_env_settings(cptestctx: &ControlPlaneTestContext) {
     // Database URL
     // Case 1: specified on the command line
     let args = &["db", "--db-url", &postgres_url, "sleds"];
-    do_run(&mut output, |exec| exec, &cmd_path, args).await;
+    do_run(&mut output, |exec| exec, &cmd_path, args, &[]).await;
 
     // Case 2: specified in multiple places (command-line argument wins)
     let args = &["db", "--db-url", "junk", "sleds"];
@@ -142,6 +173,7 @@ async fn test_omdb_env_settings(cptestctx: &ControlPlaneTestContext) {
         move |exec| exec.env("OMDB_DB_URL", &p),
         &cmd_path,
         args,
+        &[],
     )
     .await;
 
@@ -154,7 +186,7 @@ async fn test_omdb_env_settings(cptestctx: &ControlPlaneTestContext) {
         "background-tasks",
         "doc",
     ];
-    do_run(&mut output, |exec| exec, &cmd_path.clone(), args).await;
+    do_run(&mut output, |exec| exec, &cmd_path.clone(), args, &[]).await;
 
     // Case 2: specified in multiple places (command-line argument wins)
     let args =
@@ -165,6 +197,7 @@ async fn test_omdb_env_settings(cptestctx: &ControlPlaneTestContext) {
         move |exec| exec.env("OMDB_NEXUS_URL", &n),
         &cmd_path,
         args,
+        &[],
     )
     .await;
 
@@ -177,6 +210,7 @@ async fn test_omdb_env_settings(cptestctx: &ControlPlaneTestContext) {
         move |exec| exec.env("OMDB_DNS_SERVER", dns_sockaddr.to_string()),
         &cmd_path,
         args,
+        &[],
     )
     .await;
 
@@ -187,7 +221,7 @@ async fn test_omdb_env_settings(cptestctx: &ControlPlaneTestContext) {
         "background-tasks",
         "doc",
     ];
-    do_run(&mut output, move |exec| exec, &cmd_path, args).await;
+    do_run(&mut output, move |exec| exec, &cmd_path, args, &[]).await;
 
     let args = &["db", "sleds"];
     do_run(
@@ -195,11 +229,12 @@ async fn test_omdb_env_settings(cptestctx: &ControlPlaneTestContext) {
         move |exec| exec.env("OMDB_DNS_SERVER", dns_sockaddr.to_string()),
         &cmd_path,
         args,
+        &[],
     )
     .await;
 
     let args = &["--dns-server", &dns_sockaddr.to_string(), "db", "sleds"];
-    do_run(&mut output, move |exec| exec, &cmd_path, args).await;
+    do_run(&mut output, move |exec| exec, &cmd_path, args, &[]).await;
 
     assert_contents("tests/env.out", &output);
 }
@@ -209,6 +244,7 @@ async fn do_run<F>(
     modexec: F,
     cmd_path: &Path,
     args: &[&str],
+    extra_redactions: &[&str],
 ) where
     F: FnOnce(Exec) -> Exec + Send + 'static,
 {
@@ -217,7 +253,9 @@ async fn do_run<F>(
         output,
         "EXECUTING COMMAND: {} {:?}\n",
         cmd_path.file_name().expect("missing command").to_string_lossy(),
-        args.iter().map(|r| redact_variable(r)).collect::<Vec<_>>(),
+        args.iter()
+            .map(|r| redact_variable(r, extra_redactions))
+            .collect::<Vec<_>>(),
     )
     .unwrap();
 
@@ -249,75 +287,9 @@ async fn do_run<F>(
     write!(output, "termination: {:?}\n", exit_status).unwrap();
     write!(output, "---------------------------------------------\n").unwrap();
     write!(output, "stdout:\n").unwrap();
-    output.push_str(&redact_variable(&stdout_text));
+    output.push_str(&redact_variable(&stdout_text, extra_redactions));
     write!(output, "---------------------------------------------\n").unwrap();
     write!(output, "stderr:\n").unwrap();
-    output.push_str(&redact_variable(&stderr_text));
+    output.push_str(&redact_variable(&stderr_text, extra_redactions));
     write!(output, "=============================================\n").unwrap();
 }
-
-/// Redacts text from stdout/stderr that may change from invocation to invocation
-/// (e.g., assigned TCP port numbers, timestamps)
-///
-/// This allows use to use expectorate to verify the shape of the CLI output.
-fn redact_variable(input: &str) -> String {
-    // Replace TCP port numbers.  We include the localhost characters to avoid
-    // catching any random sequence of numbers.
-    let s = regex::Regex::new(r"\[::1\]:\d{4,5}")
-        .unwrap()
-        .replace_all(input, "[::1]:REDACTED_PORT")
-        .to_string();
-    let s = regex::Regex::new(r"\[::ffff:127.0.0.1\]:\d{4,5}")
-        .unwrap()
-        .replace_all(&s, "[::ffff:127.0.0.1]:REDACTED_PORT")
-        .to_string();
-    let s = regex::Regex::new(r"127\.0\.0\.1:\d{4,5}")
-        .unwrap()
-        .replace_all(&s, "127.0.0.1:REDACTED_PORT")
-        .to_string();
-
-    // Replace uuids.
-    let s = regex::Regex::new(
-        "[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-\
-        [a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}",
-    )
-    .unwrap()
-    .replace_all(&s, "REDACTED_UUID_REDACTED_UUID_REDACTED")
-    .to_string();
-
-    // Replace timestamps.
-    let s = regex::Regex::new(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z")
-        .unwrap()
-        .replace_all(&s, "<REDACTED_TIMESTAMP>")
-        .to_string();
-
-    let s = regex::Regex::new(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z")
-        .unwrap()
-        .replace_all(&s, "<REDACTED     TIMESTAMP>")
-        .to_string();
-
-    // Replace formatted durations.  These are pretty specific to the background
-    // task output.
-    let s = regex::Regex::new(r"\d+s ago")
-        .unwrap()
-        .replace_all(&s, "<REDACTED DURATION>s ago")
-        .to_string();
-
-    let s = regex::Regex::new(r"\d+ms")
-        .unwrap()
-        .replace_all(&s, "<REDACTED DURATION>ms")
-        .to_string();
-
-    let s = regex::Regex::new(
-        r"note: database schema version matches expected \(\d+\.\d+\.\d+\)",
-    )
-    .unwrap()
-    .replace_all(
-        &s,
-        "note: database schema version matches expected \
-        (<redacted database version>)",
-    )
-    .to_string();
-
-    s
-}
diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out
index 2f9001671d..b704982266 100644
--- a/dev-tools/omdb/tests/usage_errors.out
+++ b/dev-tools/omdb/tests/usage_errors.out
@@ -94,17 +94,18 @@ Query the control plane database (CockroachDB)
 Usage: omdb db [OPTIONS] <COMMAND>
 
 Commands:
-  rack       Print information about the rack
-  disks      Print information about disks
-  dns        Print information about internal and external DNS
-  inventory  Print information about collected hardware/software inventory
-  services   Print information about control plane services
-  sleds      Print information about sleds
-  instances  Print information about customer instances
-  network    Print information about the network
-  snapshots  Print information about snapshots
-  validate   Validate the contents of the database
-  help       Print this message or the help of the given subcommand(s)
+  rack                 Print information about the rack
+  disks                Print information about disks
+  dns                  Print information about internal and external DNS
+  inventory            Print information about collected hardware/software inventory
+  reconfigurator-save  Save the current Reconfigurator inputs to a file
+  services             Print information about control plane services
+  sleds                Print information about sleds
+  instances            Print information about customer instances
+  network              Print information about the network
+  snapshots            Print information about snapshots
+  validate             Validate the contents of the database
+  help                 Print this message or the help of the given subcommand(s)
 
 Options:
       --db-url <DB_URL>            URL of the database SQL interface [env: OMDB_DB_URL=]
@@ -123,17 +124,18 @@ Query the control plane database (CockroachDB)
 Usage: omdb db [OPTIONS] <COMMAND>
 
 Commands:
-  rack       Print information about the rack
-  disks      Print information about disks
-  dns        Print information about internal and external DNS
-  inventory  Print information about collected hardware/software inventory
-  services   Print information about control plane services
-  sleds      Print information about sleds
-  instances  Print information about customer instances
-  network    Print information about the network
-  snapshots  Print information about snapshots
-  validate   Validate the contents of the database
-  help       Print this message or the help of the given subcommand(s)
+  rack                 Print information about the rack
+  disks                Print information about disks
+  dns                  Print information about internal and external DNS
+  inventory            Print information about collected hardware/software inventory
+  reconfigurator-save  Save the current Reconfigurator inputs to a file
+  services             Print information about control plane services
+  sleds                Print information about sleds
+  instances            Print information about customer instances
+  network              Print information about the network
+  snapshots            Print information about snapshots
+  validate             Validate the contents of the database
+  help                 Print this message or the help of the given subcommand(s)
 
 Options:
       --db-url <DB_URL>            URL of the database SQL interface [env: OMDB_DB_URL=]
diff --git a/dev-tools/reconfigurator-cli/Cargo.toml b/dev-tools/reconfigurator-cli/Cargo.toml
new file mode 100644
index 0000000000..8a8ea85544
--- /dev/null
+++ b/dev-tools/reconfigurator-cli/Cargo.toml
@@ -0,0 +1,40 @@
+[package]
+name = "reconfigurator-cli"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+
+[build-dependencies]
+omicron-rpaths.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+camino.workspace = true
+clap.workspace = true
+dropshot.workspace = true
+humantime.workspace = true
+indexmap.workspace = true
+nexus-reconfigurator-planning.workspace = true
+nexus-types.workspace = true
+omicron-common.workspace = true
+reedline.workspace = true
+serde_json.workspace = true
+slog-error-chain.workspace = true
+slog.workspace = true
+swrite.workspace = true
+tabled.workspace = true
+uuid.workspace = true
+omicron-workspace-hack.workspace = true
+
+[dev-dependencies]
+camino-tempfile.workspace = true
+expectorate.workspace = true
+omicron-test-utils.workspace = true
+regex.workspace = true
+subprocess.workspace = true
+
+# Disable doc builds by default for our binaries to work around issue
+# rust-lang/cargo#8373.  These docs would not be very useful anyway.
+[[bin]]
+name = "reconfigurator-cli"
+doc = false
diff --git a/dev-tools/reconfigurator-cli/src/main.rs b/dev-tools/reconfigurator-cli/src/main.rs
new file mode 100644
index 0000000000..9cdf09a6f9
--- /dev/null
+++ b/dev-tools/reconfigurator-cli/src/main.rs
@@ -0,0 +1,768 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! developer REPL for driving blueprint planning
+
+use anyhow::{anyhow, bail, Context};
+use camino::Utf8PathBuf;
+use clap::CommandFactory;
+use clap::FromArgMatches;
+use clap::{Args, Parser, Subcommand};
+use indexmap::IndexMap;
+use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder;
+use nexus_reconfigurator_planning::planner::Planner;
+use nexus_reconfigurator_planning::system::{
+    SledBuilder, SledHwInventory, SystemDescription,
+};
+use nexus_types::deployment::{Blueprint, UnstableReconfiguratorState};
+use nexus_types::inventory::Collection;
+use nexus_types::inventory::OmicronZonesConfig;
+use omicron_common::api::external::Generation;
+use reedline::{Reedline, Signal};
+use std::io::BufRead;
+use swrite::{swriteln, SWrite};
+use tabled::Tabled;
+use uuid::Uuid;
+
+/// REPL state
+#[derive(Debug)]
+struct ReconfiguratorSim {
+    /// describes the sleds in the system
+    ///
+    /// This resembles what we get from the `sled` table in a real system.  It
+    /// also contains enough information to generate inventory collections that
+    /// describe the system.
+    system: SystemDescription,
+
+    /// inventory collections created by the user
+    collections: IndexMap<Uuid, Collection>,
+
+    /// blueprints created by the user
+    blueprints: IndexMap<Uuid, Blueprint>,
+
+    log: slog::Logger,
+}
+
+/// interactive REPL for exploring the planner
+#[derive(Parser, Debug)]
+struct CmdReconfiguratorSim {
+    input_file: Option<Utf8PathBuf>,
+}
+
+// REPL implementation
+
+fn main() -> anyhow::Result<()> {
+    let cmd = CmdReconfiguratorSim::parse();
+
+    let log = dropshot::ConfigLogging::StderrTerminal {
+        level: dropshot::ConfigLoggingLevel::Debug,
+    }
+    .to_logger("reconfigurator-sim")
+    .context("creating logger")?;
+
+    let mut sim = ReconfiguratorSim {
+        system: SystemDescription::new(),
+        collections: IndexMap::new(),
+        blueprints: IndexMap::new(),
+        log,
+    };
+
+    if let Some(input_file) = cmd.input_file {
+        let file = std::fs::File::open(&input_file)
+            .with_context(|| format!("open {:?}", &input_file))?;
+        let bufread = std::io::BufReader::new(file);
+        for maybe_buffer in bufread.lines() {
+            let buffer = maybe_buffer
+                .with_context(|| format!("read {:?}", &input_file))?;
+            println!("> {}", buffer);
+            match process_entry(&mut sim, buffer) {
+                LoopResult::Continue => (),
+                LoopResult::Bail(error) => return Err(error),
+            }
+            println!("");
+        }
+    } else {
+        let mut ed = Reedline::create();
+        let prompt = reedline::DefaultPrompt::new(
+            reedline::DefaultPromptSegment::Empty,
+            reedline::DefaultPromptSegment::Empty,
+        );
+        loop {
+            match ed.read_line(&prompt) {
+                Ok(Signal::Success(buffer)) => {
+                    match process_entry(&mut sim, buffer) {
+                        LoopResult::Continue => (),
+                        LoopResult::Bail(error) => return Err(error),
+                    }
+                }
+                Ok(Signal::CtrlD) | Ok(Signal::CtrlC) => break,
+                Err(error) => {
+                    bail!("reconfigurator-cli: unexpected error: {:#}", error);
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
+/// Describes next steps after evaluating one "line" of user input
+///
+/// This could just be `Result`, but it's easy to misuse that here because
+/// _commands_ might fail all the time without needing to bail out of the REPL.
+/// We use a separate type for clarity about what success/failure actually
+/// means.
+enum LoopResult {
+    /// Show the prompt and accept another command
+    Continue,
+
+    /// Exit the REPL with a fatal error
+    Bail(anyhow::Error),
+}
+
+/// Processes one "line" of user input.
+fn process_entry(sim: &mut ReconfiguratorSim, entry: String) -> LoopResult {
+    // If no input was provided, take another lap (print the prompt and accept
+    // another line).  This gets handled specially because otherwise clap would
+    // treat this as a usage error and print a help message, which isn't what we
+    // want here.
+    if entry.trim().is_empty() {
+        return LoopResult::Continue;
+    }
+
+    // Parse the line of input as a REPL command.
+    //
+    // Using `split_whitespace()` like this is going to be a problem if we ever
+    // want to support arguments with whitespace in them (using quotes).  But
+    // it's good enough for now.
+    let parts = entry.split_whitespace();
+    let parsed_command = TopLevelArgs::command()
+        .multicall(true)
+        .try_get_matches_from(parts)
+        .and_then(|matches| TopLevelArgs::from_arg_matches(&matches));
+    let command = match parsed_command {
+        Err(error) => {
+            // We failed to parse the command.  Print the error.
+            return match error.print() {
+                // Assuming that worked, just take another lap.
+                Ok(_) => LoopResult::Continue,
+                // If we failed to even print the error, that itself is a fatal
+                // error.
+                Err(error) => LoopResult::Bail(
+                    anyhow!(error).context("printing previous error"),
+                ),
+            };
+        }
+        Ok(TopLevelArgs { command }) => command,
+    };
+
+    // Dispatch to the command's handler.
+    let cmd_result = match command {
+        Commands::SledList => cmd_sled_list(sim),
+        Commands::SledAdd(args) => cmd_sled_add(sim, args),
+        Commands::SledShow(args) => cmd_sled_show(sim, args),
+        Commands::InventoryList => cmd_inventory_list(sim),
+        Commands::InventoryGenerate => cmd_inventory_generate(sim),
+        Commands::BlueprintList => cmd_blueprint_list(sim),
+        Commands::BlueprintFromInventory(args) => {
+            cmd_blueprint_from_inventory(sim, args)
+        }
+        Commands::BlueprintPlan(args) => cmd_blueprint_plan(sim, args),
+        Commands::BlueprintShow(args) => cmd_blueprint_show(sim, args),
+        Commands::BlueprintDiff(args) => cmd_blueprint_diff(sim, args),
+        Commands::BlueprintDiffInventory(args) => {
+            cmd_blueprint_diff_inventory(sim, args)
+        }
+        Commands::Load(args) => cmd_load(sim, args),
+        Commands::FileContents(args) => cmd_file_contents(args),
+        Commands::Save(args) => cmd_save(sim, args),
+    };
+
+    match cmd_result {
+        Err(error) => println!("error: {:#}", error),
+        Ok(Some(s)) => println!("{}", s),
+        Ok(None) => (),
+    }
+
+    LoopResult::Continue
+}
+
+// clap configuration for the REPL commands
+
+/// reconfigurator-sim: simulate blueprint planning and execution
+#[derive(Debug, Parser)]
+struct TopLevelArgs {
+    #[command(subcommand)]
+    command: Commands,
+}
+
+#[derive(Debug, Subcommand)]
+enum Commands {
+    /// list sleds
+    SledList,
+    /// add a new sled
+    SledAdd(SledAddArgs),
+    /// show details about one sled
+    SledShow(SledArgs),
+
+    /// list all inventory collections
+    InventoryList,
+    /// generates an inventory collection from the configured sleds
+    InventoryGenerate,
+
+    /// list all blueprints
+    BlueprintList,
+    /// generate a blueprint that represents the contents of an inventory
+    BlueprintFromInventory(InventoryArgs),
+    /// run planner to generate a new blueprint
+    BlueprintPlan(BlueprintPlanArgs),
+    /// show details about a blueprint
+    BlueprintShow(BlueprintArgs),
+    /// show differences between two blueprints
+    BlueprintDiff(BlueprintDiffArgs),
+    /// show differences between a blueprint and an inventory collection
+    BlueprintDiffInventory(BlueprintDiffInventoryArgs),
+
+    /// save state to a file
+    Save(SaveArgs),
+    /// load state from a file
+    Load(LoadArgs),
+    /// show information about what's in a saved file
+    FileContents(FileContentsArgs),
+}
+
+#[derive(Debug, Args)]
+struct SledAddArgs {
+    /// id of the new sled
+    sled_id: Option<Uuid>,
+}
+
+#[derive(Debug, Args)]
+struct SledArgs {
+    /// id of the sled
+    sled_id: Uuid,
+}
+
+#[derive(Debug, Args)]
+struct InventoryArgs {
+    /// id of the inventory collection to use in planning
+    collection_id: Uuid,
+}
+
+#[derive(Debug, Args)]
+struct BlueprintPlanArgs {
+    /// id of the blueprint on which this one will be based
+    parent_blueprint_id: Uuid,
+    /// id of the inventory collection to use in planning
+    collection_id: Uuid,
+}
+
+#[derive(Debug, Args)]
+struct BlueprintArgs {
+    /// id of the blueprint
+    blueprint_id: Uuid,
+}
+
+#[derive(Debug, Args)]
+struct BlueprintDiffInventoryArgs {
+    /// id of the inventory collection
+    collection_id: Uuid,
+    /// id of the blueprint
+    blueprint_id: Uuid,
+}
+
+#[derive(Debug, Args)]
+struct BlueprintDiffArgs {
+    /// id of the first blueprint
+    blueprint1_id: Uuid,
+    /// id of the second blueprint
+    blueprint2_id: Uuid,
+}
+
+#[derive(Debug, Args)]
+struct LoadArgs {
+    /// input file
+    filename: Utf8PathBuf,
+
+    /// id of inventory collection to use for sled details
+    /// (may be omitted only if the file contains only one collection)
+    collection_id: Option<Uuid>,
+}
+
+#[derive(Debug, Args)]
+struct FileContentsArgs {
+    /// input file
+    filename: Utf8PathBuf,
+}
+
+#[derive(Debug, Args)]
+struct SaveArgs {
+    /// output file
+    filename: Utf8PathBuf,
+}
+
+// Command handlers
+
+fn cmd_sled_list(
+    sim: &mut ReconfiguratorSim,
+) -> anyhow::Result<Option<String>> {
+    #[derive(Tabled)]
+    #[tabled(rename_all = "SCREAMING_SNAKE_CASE")]
+    struct Sled {
+        id: Uuid,
+        nzpools: usize,
+        subnet: String,
+    }
+
+    let policy = sim.system.to_policy().context("failed to generate policy")?;
+    let rows = policy.sleds.iter().map(|(sled_id, sled_resources)| Sled {
+        id: *sled_id,
+        subnet: sled_resources.subnet.net().to_string(),
+        nzpools: sled_resources.zpools.len(),
+    });
+    let table = tabled::Table::new(rows)
+        .with(tabled::settings::Style::empty())
+        .with(tabled::settings::Padding::new(0, 1, 0, 0))
+        .to_string();
+    Ok(Some(table))
+}
+
+fn cmd_sled_add(
+    sim: &mut ReconfiguratorSim,
+    add: SledAddArgs,
+) -> anyhow::Result<Option<String>> {
+    let mut new_sled = SledBuilder::new();
+    if let Some(sled_id) = add.sled_id {
+        new_sled = new_sled.id(sled_id);
+    }
+
+    let _ = sim.system.sled(new_sled).context("adding sled")?;
+    Ok(Some(String::from("added sled")))
+}
+
+fn cmd_sled_show(
+    sim: &mut ReconfiguratorSim,
+    args: SledArgs,
+) -> anyhow::Result<Option<String>> {
+    let policy = sim.system.to_policy().context("failed to generate policy")?;
+    let sled_id = args.sled_id;
+    let sled_resources = policy
+        .sleds
+        .get(&sled_id)
+        .ok_or_else(|| anyhow!("no sled with id {:?}", sled_id))?;
+    let mut s = String::new();
+    swriteln!(s, "sled {}", sled_id);
+    swriteln!(s, "subnet {}", sled_resources.subnet.net());
+    swriteln!(s, "zpools ({}):", sled_resources.zpools.len());
+    for z in &sled_resources.zpools {
+        swriteln!(s, "    {:?}", z);
+    }
+    Ok(Some(s))
+}
+
+fn cmd_inventory_list(
+    sim: &mut ReconfiguratorSim,
+) -> anyhow::Result<Option<String>> {
+    #[derive(Tabled)]
+    #[tabled(rename_all = "SCREAMING_SNAKE_CASE")]
+    struct InventoryRow {
+        id: Uuid,
+        nerrors: usize,
+        time_done: String,
+    }
+
+    let rows = sim.collections.values().map(|collection| {
+        let id = collection.id;
+        InventoryRow {
+            id,
+            nerrors: collection.errors.len(),
+            time_done: humantime::format_rfc3339_millis(
+                collection.time_done.into(),
+            )
+            .to_string(),
+        }
+    });
+    let table = tabled::Table::new(rows)
+        .with(tabled::settings::Style::empty())
+        .with(tabled::settings::Padding::new(0, 1, 0, 0))
+        .to_string();
+    Ok(Some(table))
+}
+
+fn cmd_inventory_generate(
+    sim: &mut ReconfiguratorSim,
+) -> anyhow::Result<Option<String>> {
+    let mut builder =
+        sim.system.to_collection_builder().context("generating inventory")?;
+    // For an inventory we just generated from thin air, pretend like each sled
+    // has no zones on it.
+    let sled_ids = sim.system.to_policy().unwrap().sleds.into_keys();
+    for sled_id in sled_ids {
+        builder
+            .found_sled_omicron_zones(
+                "fake sled agent",
+                sled_id,
+                OmicronZonesConfig {
+                    generation: Generation::new(),
+                    zones: vec![],
+                },
+            )
+            .context("recording Omicron zones")?;
+    }
+    let inventory = builder.build();
+    let rv = format!(
+        "generated inventory collection {} from configured sleds",
+        inventory.id
+    );
+    sim.collections.insert(inventory.id, inventory);
+    Ok(Some(rv))
+}
+
+fn cmd_blueprint_list(
+    sim: &mut ReconfiguratorSim,
+) -> anyhow::Result<Option<String>> {
+    #[derive(Tabled)]
+    #[tabled(rename_all = "SCREAMING_SNAKE_CASE")]
+    struct BlueprintRow {
+        id: Uuid,
+    }
+
+    let rows = sim
+        .blueprints
+        .values()
+        .map(|blueprint| BlueprintRow { id: blueprint.id });
+    let table = tabled::Table::new(rows)
+        .with(tabled::settings::Style::empty())
+        .with(tabled::settings::Padding::new(0, 1, 0, 0))
+        .to_string();
+    Ok(Some(table))
+}
+
+fn cmd_blueprint_from_inventory(
+    sim: &mut ReconfiguratorSim,
+    args: InventoryArgs,
+) -> anyhow::Result<Option<String>> {
+    let collection_id = args.collection_id;
+    let collection = sim
+        .collections
+        .get(&collection_id)
+        .ok_or_else(|| anyhow!("no such collection: {}", collection_id))?;
+    let dns_version = Generation::new();
+    let policy = sim.system.to_policy().context("generating policy")?;
+    let creator = "reconfigurator-sim";
+    let blueprint = BlueprintBuilder::build_initial_from_collection(
+        collection,
+        dns_version,
+        &policy,
+        creator,
+    )
+    .context("building collection")?;
+    let rv = format!(
+        "generated blueprint {} from inventory collection {}",
+        blueprint.id, collection_id
+    );
+    sim.blueprints.insert(blueprint.id, blueprint);
+    Ok(Some(rv))
+}
+
+fn cmd_blueprint_plan(
+    sim: &mut ReconfiguratorSim,
+    args: BlueprintPlanArgs,
+) -> anyhow::Result<Option<String>> {
+    let parent_blueprint_id = args.parent_blueprint_id;
+    let collection_id = args.collection_id;
+    let parent_blueprint = sim
+        .blueprints
+        .get(&parent_blueprint_id)
+        .ok_or_else(|| anyhow!("no such blueprint: {}", parent_blueprint_id))?;
+    let collection = sim
+        .collections
+        .get(&collection_id)
+        .ok_or_else(|| anyhow!("no such collection: {}", collection_id))?;
+    let dns_version = Generation::new();
+    let policy = sim.system.to_policy().context("generating policy")?;
+    let creator = "reconfigurator-sim";
+    let planner = Planner::new_based_on(
+        sim.log.clone(),
+        parent_blueprint,
+        dns_version,
+        &policy,
+        creator,
+        collection,
+    )
+    .context("creating planner")?;
+    let blueprint = planner.plan().context("generating blueprint")?;
+    let rv = format!(
+        "generated blueprint {} based on parent blueprint {}",
+        blueprint.id, parent_blueprint_id,
+    );
+    sim.blueprints.insert(blueprint.id, blueprint);
+    Ok(Some(rv))
+}
+
+fn cmd_blueprint_show(
+    sim: &mut ReconfiguratorSim,
+    args: BlueprintArgs,
+) -> anyhow::Result<Option<String>> {
+    let blueprint = sim
+        .blueprints
+        .get(&args.blueprint_id)
+        .ok_or_else(|| anyhow!("no such blueprint: {}", args.blueprint_id))?;
+    Ok(Some(format!("{:?}", blueprint)))
+}
+
+fn cmd_blueprint_diff(
+    sim: &mut ReconfiguratorSim,
+    args: BlueprintDiffArgs,
+) -> anyhow::Result<Option<String>> {
+    let blueprint1_id = args.blueprint1_id;
+    let blueprint2_id = args.blueprint2_id;
+    let blueprint1 = sim
+        .blueprints
+        .get(&blueprint1_id)
+        .ok_or_else(|| anyhow!("no such blueprint: {}", blueprint1_id))?;
+    let blueprint2 = sim
+        .blueprints
+        .get(&blueprint2_id)
+        .ok_or_else(|| anyhow!("no such blueprint: {}", blueprint2_id))?;
+
+    let diff = blueprint1.diff_sleds(&blueprint2);
+    Ok(Some(diff.to_string()))
+}
+
+fn cmd_blueprint_diff_inventory(
+    sim: &mut ReconfiguratorSim,
+    args: BlueprintDiffInventoryArgs,
+) -> anyhow::Result<Option<String>> {
+    let collection_id = args.collection_id;
+    let blueprint_id = args.blueprint_id;
+    let collection = sim.collections.get(&collection_id).ok_or_else(|| {
+        anyhow!("no such inventory collection: {}", collection_id)
+    })?;
+    let blueprint = sim
+        .blueprints
+        .get(&blueprint_id)
+        .ok_or_else(|| anyhow!("no such blueprint: {}", blueprint_id))?;
+
+    let zones = collection.all_omicron_zones().map(|z| z.id).collect();
+    let diff = blueprint.diff_sleds_from_collection(&collection, &zones);
+    Ok(Some(diff.to_string()))
+}
+
+fn cmd_save(
+    sim: &mut ReconfiguratorSim,
+    args: SaveArgs,
+) -> anyhow::Result<Option<String>> {
+    let policy = sim.system.to_policy().context("creating policy")?;
+    let saved = UnstableReconfiguratorState {
+        policy,
+        collections: sim.collections.values().cloned().collect(),
+        blueprints: sim.blueprints.values().cloned().collect(),
+    };
+
+    let output_path = &args.filename;
+    let outfile = std::fs::OpenOptions::new()
+        .create_new(true)
+        .write(true)
+        .open(output_path)
+        .with_context(|| format!("open {:?}", output_path))?;
+    serde_json::to_writer_pretty(&outfile, &saved)
+        .with_context(|| format!("writing to {:?}", output_path))
+        .unwrap_or_else(|e| panic!("{:#}", e));
+    Ok(Some(format!(
+        "saved policy, collections, and blueprints to {:?}",
+        output_path
+    )))
+}
+
+fn read_file(
+    input_path: &camino::Utf8Path,
+) -> anyhow::Result<UnstableReconfiguratorState> {
+    let file = std::fs::File::open(input_path)
+        .with_context(|| format!("open {:?}", input_path))?;
+    serde_json::from_reader(file)
+        .with_context(|| format!("read {:?}", input_path))
+}
+
+fn cmd_load(
+    sim: &mut ReconfiguratorSim,
+    args: LoadArgs,
+) -> anyhow::Result<Option<String>> {
+    let input_path = args.filename;
+    let collection_id = args.collection_id;
+    let loaded = read_file(&input_path)?;
+
+    let mut s = String::new();
+
+    let collection_id = match collection_id {
+        Some(s) => s,
+        None => match loaded.collections.len() {
+            1 => loaded.collections[0].id,
+            0 => bail!(
+                "no collection_id specified and file contains 0 collections"
+            ),
+            count => bail!(
+                "no collection_id specified and file contains {} \
+                    collections: {}",
+                count,
+                loaded
+                    .collections
+                    .iter()
+                    .map(|c| c.id.to_string())
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            ),
+        },
+    };
+
+    swriteln!(
+        s,
+        "using collection {} as source of sled inventory data",
+        collection_id
+    );
+    let primary_collection =
+        loaded.collections.iter().find(|c| c.id == collection_id).ok_or_else(
+            || {
+                anyhow!(
+                    "collection {} not found in file {:?}",
+                    collection_id,
+                    input_path
+                )
+            },
+        )?;
+
+    let current_policy = sim.system.to_policy().context("generating policy")?;
+    for (sled_id, sled_resources) in loaded.policy.sleds {
+        if current_policy.sleds.contains_key(&sled_id) {
+            swriteln!(
+                s,
+                "sled {}: skipped (one with \
+                the same id is already loaded)",
+                sled_id
+            );
+            continue;
+        }
+
+        let Some(inventory_sled_agent) =
+            primary_collection.sled_agents.get(&sled_id)
+        else {
+            swriteln!(
+                s,
+                "error: load sled {}: no inventory found for sled agent in \
+                collection {}",
+                sled_id,
+                collection_id
+            );
+            continue;
+        };
+
+        let inventory_sp = match &inventory_sled_agent.baseboard_id {
+            Some(baseboard_id) => {
+                let inv_sp = primary_collection
+                    .sps
+                    .get(baseboard_id)
+                    .ok_or_else(|| {
+                        anyhow!(
+                            "error: load sled {}: missing SP inventory",
+                            sled_id
+                        )
+                    })?;
+                let inv_rot = primary_collection
+                    .rots
+                    .get(baseboard_id)
+                    .ok_or_else(|| {
+                        anyhow!(
+                            "error: load sled {}: missing RoT inventory",
+                            sled_id
+                        )
+                    })?;
+                Some(SledHwInventory { baseboard_id, sp: inv_sp, rot: inv_rot })
+            }
+            None => None,
+        };
+
+        let result = sim.system.sled_full(
+            sled_id,
+            sled_resources,
+            inventory_sp,
+            inventory_sled_agent,
+        );
+
+        match result {
+            Ok(_) => swriteln!(s, "sled {} loaded", sled_id),
+            Err(error) => {
+                swriteln!(s, "error: load sled {}: {:#}", sled_id, error)
+            }
+        };
+    }
+
+    for collection in loaded.collections {
+        if sim.collections.contains_key(&collection.id) {
+            swriteln!(
+                s,
+                "collection {}: skipped (one with the \
+                same id is already loaded)",
+                collection.id
+            );
+        } else {
+            swriteln!(s, "collection {} loaded", collection.id);
+            sim.collections.insert(collection.id, collection);
+        }
+    }
+
+    for blueprint in loaded.blueprints {
+        if sim.blueprints.contains_key(&blueprint.id) {
+            swriteln!(
+                s,
+                "blueprint {}: skipped (one with the \
+                same id is already loaded)",
+                blueprint.id
+            );
+        } else {
+            swriteln!(s, "blueprint {} loaded", blueprint.id);
+            sim.blueprints.insert(blueprint.id, blueprint);
+        }
+    }
+
+    swriteln!(s, "loaded data from {:?}", input_path);
+    Ok(Some(s))
+}
+
+fn cmd_file_contents(args: FileContentsArgs) -> anyhow::Result<Option<String>> {
+    let loaded = read_file(&args.filename)?;
+
+    let mut s = String::new();
+
+    for (sled_id, sled_resources) in loaded.policy.sleds {
+        swriteln!(
+            s,
+            "sled: {} (subnet: {}, zpools: {})",
+            sled_id,
+            sled_resources.subnet.net(),
+            sled_resources.zpools.len()
+        );
+    }
+
+    for collection in loaded.collections {
+        swriteln!(
+            s,
+            "collection: {} (errors: {}, completed at: {})",
+            collection.id,
+            collection.errors.len(),
+            humantime::format_rfc3339_millis(collection.time_done.into())
+                .to_string(),
+        );
+    }
+
+    for blueprint in loaded.blueprints {
+        swriteln!(
+            s,
+            "blueprint:  {} (created at: {})",
+            blueprint.id,
+            blueprint.time_created
+        );
+    }
+
+    Ok(Some(s))
+}
diff --git a/dev-tools/reconfigurator-cli/tests/input/cmds.txt b/dev-tools/reconfigurator-cli/tests/input/cmds.txt
new file mode 100644
index 0000000000..3997812e55
--- /dev/null
+++ b/dev-tools/reconfigurator-cli/tests/input/cmds.txt
@@ -0,0 +1,14 @@
+sled-list
+inventory-list
+blueprint-list
+
+sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a
+sled-add dde1c0e2-b10d-4621-b420-f179f7a7a00a
+sled-list
+sled-show dde1c0e2-b10d-4621-b420-f179f7a7a00a
+sled-add 90c1102a-b9f5-4d88-92a2-60d54a2d98cc
+sled-add 04ef3330-c682-4a08-8def-fcc4bef31bcd
+sled-list
+
+inventory-generate
+inventory-list
diff --git a/dev-tools/reconfigurator-cli/tests/input/complex.json b/dev-tools/reconfigurator-cli/tests/input/complex.json
new file mode 100644
index 0000000000..c168e153c1
--- /dev/null
+++ b/dev-tools/reconfigurator-cli/tests/input/complex.json
@@ -0,0 +1,11558 @@
+{
+  "policy": {
+    "sleds": {
+      "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+        "policy": {
+          "kind": "in_service",
+          "provision_policy": "provisionable"
+        },
+        "state": "active",
+        "zpools": [
+          "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0",
+          "oxp_1d864940-c723-4f58-82e8-b03772b42356",
+          "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698",
+          "oxp_466351db-2ced-45fa-9431-22de1c4bd480",
+          "oxp_88d39560-b091-4fad-bad0-b0b1c514033b",
+          "oxp_88ddd98f-3181-4797-864a-7e8d5b028657",
+          "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d",
+          "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c",
+          "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+        ],
+        "subnet": {
+          "net": "fd00:1122:3344:101::/64"
+        }
+      },
+      "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+        "policy": {
+          "kind": "in_service",
+          "provision_policy": "provisionable"
+        },
+        "state": "active",
+        "zpools": [
+          "oxp_0622b2cf-297e-4356-b94c-d74309f443ba",
+          "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37",
+          "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1",
+          "oxp_574728ca-1559-48f3-a46c-1e608966e6a0",
+          "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945",
+          "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c",
+          "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3",
+          "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
+        ],
+        "subnet": {
+          "net": "fd00:1122:3344:121::/64"
+        }
+      },
+      "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+        "policy": {
+          "kind": "in_service",
+          "provision_policy": "provisionable"
+        },
+        "state": "active",
+        "zpools": [
+          "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd",
+          "oxp_0c07bcc1-7666-4754-881d-4135992ff561",
+          "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9",
+          "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae",
+          "oxp_6a275151-d85f-488b-81a6-e558fdede658",
+          "oxp_742699b5-4ded-406d-9a08-24648d3b2efb",
+          "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9",
+          "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42",
+          "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c",
+          "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
+        ],
+        "subnet": {
+          "net": "fd00:1122:3344:102::/64"
+        }
+      },
+      "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+        "policy": {
+          "kind": "in_service",
+          "provision_policy": "provisionable"
+        },
+        "state": "active",
+        "zpools": [
+          "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98",
+          "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960",
+          "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea",
+          "oxp_6aa59516-2018-4494-af6b-46704b20f6dc",
+          "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3",
+          "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3",
+          "oxp_c8feed09-6556-4409-af8e-563c97d556eb",
+          "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a",
+          "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62",
+          "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+        ],
+        "subnet": {
+          "net": "fd00:1122:3344:103::/64"
+        }
+      }
+    },
+    "service_ip_pool_ranges": [
+      {
+        "first": "172.20.28.1",
+        "last": "172.20.28.10"
+      }
+    ],
+    "target_nexus_zone_count": 3
+  },
+  "collections": [
+    {
+      "id": "83ed3949-d221-44f3-a901-02e9ff16d2a5",
+      "errors": [
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d7328126-995e-437c-b92e-c6c220d508b2\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"d7328126-995e-437c-b92e-c6c220d508b2\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a94be62-a0b1-4c63-b54c-7083266de0ab\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"6a94be62-a0b1-4c63-b54c-7083266de0ab\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"858a0aad-c3df-4cdf-9c20-be6b1effdb20\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"858a0aad-c3df-4cdf-9c20-be6b1effdb20\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"40a5f1ae-5e03-41ec-8c45-6b8a83ce7ad4\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"40a5f1ae-5e03-41ec-8c45-6b8a83ce7ad4\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fe9383e3-5b7c-45f2-bec6-c1f5df8caaa4\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"fe9383e3-5b7c-45f2-bec6-c1f5df8caaa4\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9c467503-9155-4d79-b048-aea0c4153f9e\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"9c467503-9155-4d79-b048-aea0c4153f9e\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d397fe11-ec68-4a1f-9739-e42313158ae4\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"d397fe11-ec68-4a1f-9739-e42313158ae4\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"db83ac82-ec73-4914-adb7-6a243409702c\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"db83ac82-ec73-4914-adb7-6a243409702c\" }"
+      ],
+      "time_started": "2024-03-01T19:17:06.797656Z",
+      "time_done": "2024-03-01T19:17:10.594988Z",
+      "collector": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+      "baseboards": [
+        {
+          "part_number": "913-0000006",
+          "serial_number": "BRM44220005"
+        },
+        {
+          "part_number": "913-0000006",
+          "serial_number": "BRM44220011"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220004"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220046"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220081"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM44220001"
+        }
+      ],
+      "cabooses": [
+        {
+          "board": "gimlet-c",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "gimlet-c",
+          "version": "1.0.6"
+        },
+        {
+          "board": "gimlet-c",
+          "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
+          "name": "gimlet-c",
+          "version": "1.0.5"
+        },
+        {
+          "board": "gimlet-c",
+          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+          "name": "gimlet-c",
+          "version": "1.0.8"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "oxide-rot-1",
+          "version": "1.0.5"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
+          "name": "oxide-rot-1",
+          "version": "1.0.4"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
+          "name": "oxide-rot-1",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+          "name": "oxide-rot-1",
+          "version": "1.0.6"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+          "name": "oxide-rot-1",
+          "version": "1.0.0"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+          "name": "oxide-rot-1-dev",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "sidecar-b",
+          "version": "1.0.6"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
+          "name": "sidecar-b",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
+          "name": "sidecar-b",
+          "version": "0.0.0-git"
+        }
+      ],
+      "rot_pages": [
+        {
+          "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+        },
+        {
+          "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+        },
+        {
+          "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+        },
+        {
+          "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+        },
+        {
+          "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+        },
+        {
+          "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+        },
+        {
+          "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+        }
+      ],
+      "sps": [
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220005"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:09.327993Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "switch",
+            "sp_slot": 0,
+            "baseboard_revision": 4,
+            "hubris_archive": "9184be93590e655e",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220011"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:09.913395Z",
+            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "sp_type": "switch",
+            "sp_slot": 1,
+            "baseboard_revision": 4,
+            "hubris_archive": "9184be93590e655e",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:08.168996Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 17,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:08.746433Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 15,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:07.575660Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 14,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:06.810258Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 16,
+            "baseboard_revision": 6,
+            "hubris_archive": "3c670b44fdeef4af",
+            "power_state": "A0"
+          }
+        ]
+      ],
+      "rots": [
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220005"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:09.327993Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "4599e4d7bc9b2164e8c0f0f7ce4bcae2bd35d635c8450854ea9a94ce49a2ca0f",
+            "slot_b_sha3_256_digest": "1e42a0761e5bd1d9f9a5e2c44acc34e7b7e007792977cdb0523d271db2e80c8c"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220011"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:09.913395Z",
+            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "active_slot": {
+              "slot": "b"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "92e07077972c83af6872ec817fb88d74a36f6956276505e9771302c5bb7ec2a8",
+            "slot_b_sha3_256_digest": "fb25825eaf97f8f2307afe3caebf0b70865499f6d116db3aff382c505b4d8dd7"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:08.168996Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
+            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:08.746433Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
+            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:07.575660Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "b"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
+            "slot_b_sha3_256_digest": "d416d5ba9d0d3bd495f0f5fcd2aecc28cd83b53d1a8f94a28241d85b30761451"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:06.810258Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
+            "slot_b_sha3_256_digest": "4ec328195a23e86a1545d5ee576e73834aae81fb79830a54d0503475875f1760"
+          }
+        ]
+      ],
+      "cabooses_found": {
+        "SpSlot0": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.329382Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "sidecar-b",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.915072Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "sidecar-b",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.170364Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.751437Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.577731Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:06.811674Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ]
+        ],
+        "SpSlot1": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.330756Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
+                "name": "sidecar-b",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.916864Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
+                "name": "sidecar-b",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.172087Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.752817Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.579882Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:06.813085Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
+                "name": "gimlet-c",
+                "version": "1.0.5"
+              }
+            }
+          ]
+        ],
+        "RotSlotA": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.537712Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+                "name": "oxide-rot-1-dev",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:10.123027Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
+                "name": "oxide-rot-1",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.423630Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.004153Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.831036Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.074158Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ]
+        ],
+        "RotSlotB": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.744584Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:10.331006Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+                "name": "oxide-rot-1",
+                "version": "1.0.0"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.675629Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.256102Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.090096Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.327673Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
+                "name": "oxide-rot-1",
+                "version": "1.0.4"
+              }
+            }
+          ]
+        ]
+      },
+      "rot_pages_found": {
+        "Cmpa": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.691258Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.272635Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.107097Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.344150Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ]
+        ],
+        "CfpaActive": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.707253Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.288646Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.123462Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.416190Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ],
+        "CfpaInactive": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.723232Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.304669Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.139558Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.487114Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ],
+        "CfpaScratch": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.739167Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.320638Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.162584Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.568207Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ]
+      },
+      "sled_agents": {
+        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+          "time_collected": "2024-03-01T19:17:10.485528Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+          "time_collected": "2024-03-01T19:17:10.522140Z",
+          "source": "http://[fd00:1122:3344:121::1]:12345",
+          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          "sled_agent_address": "[fd00:1122:3344:121::1]:12345",
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+          "time_collected": "2024-03-01T19:17:10.557759Z",
+          "source": "http://[fd00:1122:3344:102::1]:12345",
+          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+          "time_collected": "2024-03-01T19:17:10.593424Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        }
+      },
+      "omicron_zones": {
+        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+          "time_collected": "2024-03-01T19:17:10.486656Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "0706860a-3801-42a7-8000-fff741f7665b",
+                "underlay_address": "fd00:1122:3344:101::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
+                  }
+                }
+              },
+              {
+                "id": "34a279a2-a025-41ff-a392-1addac400f38",
+                "underlay_address": "fd00:1122:3344:101::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  }
+                }
+              },
+              {
+                "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
+                "underlay_address": "fd00:1122:3344:1::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  },
+                  "dns_address": "[fd00:1122:3344:1::1]:53",
+                  "gz_address": "fd00:1122:3344:1::2",
+                  "gz_address_index": 0,
+                  "http_address": "[fd00:1122:3344:1::1]:5353"
+                }
+              },
+              {
+                "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+                "underlay_address": "fd00:1122:3344:101::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.2",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:101::5]:12221",
+                  "nic": {
+                    "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
+                    "ip": "172.30.2.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
+                    },
+                    "mac": "A8:40:25:FF:E1:17",
+                    "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
+                "underlay_address": "fd00:1122:3344:101::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+                  }
+                }
+              },
+              {
+                "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
+                "underlay_address": "fd00:1122:3344:101::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
+                  }
+                }
+              },
+              {
+                "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
+                "underlay_address": "fd00:1122:3344:101::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                  }
+                }
+              },
+              {
+                "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
+                "underlay_address": "fd00:1122:3344:101::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  }
+                }
+              },
+              {
+                "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
+                "underlay_address": "fd00:1122:3344:101::4",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::4]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                  }
+                }
+              },
+              {
+                "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
+                "underlay_address": "fd00:1122:3344:101::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
+                  }
+                }
+              },
+              {
+                "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
+                "underlay_address": "fd00:1122:3344:101::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
+                  }
+                }
+              },
+              {
+                "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
+                "underlay_address": "fd00:1122:3344:101::6",
+                "zone_type": {
+                  "type": "oximeter",
+                  "address": "[fd00:1122:3344:101::6]:12223"
+                }
+              },
+              {
+                "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                "underlay_address": "fd00:1122:3344:101::11",
+                "zone_type": {
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:101::11]:123",
+                  "dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "nic": {
+                    "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
+                    "ip": "172.30.3.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
+                    },
+                    "mac": "A8:40:25:FF:E7:A8",
+                    "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100
+                  },
+                  "ntp_servers": [
+                    "ntp.eng.oxide.computer"
+                  ],
+                  "snat_cfg": {
+                    "ip": "172.20.28.5",
+                    "first_port": 0,
+                    "last_port": 16383
+                  }
+                }
+              },
+              {
+                "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
+                "underlay_address": "fd00:1122:3344:101::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                  }
+                }
+              },
+              {
+                "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
+                "underlay_address": "fd00:1122:3344:101::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
+                  }
+                }
+              },
+              {
+                "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
+                "underlay_address": "fd00:1122:3344:101::7",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:101::7]:17000"
+                }
+              }
+            ]
+          }
+        },
+        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+          "time_collected": "2024-03-01T19:17:10.522942Z",
+          "source": "http://[fd00:1122:3344:121::1]:12345",
+          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
+          "zones": {
+            "generation": 3,
+            "zones": [
+              {
+                "id": "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
+                "underlay_address": "fd00:1122:3344:121::28",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::28]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3"
+                  }
+                }
+              },
+              {
+                "id": "12431d3e-1516-4822-98b3-3711ff912e69",
+                "underlay_address": "fd00:1122:3344:121::22",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::22]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_0622b2cf-297e-4356-b94c-d74309f443ba"
+                  }
+                }
+              },
+              {
+                "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
+                "underlay_address": "fd00:1122:3344:121::21",
+                "zone_type": {
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:121::21]:123",
+                  "dns_servers": [
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
+                  ],
+                  "ntp_servers": [
+                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                  ]
+                }
+              },
+              {
+                "id": "56414d76-9b8c-4e15-acf1-cd269c2d8638",
+                "underlay_address": "fd00:1122:3344:121::25",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::25]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_574728ca-1559-48f3-a46c-1e608966e6a0"
+                  }
+                }
+              },
+              {
+                "id": "891cb9da-301d-40cc-b93d-5fcb350fa968",
+                "underlay_address": "fd00:1122:3344:121::29",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::29]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
+                  }
+                }
+              },
+              {
+                "id": "d9014907-24c0-4c33-a475-db15942f4f8f",
+                "underlay_address": "fd00:1122:3344:121::23",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::23]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37"
+                  }
+                }
+              },
+              {
+                "id": "e950de25-7306-4cf7-a6f4-6db11adaa310",
+                "underlay_address": "fd00:1122:3344:121::27",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::27]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c"
+                  }
+                }
+              },
+              {
+                "id": "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
+                "underlay_address": "fd00:1122:3344:121::26",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::26]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945"
+                  }
+                }
+              },
+              {
+                "id": "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00",
+                "underlay_address": "fd00:1122:3344:121::24",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::24]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1"
+                  }
+                }
+              }
+            ]
+          }
+        },
+        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+          "time_collected": "2024-03-01T19:17:10.558583Z",
+          "source": "http://[fd00:1122:3344:102::1]:12345",
+          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
+                "underlay_address": "fd00:1122:3344:102::11",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::11]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                  }
+                }
+              },
+              {
+                "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
+                "underlay_address": "fd00:1122:3344:102::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
+                  }
+                }
+              },
+              {
+                "id": "479811e7-6409-45d4-b294-5333534e47ed",
+                "underlay_address": "fd00:1122:3344:102::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                  }
+                }
+              },
+              {
+                "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
+                "underlay_address": "fd00:1122:3344:102::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                  }
+                }
+              },
+              {
+                "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
+                "underlay_address": "fd00:1122:3344:2::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  },
+                  "dns_address": "[fd00:1122:3344:2::1]:53",
+                  "gz_address": "fd00:1122:3344:2::2",
+                  "gz_address_index": 1,
+                  "http_address": "[fd00:1122:3344:2::1]:5353"
+                }
+              },
+              {
+                "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+                "underlay_address": "fd00:1122:3344:102::7",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:102::7]:17000"
+                }
+              },
+              {
+                "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
+                "underlay_address": "fd00:1122:3344:102::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "658076ed-fb43-49e7-a813-df3d23497848",
+                "underlay_address": "fd00:1122:3344:102::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
+                  }
+                }
+              },
+              {
+                "id": "6edb836e-b100-4080-bb03-738d6441d571",
+                "underlay_address": "fd00:1122:3344:102::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
+                  }
+                }
+              },
+              {
+                "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                "underlay_address": "fd00:1122:3344:102::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.3",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:102::5]:12221",
+                  "nic": {
+                    "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
+                    "ip": "172.30.2.6",
+                    "kind": {
+                      "type": "service",
+                      "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
+                    },
+                    "mac": "A8:40:25:FF:B2:AF",
+                    "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
+                "underlay_address": "fd00:1122:3344:102::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                  }
+                }
+              },
+              {
+                "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+                "underlay_address": "fd00:1122:3344:102::4",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::4]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                  }
+                }
+              },
+              {
+                "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
+                "underlay_address": "fd00:1122:3344:102::6",
+                "zone_type": {
+                  "type": "clickhouse",
+                  "address": "[fd00:1122:3344:102::6]:8123",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
+                "underlay_address": "fd00:1122:3344:102::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
+                  }
+                }
+              },
+              {
+                "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
+                "underlay_address": "fd00:1122:3344:102::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                "underlay_address": "fd00:1122:3344:102::12",
+                "zone_type": {
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:102::12]:123",
+                  "dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "nic": {
+                    "id": "9f03c82c-3192-498d-978d-a175049ac389",
+                    "ip": "172.30.3.6",
+                    "kind": {
+                      "type": "service",
+                      "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
+                    },
+                    "mac": "A8:40:25:FF:E4:11",
+                    "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100
+                  },
+                  "ntp_servers": [
+                    "ntp.eng.oxide.computer"
+                  ],
+                  "snat_cfg": {
+                    "ip": "172.20.28.6",
+                    "first_port": 16384,
+                    "last_port": 32767
+                  }
+                }
+              },
+              {
+                "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
+                "underlay_address": "fd00:1122:3344:102::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
+                  }
+                }
+              }
+            ]
+          }
+        },
+        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+          "time_collected": "2024-03-01T19:17:10.594368Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
+                "underlay_address": "fd00:1122:3344:103::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
+                  }
+                }
+              },
+              {
+                "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
+                "underlay_address": "fd00:1122:3344:103::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
+                  }
+                }
+              },
+              {
+                "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
+                "underlay_address": "fd00:1122:3344:3::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  },
+                  "dns_address": "[fd00:1122:3344:3::1]:53",
+                  "gz_address": "fd00:1122:3344:3::2",
+                  "gz_address_index": 2,
+                  "http_address": "[fd00:1122:3344:3::1]:5353"
+                }
+              },
+              {
+                "id": "4b74c418-0a08-4489-b652-e97267c1c220",
+                "underlay_address": "fd00:1122:3344:103::7",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::7]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  }
+                }
+              },
+              {
+                "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+                "underlay_address": "fd00:1122:3344:103::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.4",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:103::5]:12221",
+                  "nic": {
+                    "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
+                    "ip": "172.30.2.7",
+                    "kind": {
+                      "type": "service",
+                      "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
+                    },
+                    "mac": "A8:40:25:FF:96:05",
+                    "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
+                "underlay_address": "fd00:1122:3344:103::6",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:103::6]:17000"
+                }
+              },
+              {
+                "id": "921958c0-7761-4294-945d-2a078327bf2c",
+                "underlay_address": "fd00:1122:3344:103::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
+                  }
+                }
+              },
+              {
+                "id": "9df22034-59f6-431f-82e7-258fe983879b",
+                "underlay_address": "fd00:1122:3344:103::4",
+                "zone_type": {
+                  "type": "external_dns",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  },
+                  "dns_address": "172.20.28.1:53",
+                  "http_address": "[fd00:1122:3344:103::4]:5353",
+                  "nic": {
+                    "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
+                    "ip": "172.30.1.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "9df22034-59f6-431f-82e7-258fe983879b"
+                    },
+                    "mac": "A8:40:25:FF:90:12",
+                    "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.1.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
+                "underlay_address": "fd00:1122:3344:103::11",
+                "zone_type": {
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:103::11]:123",
+                  "dns_servers": [
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
+                  ],
+                  "ntp_servers": [
+                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                  ]
+                }
+              },
+              {
+                "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
+                "underlay_address": "fd00:1122:3344:103::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
+                  }
+                }
+              },
+              {
+                "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
+                "underlay_address": "fd00:1122:3344:103::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
+                  }
+                }
+              },
+              {
+                "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
+                "underlay_address": "fd00:1122:3344:103::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
+                  }
+                }
+              },
+              {
+                "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
+                "underlay_address": "fd00:1122:3344:103::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
+                  }
+                }
+              },
+              {
+                "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
+                "underlay_address": "fd00:1122:3344:103::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:103::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  }
+                }
+              },
+              {
+                "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
+                "underlay_address": "fd00:1122:3344:103::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
+                  }
+                }
+              },
+              {
+                "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
+                "underlay_address": "fd00:1122:3344:103::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                  }
+                }
+              }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "id": "0a8dc8ff-9013-43aa-9f06-9540b24ea902",
+      "errors": [
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bb340ba7-cf9e-48c9-9e87-4b26fcb9b2aa\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"bb340ba7-cf9e-48c9-9e87-4b26fcb9b2aa\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"86914fc3-f142-4dbf-8ac5-a94c03c38a93\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"86914fc3-f142-4dbf-8ac5-a94c03c38a93\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e29b10e1-aea9-48a9-8054-951715c06bdb\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"e29b10e1-aea9-48a9-8054-951715c06bdb\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"719a3865-65bc-4228-810f-20b93a5199d0\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:10 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"719a3865-65bc-4228-810f-20b93a5199d0\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"70ee4f97-6ffe-46fe-810d-6d0f602e8749\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:10 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"70ee4f97-6ffe-46fe-810d-6d0f602e8749\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6d81dff5-4390-43dc-a976-489980a574f3\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:10 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"6d81dff5-4390-43dc-a976-489980a574f3\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f83589b8-03d7-47f6-8e49-4090810c6485\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:10 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"f83589b8-03d7-47f6-8e49-4090810c6485\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"73e37767-229b-48ca-9fcd-d98988c1a131\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:10 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"73e37767-229b-48ca-9fcd-d98988c1a131\" }"
+      ],
+      "time_started": "2024-03-01T19:17:06.813002Z",
+      "time_done": "2024-03-01T19:17:11.153859Z",
+      "collector": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+      "baseboards": [
+        {
+          "part_number": "913-0000006",
+          "serial_number": "BRM44220005"
+        },
+        {
+          "part_number": "913-0000006",
+          "serial_number": "BRM44220011"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220004"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220046"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220081"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM44220001"
+        }
+      ],
+      "cabooses": [
+        {
+          "board": "gimlet-c",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "gimlet-c",
+          "version": "1.0.6"
+        },
+        {
+          "board": "gimlet-c",
+          "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
+          "name": "gimlet-c",
+          "version": "1.0.5"
+        },
+        {
+          "board": "gimlet-c",
+          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+          "name": "gimlet-c",
+          "version": "1.0.8"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "oxide-rot-1",
+          "version": "1.0.5"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
+          "name": "oxide-rot-1",
+          "version": "1.0.4"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
+          "name": "oxide-rot-1",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+          "name": "oxide-rot-1",
+          "version": "1.0.6"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+          "name": "oxide-rot-1",
+          "version": "1.0.0"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+          "name": "oxide-rot-1-dev",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "sidecar-b",
+          "version": "1.0.6"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
+          "name": "sidecar-b",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
+          "name": "sidecar-b",
+          "version": "0.0.0-git"
+        }
+      ],
+      "rot_pages": [
+        {
+          "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+        },
+        {
+          "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+        },
+        {
+          "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+        },
+        {
+          "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+        },
+        {
+          "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+        },
+        {
+          "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+        },
+        {
+          "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+        }
+      ],
+      "sps": [
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220005"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:09.906135Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "switch",
+            "sp_slot": 0,
+            "baseboard_revision": 4,
+            "hubris_archive": "9184be93590e655e",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220011"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:10.475436Z",
+            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "sp_type": "switch",
+            "sp_slot": 1,
+            "baseboard_revision": 4,
+            "hubris_archive": "9184be93590e655e",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:08.746072Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 17,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:09.327592Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 15,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:08.014626Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 14,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:06.876242Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 16,
+            "baseboard_revision": 6,
+            "hubris_archive": "3c670b44fdeef4af",
+            "power_state": "A0"
+          }
+        ]
+      ],
+      "rots": [
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220005"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:09.906135Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "4599e4d7bc9b2164e8c0f0f7ce4bcae2bd35d635c8450854ea9a94ce49a2ca0f",
+            "slot_b_sha3_256_digest": "1e42a0761e5bd1d9f9a5e2c44acc34e7b7e007792977cdb0523d271db2e80c8c"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220011"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:10.475436Z",
+            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "active_slot": {
+              "slot": "b"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "92e07077972c83af6872ec817fb88d74a36f6956276505e9771302c5bb7ec2a8",
+            "slot_b_sha3_256_digest": "fb25825eaf97f8f2307afe3caebf0b70865499f6d116db3aff382c505b4d8dd7"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:08.746072Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
+            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:09.327592Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
+            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:08.014626Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "b"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
+            "slot_b_sha3_256_digest": "d416d5ba9d0d3bd495f0f5fcd2aecc28cd83b53d1a8f94a28241d85b30761451"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          {
+            "time_collected": "2024-03-01T19:17:06.876242Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
+            "slot_b_sha3_256_digest": "4ec328195a23e86a1545d5ee576e73834aae81fb79830a54d0503475875f1760"
+          }
+        ]
+      ],
+      "cabooses_found": {
+        "SpSlot0": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.907517Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "sidecar-b",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:10.476830Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "sidecar-b",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.751041Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.329018Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.123957Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.130790Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ]
+        ],
+        "SpSlot1": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.909025Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
+                "name": "sidecar-b",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:10.478263Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
+                "name": "sidecar-b",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.752437Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.330438Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.163634Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.328321Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
+                "name": "gimlet-c",
+                "version": "1.0.5"
+              }
+            }
+          ]
+        ],
+        "RotSlotA": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:10.115796Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+                "name": "oxide-rot-1-dev",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:10.685118Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
+                "name": "oxide-rot-1",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.003752Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.582176Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.415126Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.643681Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ]
+        ],
+        "RotSlotB": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:10.322717Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:10.892030Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+                "name": "oxide-rot-1",
+                "version": "1.0.0"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.255745Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.834209Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.667106Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.894743Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
+                "name": "oxide-rot-1",
+                "version": "1.0.4"
+              }
+            }
+          ]
+        ]
+      },
+      "rot_pages_found": {
+        "Cmpa": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.272691Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.850755Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.682710Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.911203Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ]
+        ],
+        "CfpaActive": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.288403Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.866661Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.699070Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.927224Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ],
+        "CfpaInactive": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.304293Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.882683Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.714707Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.943244Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ],
+        "CfpaScratch": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.320329Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:09.898662Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:08.730680Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:17:07.959254Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ]
+      },
+      "sled_agents": {
+        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+          "time_collected": "2024-03-01T19:17:11.045570Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+          "time_collected": "2024-03-01T19:17:11.081825Z",
+          "source": "http://[fd00:1122:3344:121::1]:12345",
+          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          "sled_agent_address": "[fd00:1122:3344:121::1]:12345",
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+          "time_collected": "2024-03-01T19:17:11.117162Z",
+          "source": "http://[fd00:1122:3344:102::1]:12345",
+          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+          "time_collected": "2024-03-01T19:17:11.152394Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        }
+      },
+      "omicron_zones": {
+        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+          "time_collected": "2024-03-01T19:17:11.046682Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "0706860a-3801-42a7-8000-fff741f7665b",
+                "underlay_address": "fd00:1122:3344:101::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
+                  }
+                }
+              },
+              {
+                "id": "34a279a2-a025-41ff-a392-1addac400f38",
+                "underlay_address": "fd00:1122:3344:101::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  }
+                }
+              },
+              {
+                "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
+                "underlay_address": "fd00:1122:3344:1::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  },
+                  "dns_address": "[fd00:1122:3344:1::1]:53",
+                  "gz_address": "fd00:1122:3344:1::2",
+                  "gz_address_index": 0,
+                  "http_address": "[fd00:1122:3344:1::1]:5353"
+                }
+              },
+              {
+                "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+                "underlay_address": "fd00:1122:3344:101::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.2",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:101::5]:12221",
+                  "nic": {
+                    "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
+                    "ip": "172.30.2.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
+                    },
+                    "mac": "A8:40:25:FF:E1:17",
+                    "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
+                "underlay_address": "fd00:1122:3344:101::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+                  }
+                }
+              },
+              {
+                "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
+                "underlay_address": "fd00:1122:3344:101::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
+                  }
+                }
+              },
+              {
+                "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
+                "underlay_address": "fd00:1122:3344:101::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                  }
+                }
+              },
+              {
+                "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
+                "underlay_address": "fd00:1122:3344:101::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  }
+                }
+              },
+              {
+                "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
+                "underlay_address": "fd00:1122:3344:101::4",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::4]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                  }
+                }
+              },
+              {
+                "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
+                "underlay_address": "fd00:1122:3344:101::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
+                  }
+                }
+              },
+              {
+                "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
+                "underlay_address": "fd00:1122:3344:101::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
+                  }
+                }
+              },
+              {
+                "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
+                "underlay_address": "fd00:1122:3344:101::6",
+                "zone_type": {
+                  "type": "oximeter",
+                  "address": "[fd00:1122:3344:101::6]:12223"
+                }
+              },
+              {
+                "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                "underlay_address": "fd00:1122:3344:101::11",
+                "zone_type": {
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:101::11]:123",
+                  "dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "nic": {
+                    "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
+                    "ip": "172.30.3.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
+                    },
+                    "mac": "A8:40:25:FF:E7:A8",
+                    "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100
+                  },
+                  "ntp_servers": [
+                    "ntp.eng.oxide.computer"
+                  ],
+                  "snat_cfg": {
+                    "ip": "172.20.28.5",
+                    "first_port": 0,
+                    "last_port": 16383
+                  }
+                }
+              },
+              {
+                "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
+                "underlay_address": "fd00:1122:3344:101::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                  }
+                }
+              },
+              {
+                "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
+                "underlay_address": "fd00:1122:3344:101::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
+                  }
+                }
+              },
+              {
+                "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
+                "underlay_address": "fd00:1122:3344:101::7",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:101::7]:17000"
+                }
+              }
+            ]
+          }
+        },
+        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+          "time_collected": "2024-03-01T19:17:11.082540Z",
+          "source": "http://[fd00:1122:3344:121::1]:12345",
+          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
+          "zones": {
+            "generation": 3,
+            "zones": [
+              {
+                "id": "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
+                "underlay_address": "fd00:1122:3344:121::28",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::28]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3"
+                  }
+                }
+              },
+              {
+                "id": "12431d3e-1516-4822-98b3-3711ff912e69",
+                "underlay_address": "fd00:1122:3344:121::22",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::22]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_0622b2cf-297e-4356-b94c-d74309f443ba"
+                  }
+                }
+              },
+              {
+                "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
+                "underlay_address": "fd00:1122:3344:121::21",
+                "zone_type": {
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:121::21]:123",
+                  "dns_servers": [
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
+                  ],
+                  "ntp_servers": [
+                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                  ]
+                }
+              },
+              {
+                "id": "56414d76-9b8c-4e15-acf1-cd269c2d8638",
+                "underlay_address": "fd00:1122:3344:121::25",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::25]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_574728ca-1559-48f3-a46c-1e608966e6a0"
+                  }
+                }
+              },
+              {
+                "id": "891cb9da-301d-40cc-b93d-5fcb350fa968",
+                "underlay_address": "fd00:1122:3344:121::29",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::29]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
+                  }
+                }
+              },
+              {
+                "id": "d9014907-24c0-4c33-a475-db15942f4f8f",
+                "underlay_address": "fd00:1122:3344:121::23",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::23]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37"
+                  }
+                }
+              },
+              {
+                "id": "e950de25-7306-4cf7-a6f4-6db11adaa310",
+                "underlay_address": "fd00:1122:3344:121::27",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::27]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c"
+                  }
+                }
+              },
+              {
+                "id": "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
+                "underlay_address": "fd00:1122:3344:121::26",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::26]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945"
+                  }
+                }
+              },
+              {
+                "id": "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00",
+                "underlay_address": "fd00:1122:3344:121::24",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::24]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1"
+                  }
+                }
+              }
+            ]
+          }
+        },
+        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+          "time_collected": "2024-03-01T19:17:11.118025Z",
+          "source": "http://[fd00:1122:3344:102::1]:12345",
+          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
+                "underlay_address": "fd00:1122:3344:102::11",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::11]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                  }
+                }
+              },
+              {
+                "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
+                "underlay_address": "fd00:1122:3344:102::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
+                  }
+                }
+              },
+              {
+                "id": "479811e7-6409-45d4-b294-5333534e47ed",
+                "underlay_address": "fd00:1122:3344:102::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                  }
+                }
+              },
+              {
+                "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
+                "underlay_address": "fd00:1122:3344:102::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                  }
+                }
+              },
+              {
+                "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
+                "underlay_address": "fd00:1122:3344:2::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  },
+                  "dns_address": "[fd00:1122:3344:2::1]:53",
+                  "gz_address": "fd00:1122:3344:2::2",
+                  "gz_address_index": 1,
+                  "http_address": "[fd00:1122:3344:2::1]:5353"
+                }
+              },
+              {
+                "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+                "underlay_address": "fd00:1122:3344:102::7",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:102::7]:17000"
+                }
+              },
+              {
+                "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
+                "underlay_address": "fd00:1122:3344:102::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "658076ed-fb43-49e7-a813-df3d23497848",
+                "underlay_address": "fd00:1122:3344:102::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
+                  }
+                }
+              },
+              {
+                "id": "6edb836e-b100-4080-bb03-738d6441d571",
+                "underlay_address": "fd00:1122:3344:102::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
+                  }
+                }
+              },
+              {
+                "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                "underlay_address": "fd00:1122:3344:102::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.3",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:102::5]:12221",
+                  "nic": {
+                    "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
+                    "ip": "172.30.2.6",
+                    "kind": {
+                      "type": "service",
+                      "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
+                    },
+                    "mac": "A8:40:25:FF:B2:AF",
+                    "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
+                "underlay_address": "fd00:1122:3344:102::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                  }
+                }
+              },
+              {
+                "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+                "underlay_address": "fd00:1122:3344:102::4",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::4]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                  }
+                }
+              },
+              {
+                "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
+                "underlay_address": "fd00:1122:3344:102::6",
+                "zone_type": {
+                  "type": "clickhouse",
+                  "address": "[fd00:1122:3344:102::6]:8123",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
+                "underlay_address": "fd00:1122:3344:102::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
+                  }
+                }
+              },
+              {
+                "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
+                "underlay_address": "fd00:1122:3344:102::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                "underlay_address": "fd00:1122:3344:102::12",
+                "zone_type": {
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:102::12]:123",
+                  "dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "nic": {
+                    "id": "9f03c82c-3192-498d-978d-a175049ac389",
+                    "ip": "172.30.3.6",
+                    "kind": {
+                      "type": "service",
+                      "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
+                    },
+                    "mac": "A8:40:25:FF:E4:11",
+                    "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100
+                  },
+                  "ntp_servers": [
+                    "ntp.eng.oxide.computer"
+                  ],
+                  "snat_cfg": {
+                    "ip": "172.20.28.6",
+                    "first_port": 16384,
+                    "last_port": 32767
+                  }
+                }
+              },
+              {
+                "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
+                "underlay_address": "fd00:1122:3344:102::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
+                  }
+                }
+              }
+            ]
+          }
+        },
+        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+          "time_collected": "2024-03-01T19:17:11.153275Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
+                "underlay_address": "fd00:1122:3344:103::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
+                  }
+                }
+              },
+              {
+                "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
+                "underlay_address": "fd00:1122:3344:103::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
+                  }
+                }
+              },
+              {
+                "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
+                "underlay_address": "fd00:1122:3344:3::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  },
+                  "dns_address": "[fd00:1122:3344:3::1]:53",
+                  "gz_address": "fd00:1122:3344:3::2",
+                  "gz_address_index": 2,
+                  "http_address": "[fd00:1122:3344:3::1]:5353"
+                }
+              },
+              {
+                "id": "4b74c418-0a08-4489-b652-e97267c1c220",
+                "underlay_address": "fd00:1122:3344:103::7",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::7]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  }
+                }
+              },
+              {
+                "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+                "underlay_address": "fd00:1122:3344:103::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.4",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:103::5]:12221",
+                  "nic": {
+                    "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
+                    "ip": "172.30.2.7",
+                    "kind": {
+                      "type": "service",
+                      "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
+                    },
+                    "mac": "A8:40:25:FF:96:05",
+                    "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
+                "underlay_address": "fd00:1122:3344:103::6",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:103::6]:17000"
+                }
+              },
+              {
+                "id": "921958c0-7761-4294-945d-2a078327bf2c",
+                "underlay_address": "fd00:1122:3344:103::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
+                  }
+                }
+              },
+              {
+                "id": "9df22034-59f6-431f-82e7-258fe983879b",
+                "underlay_address": "fd00:1122:3344:103::4",
+                "zone_type": {
+                  "type": "external_dns",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  },
+                  "dns_address": "172.20.28.1:53",
+                  "http_address": "[fd00:1122:3344:103::4]:5353",
+                  "nic": {
+                    "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
+                    "ip": "172.30.1.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "9df22034-59f6-431f-82e7-258fe983879b"
+                    },
+                    "mac": "A8:40:25:FF:90:12",
+                    "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.1.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
+                "underlay_address": "fd00:1122:3344:103::11",
+                "zone_type": {
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:103::11]:123",
+                  "dns_servers": [
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
+                  ],
+                  "ntp_servers": [
+                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                  ]
+                }
+              },
+              {
+                "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
+                "underlay_address": "fd00:1122:3344:103::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
+                  }
+                }
+              },
+              {
+                "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
+                "underlay_address": "fd00:1122:3344:103::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
+                  }
+                }
+              },
+              {
+                "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
+                "underlay_address": "fd00:1122:3344:103::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
+                  }
+                }
+              },
+              {
+                "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
+                "underlay_address": "fd00:1122:3344:103::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
+                  }
+                }
+              },
+              {
+                "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
+                "underlay_address": "fd00:1122:3344:103::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:103::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  }
+                }
+              },
+              {
+                "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
+                "underlay_address": "fd00:1122:3344:103::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
+                  }
+                }
+              },
+              {
+                "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
+                "underlay_address": "fd00:1122:3344:103::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                  }
+                }
+              }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "id": "cd3547d7-f172-47d2-9bbf-9730e88b0559",
+      "errors": [
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a86ce874-ddce-4497-9e34-c7053d224035\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"a86ce874-ddce-4497-9e34-c7053d224035\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bf3dbdb6-cac9-43a1-a55e-973d2397ffe6\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"bf3dbdb6-cac9-43a1-a55e-973d2397ffe6\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fabd84d3-7c0c-4e60-99b4-f97b9288839c\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"fabd84d3-7c0c-4e60-99b4-f97b9288839c\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c0160967-c6b3-49b7-b801-0fcd6d4f5e31\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"c0160967-c6b3-49b7-b801-0fcd6d4f5e31\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"247b36be-d31b-460a-8722-58a21008eba3\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"247b36be-d31b-460a-8722-58a21008eba3\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"81998378-2641-4eab-8bcb-f335995e0af8\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"81998378-2641-4eab-8bcb-f335995e0af8\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8fcd5a26-f45a-47fc-9219-e57ac207fd2e\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"8fcd5a26-f45a-47fc-9219-e57ac207fd2e\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"20e66cd5-274f-487d-8186-c5e4cd25ae74\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"20e66cd5-274f-487d-8186-c5e4cd25ae74\" }"
+      ],
+      "time_started": "2024-03-01T19:18:03.580607Z",
+      "time_done": "2024-03-01T19:18:07.486108Z",
+      "collector": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+      "baseboards": [
+        {
+          "part_number": "913-0000006",
+          "serial_number": "BRM44220005"
+        },
+        {
+          "part_number": "913-0000006",
+          "serial_number": "BRM44220011"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220004"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220046"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220081"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM44220001"
+        }
+      ],
+      "cabooses": [
+        {
+          "board": "gimlet-c",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "gimlet-c",
+          "version": "1.0.6"
+        },
+        {
+          "board": "gimlet-c",
+          "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
+          "name": "gimlet-c",
+          "version": "1.0.5"
+        },
+        {
+          "board": "gimlet-c",
+          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+          "name": "gimlet-c",
+          "version": "1.0.8"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "oxide-rot-1",
+          "version": "1.0.5"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
+          "name": "oxide-rot-1",
+          "version": "1.0.4"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
+          "name": "oxide-rot-1",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+          "name": "oxide-rot-1",
+          "version": "1.0.6"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+          "name": "oxide-rot-1",
+          "version": "1.0.0"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+          "name": "oxide-rot-1-dev",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "sidecar-b",
+          "version": "1.0.6"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
+          "name": "sidecar-b",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
+          "name": "sidecar-b",
+          "version": "0.0.0-git"
+        }
+      ],
+      "rot_pages": [
+        {
+          "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+        },
+        {
+          "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+        },
+        {
+          "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+        },
+        {
+          "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+        },
+        {
+          "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+        },
+        {
+          "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+        },
+        {
+          "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+        }
+      ],
+      "sps": [
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220005"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:05.910026Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "switch",
+            "sp_slot": 0,
+            "baseboard_revision": 4,
+            "hubris_archive": "9184be93590e655e",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220011"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:06.485436Z",
+            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "sp_type": "switch",
+            "sp_slot": 1,
+            "baseboard_revision": 4,
+            "hubris_archive": "9184be93590e655e",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:04.749894Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 17,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:05.327348Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 15,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:04.172071Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 14,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:03.592833Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 16,
+            "baseboard_revision": 6,
+            "hubris_archive": "3c670b44fdeef4af",
+            "power_state": "A0"
+          }
+        ]
+      ],
+      "rots": [
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220005"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:05.910026Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "4599e4d7bc9b2164e8c0f0f7ce4bcae2bd35d635c8450854ea9a94ce49a2ca0f",
+            "slot_b_sha3_256_digest": "1e42a0761e5bd1d9f9a5e2c44acc34e7b7e007792977cdb0523d271db2e80c8c"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220011"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:06.485436Z",
+            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "active_slot": {
+              "slot": "b"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "92e07077972c83af6872ec817fb88d74a36f6956276505e9771302c5bb7ec2a8",
+            "slot_b_sha3_256_digest": "fb25825eaf97f8f2307afe3caebf0b70865499f6d116db3aff382c505b4d8dd7"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:04.749894Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
+            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:05.327348Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
+            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:04.172071Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "b"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
+            "slot_b_sha3_256_digest": "d416d5ba9d0d3bd495f0f5fcd2aecc28cd83b53d1a8f94a28241d85b30761451"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:03.592833Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
+            "slot_b_sha3_256_digest": "4ec328195a23e86a1545d5ee576e73834aae81fb79830a54d0503475875f1760"
+          }
+        ]
+      ],
+      "cabooses_found": {
+        "SpSlot0": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.911536Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "sidecar-b",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:06.486873Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "sidecar-b",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.751575Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.332841Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.173529Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:03.594379Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ]
+        ],
+        "SpSlot1": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.912952Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
+                "name": "sidecar-b",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:06.488258Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
+                "name": "sidecar-b",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.753197Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.334324Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.174886Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:03.596125Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
+                "name": "gimlet-c",
+                "version": "1.0.5"
+              }
+            }
+          ]
+        ],
+        "RotSlotA": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:06.119739Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+                "name": "oxide-rot-1-dev",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:06.695247Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
+                "name": "oxide-rot-1",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.004236Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.586015Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.426689Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:03.847427Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ]
+        ],
+        "RotSlotB": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:06.328695Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:06.903153Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+                "name": "oxide-rot-1",
+                "version": "1.0.0"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.256365Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.838048Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.678679Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.100460Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
+                "name": "oxide-rot-1",
+                "version": "1.0.4"
+              }
+            }
+          ]
+        ]
+      },
+      "rot_pages_found": {
+        "Cmpa": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.273195Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.854538Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.695195Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.116988Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ]
+        ],
+        "CfpaActive": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.288821Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.870545Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.711196Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.132904Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ],
+        "CfpaInactive": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.304991Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.886552Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.727187Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.148861Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ],
+        "CfpaScratch": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.320918Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:05.902654Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.743134Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:04.164870Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ]
+      },
+      "sled_agents": {
+        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+          "time_collected": "2024-03-01T19:18:07.325811Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+          "time_collected": "2024-03-01T19:18:07.365147Z",
+          "source": "http://[fd00:1122:3344:121::1]:12345",
+          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          "sled_agent_address": "[fd00:1122:3344:121::1]:12345",
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+          "time_collected": "2024-03-01T19:18:07.401947Z",
+          "source": "http://[fd00:1122:3344:102::1]:12345",
+          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+          "time_collected": "2024-03-01T19:18:07.440624Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        }
+      },
+      "omicron_zones": {
+        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+          "time_collected": "2024-03-01T19:18:07.327154Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "0706860a-3801-42a7-8000-fff741f7665b",
+                "underlay_address": "fd00:1122:3344:101::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
+                  }
+                }
+              },
+              {
+                "id": "34a279a2-a025-41ff-a392-1addac400f38",
+                "underlay_address": "fd00:1122:3344:101::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  }
+                }
+              },
+              {
+                "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
+                "underlay_address": "fd00:1122:3344:1::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  },
+                  "dns_address": "[fd00:1122:3344:1::1]:53",
+                  "gz_address": "fd00:1122:3344:1::2",
+                  "gz_address_index": 0,
+                  "http_address": "[fd00:1122:3344:1::1]:5353"
+                }
+              },
+              {
+                "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+                "underlay_address": "fd00:1122:3344:101::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.2",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:101::5]:12221",
+                  "nic": {
+                    "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
+                    "ip": "172.30.2.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
+                    },
+                    "mac": "A8:40:25:FF:E1:17",
+                    "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
+                "underlay_address": "fd00:1122:3344:101::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+                  }
+                }
+              },
+              {
+                "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
+                "underlay_address": "fd00:1122:3344:101::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
+                  }
+                }
+              },
+              {
+                "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
+                "underlay_address": "fd00:1122:3344:101::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                  }
+                }
+              },
+              {
+                "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
+                "underlay_address": "fd00:1122:3344:101::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  }
+                }
+              },
+              {
+                "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
+                "underlay_address": "fd00:1122:3344:101::4",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::4]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                  }
+                }
+              },
+              {
+                "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
+                "underlay_address": "fd00:1122:3344:101::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
+                  }
+                }
+              },
+              {
+                "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
+                "underlay_address": "fd00:1122:3344:101::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
+                  }
+                }
+              },
+              {
+                "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
+                "underlay_address": "fd00:1122:3344:101::6",
+                "zone_type": {
+                  "type": "oximeter",
+                  "address": "[fd00:1122:3344:101::6]:12223"
+                }
+              },
+              {
+                "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                "underlay_address": "fd00:1122:3344:101::11",
+                "zone_type": {
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:101::11]:123",
+                  "dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "nic": {
+                    "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
+                    "ip": "172.30.3.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
+                    },
+                    "mac": "A8:40:25:FF:E7:A8",
+                    "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100
+                  },
+                  "ntp_servers": [
+                    "ntp.eng.oxide.computer"
+                  ],
+                  "snat_cfg": {
+                    "ip": "172.20.28.5",
+                    "first_port": 0,
+                    "last_port": 16383
+                  }
+                }
+              },
+              {
+                "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
+                "underlay_address": "fd00:1122:3344:101::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                  }
+                }
+              },
+              {
+                "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
+                "underlay_address": "fd00:1122:3344:101::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
+                  }
+                }
+              },
+              {
+                "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
+                "underlay_address": "fd00:1122:3344:101::7",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:101::7]:17000"
+                }
+              }
+            ]
+          }
+        },
+        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+          "time_collected": "2024-03-01T19:18:07.365954Z",
+          "source": "http://[fd00:1122:3344:121::1]:12345",
+          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
+          "zones": {
+            "generation": 3,
+            "zones": [
+              {
+                "id": "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
+                "underlay_address": "fd00:1122:3344:121::28",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::28]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3"
+                  }
+                }
+              },
+              {
+                "id": "12431d3e-1516-4822-98b3-3711ff912e69",
+                "underlay_address": "fd00:1122:3344:121::22",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::22]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_0622b2cf-297e-4356-b94c-d74309f443ba"
+                  }
+                }
+              },
+              {
+                "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
+                "underlay_address": "fd00:1122:3344:121::21",
+                "zone_type": {
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:121::21]:123",
+                  "dns_servers": [
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
+                  ],
+                  "ntp_servers": [
+                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                  ]
+                }
+              },
+              {
+                "id": "56414d76-9b8c-4e15-acf1-cd269c2d8638",
+                "underlay_address": "fd00:1122:3344:121::25",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::25]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_574728ca-1559-48f3-a46c-1e608966e6a0"
+                  }
+                }
+              },
+              {
+                "id": "891cb9da-301d-40cc-b93d-5fcb350fa968",
+                "underlay_address": "fd00:1122:3344:121::29",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::29]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
+                  }
+                }
+              },
+              {
+                "id": "d9014907-24c0-4c33-a475-db15942f4f8f",
+                "underlay_address": "fd00:1122:3344:121::23",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::23]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37"
+                  }
+                }
+              },
+              {
+                "id": "e950de25-7306-4cf7-a6f4-6db11adaa310",
+                "underlay_address": "fd00:1122:3344:121::27",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::27]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c"
+                  }
+                }
+              },
+              {
+                "id": "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
+                "underlay_address": "fd00:1122:3344:121::26",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::26]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945"
+                  }
+                }
+              },
+              {
+                "id": "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00",
+                "underlay_address": "fd00:1122:3344:121::24",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::24]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1"
+                  }
+                }
+              }
+            ]
+          }
+        },
+        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+          "time_collected": "2024-03-01T19:18:07.403197Z",
+          "source": "http://[fd00:1122:3344:102::1]:12345",
+          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
+                "underlay_address": "fd00:1122:3344:102::11",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::11]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                  }
+                }
+              },
+              {
+                "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
+                "underlay_address": "fd00:1122:3344:102::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
+                  }
+                }
+              },
+              {
+                "id": "479811e7-6409-45d4-b294-5333534e47ed",
+                "underlay_address": "fd00:1122:3344:102::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                  }
+                }
+              },
+              {
+                "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
+                "underlay_address": "fd00:1122:3344:102::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                  }
+                }
+              },
+              {
+                "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
+                "underlay_address": "fd00:1122:3344:2::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  },
+                  "dns_address": "[fd00:1122:3344:2::1]:53",
+                  "gz_address": "fd00:1122:3344:2::2",
+                  "gz_address_index": 1,
+                  "http_address": "[fd00:1122:3344:2::1]:5353"
+                }
+              },
+              {
+                "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+                "underlay_address": "fd00:1122:3344:102::7",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:102::7]:17000"
+                }
+              },
+              {
+                "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
+                "underlay_address": "fd00:1122:3344:102::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "658076ed-fb43-49e7-a813-df3d23497848",
+                "underlay_address": "fd00:1122:3344:102::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
+                  }
+                }
+              },
+              {
+                "id": "6edb836e-b100-4080-bb03-738d6441d571",
+                "underlay_address": "fd00:1122:3344:102::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
+                  }
+                }
+              },
+              {
+                "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                "underlay_address": "fd00:1122:3344:102::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.3",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:102::5]:12221",
+                  "nic": {
+                    "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
+                    "ip": "172.30.2.6",
+                    "kind": {
+                      "type": "service",
+                      "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
+                    },
+                    "mac": "A8:40:25:FF:B2:AF",
+                    "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
+                "underlay_address": "fd00:1122:3344:102::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                  }
+                }
+              },
+              {
+                "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+                "underlay_address": "fd00:1122:3344:102::4",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::4]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                  }
+                }
+              },
+              {
+                "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
+                "underlay_address": "fd00:1122:3344:102::6",
+                "zone_type": {
+                  "type": "clickhouse",
+                  "address": "[fd00:1122:3344:102::6]:8123",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
+                "underlay_address": "fd00:1122:3344:102::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
+                  }
+                }
+              },
+              {
+                "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
+                "underlay_address": "fd00:1122:3344:102::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                "underlay_address": "fd00:1122:3344:102::12",
+                "zone_type": {
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:102::12]:123",
+                  "dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "nic": {
+                    "id": "9f03c82c-3192-498d-978d-a175049ac389",
+                    "ip": "172.30.3.6",
+                    "kind": {
+                      "type": "service",
+                      "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
+                    },
+                    "mac": "A8:40:25:FF:E4:11",
+                    "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100
+                  },
+                  "ntp_servers": [
+                    "ntp.eng.oxide.computer"
+                  ],
+                  "snat_cfg": {
+                    "ip": "172.20.28.6",
+                    "first_port": 16384,
+                    "last_port": 32767
+                  }
+                }
+              },
+              {
+                "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
+                "underlay_address": "fd00:1122:3344:102::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
+                  }
+                }
+              }
+            ]
+          }
+        },
+        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+          "time_collected": "2024-03-01T19:18:07.483569Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
+                "underlay_address": "fd00:1122:3344:103::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
+                  }
+                }
+              },
+              {
+                "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
+                "underlay_address": "fd00:1122:3344:103::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
+                  }
+                }
+              },
+              {
+                "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
+                "underlay_address": "fd00:1122:3344:3::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  },
+                  "dns_address": "[fd00:1122:3344:3::1]:53",
+                  "gz_address": "fd00:1122:3344:3::2",
+                  "gz_address_index": 2,
+                  "http_address": "[fd00:1122:3344:3::1]:5353"
+                }
+              },
+              {
+                "id": "4b74c418-0a08-4489-b652-e97267c1c220",
+                "underlay_address": "fd00:1122:3344:103::7",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::7]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  }
+                }
+              },
+              {
+                "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+                "underlay_address": "fd00:1122:3344:103::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.4",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:103::5]:12221",
+                  "nic": {
+                    "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
+                    "ip": "172.30.2.7",
+                    "kind": {
+                      "type": "service",
+                      "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
+                    },
+                    "mac": "A8:40:25:FF:96:05",
+                    "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
+                "underlay_address": "fd00:1122:3344:103::6",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:103::6]:17000"
+                }
+              },
+              {
+                "id": "921958c0-7761-4294-945d-2a078327bf2c",
+                "underlay_address": "fd00:1122:3344:103::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
+                  }
+                }
+              },
+              {
+                "id": "9df22034-59f6-431f-82e7-258fe983879b",
+                "underlay_address": "fd00:1122:3344:103::4",
+                "zone_type": {
+                  "type": "external_dns",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  },
+                  "dns_address": "172.20.28.1:53",
+                  "http_address": "[fd00:1122:3344:103::4]:5353",
+                  "nic": {
+                    "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
+                    "ip": "172.30.1.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "9df22034-59f6-431f-82e7-258fe983879b"
+                    },
+                    "mac": "A8:40:25:FF:90:12",
+                    "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.1.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
+                "underlay_address": "fd00:1122:3344:103::11",
+                "zone_type": {
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:103::11]:123",
+                  "dns_servers": [
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
+                  ],
+                  "ntp_servers": [
+                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                  ]
+                }
+              },
+              {
+                "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
+                "underlay_address": "fd00:1122:3344:103::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
+                  }
+                }
+              },
+              {
+                "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
+                "underlay_address": "fd00:1122:3344:103::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
+                  }
+                }
+              },
+              {
+                "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
+                "underlay_address": "fd00:1122:3344:103::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
+                  }
+                }
+              },
+              {
+                "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
+                "underlay_address": "fd00:1122:3344:103::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
+                  }
+                }
+              },
+              {
+                "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
+                "underlay_address": "fd00:1122:3344:103::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:103::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  }
+                }
+              },
+              {
+                "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
+                "underlay_address": "fd00:1122:3344:103::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
+                  }
+                }
+              },
+              {
+                "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
+                "underlay_address": "fd00:1122:3344:103::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                  }
+                }
+              }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "id": "623f7e48-6c47-4e17-8aa3-cebe98ca4287",
+      "errors": [
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"431a5722-e328-450a-89f4-017274532055\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:10 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"431a5722-e328-450a-89f4-017274532055\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"38399802-a750-4c20-8243-f8df25e6c01b\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"38399802-a750-4c20-8243-f8df25e6c01b\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cbbbe7aa-4a9f-44d3-9713-6c12a1d2b00a\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"cbbbe7aa-4a9f-44d3-9713-6c12a1d2b00a\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a93a52ae-2837-4a80-a8d9-68f43bdaabe9\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"a93a52ae-2837-4a80-a8d9-68f43bdaabe9\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0016667c-69f3-4359-9ed8-176e8d03f270\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"0016667c-69f3-4359-9ed8-176e8d03f270\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0f8d60bc-31ee-460d-a8b5-9073728c2402\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"0f8d60bc-31ee-460d-a8b5-9073728c2402\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"595cf967-cade-4ae6-87f5-d04e5dbc55b9\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"595cf967-cade-4ae6-87f5-d04e5dbc55b9\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"83fe7422-15cd-48d3-b21d-967479fb090c\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"83fe7422-15cd-48d3-b21d-967479fb090c\" }"
+      ],
+      "time_started": "2024-03-01T19:18:07.755939Z",
+      "time_done": "2024-03-01T19:18:12.110340Z",
+      "collector": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+      "baseboards": [
+        {
+          "part_number": "913-0000006",
+          "serial_number": "BRM44220005"
+        },
+        {
+          "part_number": "913-0000006",
+          "serial_number": "BRM44220011"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220004"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220046"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220081"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM44220001"
+        }
+      ],
+      "cabooses": [
+        {
+          "board": "gimlet-c",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "gimlet-c",
+          "version": "1.0.6"
+        },
+        {
+          "board": "gimlet-c",
+          "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
+          "name": "gimlet-c",
+          "version": "1.0.5"
+        },
+        {
+          "board": "gimlet-c",
+          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+          "name": "gimlet-c",
+          "version": "1.0.8"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "oxide-rot-1",
+          "version": "1.0.5"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
+          "name": "oxide-rot-1",
+          "version": "1.0.4"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
+          "name": "oxide-rot-1",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+          "name": "oxide-rot-1",
+          "version": "1.0.6"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+          "name": "oxide-rot-1",
+          "version": "1.0.0"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+          "name": "oxide-rot-1-dev",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "sidecar-b",
+          "version": "1.0.6"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
+          "name": "sidecar-b",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
+          "name": "sidecar-b",
+          "version": "0.0.0-git"
+        }
+      ],
+      "rot_pages": [
+        {
+          "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+        },
+        {
+          "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+        },
+        {
+          "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+        },
+        {
+          "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+        },
+        {
+          "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+        },
+        {
+          "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+        },
+        {
+          "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+        }
+      ],
+      "sps": [
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220005"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:10.815811Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "switch",
+            "sp_slot": 0,
+            "baseboard_revision": 4,
+            "hubris_archive": "9184be93590e655e",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220011"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:11.387242Z",
+            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "sp_type": "switch",
+            "sp_slot": 1,
+            "baseboard_revision": 4,
+            "hubris_archive": "9184be93590e655e",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:09.650491Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 17,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:10.237239Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 15,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:08.884903Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 14,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:07.768812Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 16,
+            "baseboard_revision": 6,
+            "hubris_archive": "3c670b44fdeef4af",
+            "power_state": "A0"
+          }
+        ]
+      ],
+      "rots": [
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220005"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:10.815811Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "4599e4d7bc9b2164e8c0f0f7ce4bcae2bd35d635c8450854ea9a94ce49a2ca0f",
+            "slot_b_sha3_256_digest": "1e42a0761e5bd1d9f9a5e2c44acc34e7b7e007792977cdb0523d271db2e80c8c"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220011"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:11.387242Z",
+            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "active_slot": {
+              "slot": "b"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "92e07077972c83af6872ec817fb88d74a36f6956276505e9771302c5bb7ec2a8",
+            "slot_b_sha3_256_digest": "fb25825eaf97f8f2307afe3caebf0b70865499f6d116db3aff382c505b4d8dd7"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:09.650491Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
+            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:10.237239Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
+            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:08.884903Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "b"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
+            "slot_b_sha3_256_digest": "d416d5ba9d0d3bd495f0f5fcd2aecc28cd83b53d1a8f94a28241d85b30761451"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:07.768812Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
+            "slot_b_sha3_256_digest": "4ec328195a23e86a1545d5ee576e73834aae81fb79830a54d0503475875f1760"
+          }
+        ]
+      ],
+      "cabooses_found": {
+        "SpSlot0": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.817224Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "sidecar-b",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.388557Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "sidecar-b",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.651842Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.238618Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.886274Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:07.777292Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ]
+        ],
+        "SpSlot1": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.822223Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
+                "name": "sidecar-b",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.389874Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
+                "name": "sidecar-b",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.653307Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.240066Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.887692Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:07.779701Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
+                "name": "gimlet-c",
+                "version": "1.0.5"
+              }
+            }
+          ]
+        ],
+        "RotSlotA": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.029595Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+                "name": "oxide-rot-1-dev",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.595974Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
+                "name": "oxide-rot-1",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.905198Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.491985Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.148589Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.142368Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ]
+        ],
+        "RotSlotB": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.236464Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.830868Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+                "name": "oxide-rot-1",
+                "version": "1.0.0"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.164175Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.743940Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.402608Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.642339Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
+                "name": "oxide-rot-1",
+                "version": "1.0.4"
+              }
+            }
+          ]
+        ]
+      },
+      "rot_pages_found": {
+        "Cmpa": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.181324Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.760461Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.419090Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.722917Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ]
+        ],
+        "CfpaActive": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.197812Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.776526Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.491156Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.813851Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ],
+        "CfpaInactive": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.213728Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.792457Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.562113Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.845830Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ],
+        "CfpaScratch": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.229796Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.809035Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.643193Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.878183Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ]
+      },
+      "sled_agents": {
+        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+          "time_collected": "2024-03-01T19:18:12.000624Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+          "time_collected": "2024-03-01T19:18:12.036086Z",
+          "source": "http://[fd00:1122:3344:121::1]:12345",
+          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          "sled_agent_address": "[fd00:1122:3344:121::1]:12345",
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+          "time_collected": "2024-03-01T19:18:12.071731Z",
+          "source": "http://[fd00:1122:3344:102::1]:12345",
+          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+          "time_collected": "2024-03-01T19:18:12.107230Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        }
+      },
+      "omicron_zones": {
+        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+          "time_collected": "2024-03-01T19:18:12.001507Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "0706860a-3801-42a7-8000-fff741f7665b",
+                "underlay_address": "fd00:1122:3344:101::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
+                  }
+                }
+              },
+              {
+                "id": "34a279a2-a025-41ff-a392-1addac400f38",
+                "underlay_address": "fd00:1122:3344:101::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  }
+                }
+              },
+              {
+                "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
+                "underlay_address": "fd00:1122:3344:1::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  },
+                  "dns_address": "[fd00:1122:3344:1::1]:53",
+                  "gz_address": "fd00:1122:3344:1::2",
+                  "gz_address_index": 0,
+                  "http_address": "[fd00:1122:3344:1::1]:5353"
+                }
+              },
+              {
+                "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+                "underlay_address": "fd00:1122:3344:101::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.2",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:101::5]:12221",
+                  "nic": {
+                    "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
+                    "ip": "172.30.2.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
+                    },
+                    "mac": "A8:40:25:FF:E1:17",
+                    "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
+                "underlay_address": "fd00:1122:3344:101::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+                  }
+                }
+              },
+              {
+                "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
+                "underlay_address": "fd00:1122:3344:101::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
+                  }
+                }
+              },
+              {
+                "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
+                "underlay_address": "fd00:1122:3344:101::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                  }
+                }
+              },
+              {
+                "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
+                "underlay_address": "fd00:1122:3344:101::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  }
+                }
+              },
+              {
+                "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
+                "underlay_address": "fd00:1122:3344:101::4",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::4]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                  }
+                }
+              },
+              {
+                "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
+                "underlay_address": "fd00:1122:3344:101::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
+                  }
+                }
+              },
+              {
+                "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
+                "underlay_address": "fd00:1122:3344:101::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
+                  }
+                }
+              },
+              {
+                "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
+                "underlay_address": "fd00:1122:3344:101::6",
+                "zone_type": {
+                  "type": "oximeter",
+                  "address": "[fd00:1122:3344:101::6]:12223"
+                }
+              },
+              {
+                "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                "underlay_address": "fd00:1122:3344:101::11",
+                "zone_type": {
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:101::11]:123",
+                  "dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "nic": {
+                    "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
+                    "ip": "172.30.3.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
+                    },
+                    "mac": "A8:40:25:FF:E7:A8",
+                    "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100
+                  },
+                  "ntp_servers": [
+                    "ntp.eng.oxide.computer"
+                  ],
+                  "snat_cfg": {
+                    "ip": "172.20.28.5",
+                    "first_port": 0,
+                    "last_port": 16383
+                  }
+                }
+              },
+              {
+                "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
+                "underlay_address": "fd00:1122:3344:101::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                  }
+                }
+              },
+              {
+                "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
+                "underlay_address": "fd00:1122:3344:101::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
+                  }
+                }
+              },
+              {
+                "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
+                "underlay_address": "fd00:1122:3344:101::7",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:101::7]:17000"
+                }
+              }
+            ]
+          }
+        },
+        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+          "time_collected": "2024-03-01T19:18:12.036797Z",
+          "source": "http://[fd00:1122:3344:121::1]:12345",
+          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
+          "zones": {
+            "generation": 3,
+            "zones": [
+              {
+                "id": "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
+                "underlay_address": "fd00:1122:3344:121::28",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::28]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3"
+                  }
+                }
+              },
+              {
+                "id": "12431d3e-1516-4822-98b3-3711ff912e69",
+                "underlay_address": "fd00:1122:3344:121::22",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::22]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_0622b2cf-297e-4356-b94c-d74309f443ba"
+                  }
+                }
+              },
+              {
+                "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
+                "underlay_address": "fd00:1122:3344:121::21",
+                "zone_type": {
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:121::21]:123",
+                  "dns_servers": [
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
+                  ],
+                  "ntp_servers": [
+                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                  ]
+                }
+              },
+              {
+                "id": "56414d76-9b8c-4e15-acf1-cd269c2d8638",
+                "underlay_address": "fd00:1122:3344:121::25",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::25]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_574728ca-1559-48f3-a46c-1e608966e6a0"
+                  }
+                }
+              },
+              {
+                "id": "891cb9da-301d-40cc-b93d-5fcb350fa968",
+                "underlay_address": "fd00:1122:3344:121::29",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::29]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
+                  }
+                }
+              },
+              {
+                "id": "d9014907-24c0-4c33-a475-db15942f4f8f",
+                "underlay_address": "fd00:1122:3344:121::23",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::23]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37"
+                  }
+                }
+              },
+              {
+                "id": "e950de25-7306-4cf7-a6f4-6db11adaa310",
+                "underlay_address": "fd00:1122:3344:121::27",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::27]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c"
+                  }
+                }
+              },
+              {
+                "id": "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
+                "underlay_address": "fd00:1122:3344:121::26",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::26]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945"
+                  }
+                }
+              },
+              {
+                "id": "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00",
+                "underlay_address": "fd00:1122:3344:121::24",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::24]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1"
+                  }
+                }
+              }
+            ]
+          }
+        },
+        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+          "time_collected": "2024-03-01T19:18:12.072810Z",
+          "source": "http://[fd00:1122:3344:102::1]:12345",
+          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
+                "underlay_address": "fd00:1122:3344:102::11",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::11]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                  }
+                }
+              },
+              {
+                "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
+                "underlay_address": "fd00:1122:3344:102::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
+                  }
+                }
+              },
+              {
+                "id": "479811e7-6409-45d4-b294-5333534e47ed",
+                "underlay_address": "fd00:1122:3344:102::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                  }
+                }
+              },
+              {
+                "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
+                "underlay_address": "fd00:1122:3344:102::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                  }
+                }
+              },
+              {
+                "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
+                "underlay_address": "fd00:1122:3344:2::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  },
+                  "dns_address": "[fd00:1122:3344:2::1]:53",
+                  "gz_address": "fd00:1122:3344:2::2",
+                  "gz_address_index": 1,
+                  "http_address": "[fd00:1122:3344:2::1]:5353"
+                }
+              },
+              {
+                "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+                "underlay_address": "fd00:1122:3344:102::7",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:102::7]:17000"
+                }
+              },
+              {
+                "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
+                "underlay_address": "fd00:1122:3344:102::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "658076ed-fb43-49e7-a813-df3d23497848",
+                "underlay_address": "fd00:1122:3344:102::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
+                  }
+                }
+              },
+              {
+                "id": "6edb836e-b100-4080-bb03-738d6441d571",
+                "underlay_address": "fd00:1122:3344:102::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
+                  }
+                }
+              },
+              {
+                "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                "underlay_address": "fd00:1122:3344:102::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.3",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:102::5]:12221",
+                  "nic": {
+                    "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
+                    "ip": "172.30.2.6",
+                    "kind": {
+                      "type": "service",
+                      "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
+                    },
+                    "mac": "A8:40:25:FF:B2:AF",
+                    "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
+                "underlay_address": "fd00:1122:3344:102::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                  }
+                }
+              },
+              {
+                "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+                "underlay_address": "fd00:1122:3344:102::4",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::4]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                  }
+                }
+              },
+              {
+                "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
+                "underlay_address": "fd00:1122:3344:102::6",
+                "zone_type": {
+                  "type": "clickhouse",
+                  "address": "[fd00:1122:3344:102::6]:8123",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
+                "underlay_address": "fd00:1122:3344:102::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
+                  }
+                }
+              },
+              {
+                "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
+                "underlay_address": "fd00:1122:3344:102::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                "underlay_address": "fd00:1122:3344:102::12",
+                "zone_type": {
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:102::12]:123",
+                  "dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "nic": {
+                    "id": "9f03c82c-3192-498d-978d-a175049ac389",
+                    "ip": "172.30.3.6",
+                    "kind": {
+                      "type": "service",
+                      "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
+                    },
+                    "mac": "A8:40:25:FF:E4:11",
+                    "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100
+                  },
+                  "ntp_servers": [
+                    "ntp.eng.oxide.computer"
+                  ],
+                  "snat_cfg": {
+                    "ip": "172.20.28.6",
+                    "first_port": 16384,
+                    "last_port": 32767
+                  }
+                }
+              },
+              {
+                "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
+                "underlay_address": "fd00:1122:3344:102::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
+                  }
+                }
+              }
+            ]
+          }
+        },
+        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+          "time_collected": "2024-03-01T19:18:12.108386Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
+                "underlay_address": "fd00:1122:3344:103::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
+                  }
+                }
+              },
+              {
+                "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
+                "underlay_address": "fd00:1122:3344:103::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
+                  }
+                }
+              },
+              {
+                "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
+                "underlay_address": "fd00:1122:3344:3::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  },
+                  "dns_address": "[fd00:1122:3344:3::1]:53",
+                  "gz_address": "fd00:1122:3344:3::2",
+                  "gz_address_index": 2,
+                  "http_address": "[fd00:1122:3344:3::1]:5353"
+                }
+              },
+              {
+                "id": "4b74c418-0a08-4489-b652-e97267c1c220",
+                "underlay_address": "fd00:1122:3344:103::7",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::7]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  }
+                }
+              },
+              {
+                "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+                "underlay_address": "fd00:1122:3344:103::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.4",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:103::5]:12221",
+                  "nic": {
+                    "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
+                    "ip": "172.30.2.7",
+                    "kind": {
+                      "type": "service",
+                      "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
+                    },
+                    "mac": "A8:40:25:FF:96:05",
+                    "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
+                "underlay_address": "fd00:1122:3344:103::6",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:103::6]:17000"
+                }
+              },
+              {
+                "id": "921958c0-7761-4294-945d-2a078327bf2c",
+                "underlay_address": "fd00:1122:3344:103::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
+                  }
+                }
+              },
+              {
+                "id": "9df22034-59f6-431f-82e7-258fe983879b",
+                "underlay_address": "fd00:1122:3344:103::4",
+                "zone_type": {
+                  "type": "external_dns",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  },
+                  "dns_address": "172.20.28.1:53",
+                  "http_address": "[fd00:1122:3344:103::4]:5353",
+                  "nic": {
+                    "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
+                    "ip": "172.30.1.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "9df22034-59f6-431f-82e7-258fe983879b"
+                    },
+                    "mac": "A8:40:25:FF:90:12",
+                    "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.1.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
+                "underlay_address": "fd00:1122:3344:103::11",
+                "zone_type": {
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:103::11]:123",
+                  "dns_servers": [
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
+                  ],
+                  "ntp_servers": [
+                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                  ]
+                }
+              },
+              {
+                "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
+                "underlay_address": "fd00:1122:3344:103::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
+                  }
+                }
+              },
+              {
+                "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
+                "underlay_address": "fd00:1122:3344:103::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
+                  }
+                }
+              },
+              {
+                "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
+                "underlay_address": "fd00:1122:3344:103::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
+                  }
+                }
+              },
+              {
+                "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
+                "underlay_address": "fd00:1122:3344:103::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
+                  }
+                }
+              },
+              {
+                "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
+                "underlay_address": "fd00:1122:3344:103::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:103::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  }
+                }
+              },
+              {
+                "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
+                "underlay_address": "fd00:1122:3344:103::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
+                  }
+                }
+              },
+              {
+                "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
+                "underlay_address": "fd00:1122:3344:103::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                  }
+                }
+              }
+            ]
+          }
+        }
+      }
+    },
+    {
+      "id": "4e9fbfc2-eb1d-4d45-bdb9-64d965a490c5",
+      "errors": [
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f6c35180-8831-4718-995a-a08427e3f7d0\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"f6c35180-8831-4718-995a-a08427e3f7d0\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2206bf05-edce-4002-bc9c-72f93cfa45b1\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"2206bf05-edce-4002-bc9c-72f93cfa45b1\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"081feef1-085b-4f9a-9c0a-9c07a7a67ece\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"081feef1-085b-4f9a-9c0a-9c07a7a67ece\" }",
+        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e04172a0-55e3-475b-a070-a383d22f3bb0\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"e04172a0-55e3-475b-a070-a383d22f3bb0\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c8feb285-1846-467e-acac-5fbfb5e401cf\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:12 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"c8feb285-1846-467e-acac-5fbfb5e401cf\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"74c364c5-4838-43e3-af27-16c1f88e771e\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:12 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"74c364c5-4838-43e3-af27-16c1f88e771e\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e72fd27c-41fd-4597-a623-0b90dde3751f\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:12 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"e72fd27c-41fd-4597-a623-0b90dde3751f\" }",
+        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"09377390-54dc-4481-a94c-fdc884b3cf03\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:12 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"09377390-54dc-4481-a94c-fdc884b3cf03\" }"
+      ],
+      "time_started": "2024-03-01T19:18:07.756995Z",
+      "time_done": "2024-03-01T19:18:13.548706Z",
+      "collector": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+      "baseboards": [
+        {
+          "part_number": "913-0000006",
+          "serial_number": "BRM44220005"
+        },
+        {
+          "part_number": "913-0000006",
+          "serial_number": "BRM44220011"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220004"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220046"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM42220081"
+        },
+        {
+          "part_number": "913-0000019",
+          "serial_number": "BRM44220001"
+        }
+      ],
+      "cabooses": [
+        {
+          "board": "gimlet-c",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "gimlet-c",
+          "version": "1.0.6"
+        },
+        {
+          "board": "gimlet-c",
+          "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
+          "name": "gimlet-c",
+          "version": "1.0.5"
+        },
+        {
+          "board": "gimlet-c",
+          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+          "name": "gimlet-c",
+          "version": "1.0.8"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "oxide-rot-1",
+          "version": "1.0.5"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
+          "name": "oxide-rot-1",
+          "version": "1.0.4"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
+          "name": "oxide-rot-1",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+          "name": "oxide-rot-1",
+          "version": "1.0.6"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+          "name": "oxide-rot-1",
+          "version": "1.0.0"
+        },
+        {
+          "board": "oxide-rot-1",
+          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+          "name": "oxide-rot-1-dev",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+          "name": "sidecar-b",
+          "version": "1.0.6"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
+          "name": "sidecar-b",
+          "version": "0.0.0-git"
+        },
+        {
+          "board": "sidecar-b",
+          "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
+          "name": "sidecar-b",
+          "version": "0.0.0-git"
+        }
+      ],
+      "rot_pages": [
+        {
+          "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+        },
+        {
+          "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+        },
+        {
+          "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+        },
+        {
+          "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+        },
+        {
+          "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+        },
+        {
+          "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+        },
+        {
+          "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+        }
+      ],
+      "sps": [
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220005"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:11.402668Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "switch",
+            "sp_slot": 0,
+            "baseboard_revision": 4,
+            "hubris_archive": "9184be93590e655e",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220011"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:12.176222Z",
+            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "sp_type": "switch",
+            "sp_slot": 1,
+            "baseboard_revision": 4,
+            "hubris_archive": "9184be93590e655e",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:10.088383Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 17,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:10.815174Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 15,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:08.950823Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 14,
+            "baseboard_revision": 6,
+            "hubris_archive": "5c745f4b4fa66f47",
+            "power_state": "A0"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:07.775665Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "sp_type": "sled",
+            "sp_slot": 16,
+            "baseboard_revision": 6,
+            "hubris_archive": "3c670b44fdeef4af",
+            "power_state": "A0"
+          }
+        ]
+      ],
+      "rots": [
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220005"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:11.402668Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "4599e4d7bc9b2164e8c0f0f7ce4bcae2bd35d635c8450854ea9a94ce49a2ca0f",
+            "slot_b_sha3_256_digest": "1e42a0761e5bd1d9f9a5e2c44acc34e7b7e007792977cdb0523d271db2e80c8c"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000006",
+            "serial_number": "BRM44220011"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:12.176222Z",
+            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "active_slot": {
+              "slot": "b"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "92e07077972c83af6872ec817fb88d74a36f6956276505e9771302c5bb7ec2a8",
+            "slot_b_sha3_256_digest": "fb25825eaf97f8f2307afe3caebf0b70865499f6d116db3aff382c505b4d8dd7"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:10.088383Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
+            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:10.815174Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
+            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:08.950823Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "b"
+            },
+            "persistent_boot_preference": {
+              "slot": "b"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
+            "slot_b_sha3_256_digest": "d416d5ba9d0d3bd495f0f5fcd2aecc28cd83b53d1a8f94a28241d85b30761451"
+          }
+        ],
+        [
+          {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          {
+            "time_collected": "2024-03-01T19:18:07.775665Z",
+            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "active_slot": {
+              "slot": "a"
+            },
+            "persistent_boot_preference": {
+              "slot": "a"
+            },
+            "pending_persistent_boot_preference": null,
+            "transient_boot_preference": null,
+            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
+            "slot_b_sha3_256_digest": "4ec328195a23e86a1545d5ee576e73834aae81fb79830a54d0503475875f1760"
+          }
+        ]
+      ],
+      "cabooses_found": {
+        "SpSlot0": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.404063Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "sidecar-b",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:12.391683Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "sidecar-b",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.181795Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.816554Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.149044Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "gimlet-c",
+                "version": "1.0.8"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:07.778136Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ]
+        ],
+        "SpSlot1": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.405544Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
+                "name": "sidecar-b",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:12.425692Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "sidecar-b",
+                "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
+                "name": "sidecar-b",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.229994Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.817966Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.402701Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "gimlet-c",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:07.836136Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "gimlet-c",
+                "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
+                "name": "gimlet-c",
+                "version": "1.0.5"
+              }
+            }
+          ]
+        ],
+        "RotSlotA": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.612405Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+                "name": "oxide-rot-1-dev",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:12.633926Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
+                "name": "oxide-rot-1",
+                "version": "0.0.0-git"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.482137Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.072878Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.718444Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.338242Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ]
+        ],
+        "RotSlotB": [
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220005"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.819343Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000006",
+              "serial_number": "BRM44220011"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:12.889742Z",
+              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
+                "name": "oxide-rot-1",
+                "version": "1.0.0"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.734260Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.324822Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
+                "name": "oxide-rot-1",
+                "version": "1.0.5"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.971380Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
+                "name": "oxide-rot-1",
+                "version": "1.0.6"
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.798226Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "caboose": {
+                "board": "oxide-rot-1",
+                "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
+                "name": "oxide-rot-1",
+                "version": "1.0.4"
+              }
+            }
+          ]
+        ]
+      },
+      "rot_pages_found": {
+        "Cmpa": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.750544Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.341290Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:09.987916Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.829941Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ]
+        ],
+        "CfpaActive": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.766709Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.357423Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.004467Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.861702Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ],
+        "CfpaInactive": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.782522Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.373385Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.021096Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.893754Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ],
+        "CfpaScratch": [
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220004"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.798536Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220046"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:11.395564Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM42220081"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:10.040934Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "913-0000019",
+              "serial_number": "BRM44220001"
+            },
+            {
+              "time_collected": "2024-03-01T19:18:08.909759Z",
+              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "page": {
+                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+              }
+            }
+          ]
+        ]
+      },
+      "sled_agents": {
+        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+          "time_collected": "2024-03-01T19:18:13.421323Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM44220001"
+          },
+          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+          "time_collected": "2024-03-01T19:18:13.464287Z",
+          "source": "http://[fd00:1122:3344:121::1]:12345",
+          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220004"
+          },
+          "sled_agent_address": "[fd00:1122:3344:121::1]:12345",
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+          "time_collected": "2024-03-01T19:18:13.506888Z",
+          "source": "http://[fd00:1122:3344:102::1]:12345",
+          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220046"
+          },
+          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        },
+        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+          "time_collected": "2024-03-01T19:18:13.546670Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
+          "baseboard_id": {
+            "part_number": "913-0000019",
+            "serial_number": "BRM42220081"
+          },
+          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 128,
+          "usable_physical_ram": 1086608900096,
+          "reservoir_size": 869286281216
+        }
+      },
+      "omicron_zones": {
+        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+          "time_collected": "2024-03-01T19:18:13.422277Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "0706860a-3801-42a7-8000-fff741f7665b",
+                "underlay_address": "fd00:1122:3344:101::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
+                  }
+                }
+              },
+              {
+                "id": "34a279a2-a025-41ff-a392-1addac400f38",
+                "underlay_address": "fd00:1122:3344:101::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  }
+                }
+              },
+              {
+                "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
+                "underlay_address": "fd00:1122:3344:1::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  },
+                  "dns_address": "[fd00:1122:3344:1::1]:53",
+                  "gz_address": "fd00:1122:3344:1::2",
+                  "gz_address_index": 0,
+                  "http_address": "[fd00:1122:3344:1::1]:5353"
+                }
+              },
+              {
+                "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+                "underlay_address": "fd00:1122:3344:101::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.2",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:101::5]:12221",
+                  "nic": {
+                    "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
+                    "ip": "172.30.2.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
+                    },
+                    "mac": "A8:40:25:FF:E1:17",
+                    "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
+                "underlay_address": "fd00:1122:3344:101::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+                  }
+                }
+              },
+              {
+                "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
+                "underlay_address": "fd00:1122:3344:101::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
+                  }
+                }
+              },
+              {
+                "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
+                "underlay_address": "fd00:1122:3344:101::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                  }
+                }
+              },
+              {
+                "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
+                "underlay_address": "fd00:1122:3344:101::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  }
+                }
+              },
+              {
+                "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
+                "underlay_address": "fd00:1122:3344:101::4",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::4]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                  }
+                }
+              },
+              {
+                "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
+                "underlay_address": "fd00:1122:3344:101::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
+                  }
+                }
+              },
+              {
+                "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
+                "underlay_address": "fd00:1122:3344:101::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
+                  }
+                }
+              },
+              {
+                "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
+                "underlay_address": "fd00:1122:3344:101::6",
+                "zone_type": {
+                  "type": "oximeter",
+                  "address": "[fd00:1122:3344:101::6]:12223"
+                }
+              },
+              {
+                "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                "underlay_address": "fd00:1122:3344:101::11",
+                "zone_type": {
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:101::11]:123",
+                  "dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "nic": {
+                    "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
+                    "ip": "172.30.3.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
+                    },
+                    "mac": "A8:40:25:FF:E7:A8",
+                    "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100
+                  },
+                  "ntp_servers": [
+                    "ntp.eng.oxide.computer"
+                  ],
+                  "snat_cfg": {
+                    "ip": "172.20.28.5",
+                    "first_port": 0,
+                    "last_port": 16383
+                  }
+                }
+              },
+              {
+                "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
+                "underlay_address": "fd00:1122:3344:101::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                  }
+                }
+              },
+              {
+                "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
+                "underlay_address": "fd00:1122:3344:101::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
+                  }
+                }
+              },
+              {
+                "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
+                "underlay_address": "fd00:1122:3344:101::7",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:101::7]:17000"
+                }
+              }
+            ]
+          }
+        },
+        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+          "time_collected": "2024-03-01T19:18:13.464998Z",
+          "source": "http://[fd00:1122:3344:121::1]:12345",
+          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
+          "zones": {
+            "generation": 3,
+            "zones": [
+              {
+                "id": "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
+                "underlay_address": "fd00:1122:3344:121::28",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::28]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3"
+                  }
+                }
+              },
+              {
+                "id": "12431d3e-1516-4822-98b3-3711ff912e69",
+                "underlay_address": "fd00:1122:3344:121::22",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::22]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_0622b2cf-297e-4356-b94c-d74309f443ba"
+                  }
+                }
+              },
+              {
+                "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
+                "underlay_address": "fd00:1122:3344:121::21",
+                "zone_type": {
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:121::21]:123",
+                  "dns_servers": [
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
+                  ],
+                  "ntp_servers": [
+                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                  ]
+                }
+              },
+              {
+                "id": "56414d76-9b8c-4e15-acf1-cd269c2d8638",
+                "underlay_address": "fd00:1122:3344:121::25",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::25]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_574728ca-1559-48f3-a46c-1e608966e6a0"
+                  }
+                }
+              },
+              {
+                "id": "891cb9da-301d-40cc-b93d-5fcb350fa968",
+                "underlay_address": "fd00:1122:3344:121::29",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::29]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
+                  }
+                }
+              },
+              {
+                "id": "d9014907-24c0-4c33-a475-db15942f4f8f",
+                "underlay_address": "fd00:1122:3344:121::23",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::23]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37"
+                  }
+                }
+              },
+              {
+                "id": "e950de25-7306-4cf7-a6f4-6db11adaa310",
+                "underlay_address": "fd00:1122:3344:121::27",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::27]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c"
+                  }
+                }
+              },
+              {
+                "id": "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
+                "underlay_address": "fd00:1122:3344:121::26",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::26]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945"
+                  }
+                }
+              },
+              {
+                "id": "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00",
+                "underlay_address": "fd00:1122:3344:121::24",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:121::24]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1"
+                  }
+                }
+              }
+            ]
+          }
+        },
+        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+          "time_collected": "2024-03-01T19:18:13.507881Z",
+          "source": "http://[fd00:1122:3344:102::1]:12345",
+          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
+                "underlay_address": "fd00:1122:3344:102::11",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::11]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                  }
+                }
+              },
+              {
+                "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
+                "underlay_address": "fd00:1122:3344:102::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
+                  }
+                }
+              },
+              {
+                "id": "479811e7-6409-45d4-b294-5333534e47ed",
+                "underlay_address": "fd00:1122:3344:102::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                  }
+                }
+              },
+              {
+                "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
+                "underlay_address": "fd00:1122:3344:102::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                  }
+                }
+              },
+              {
+                "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
+                "underlay_address": "fd00:1122:3344:2::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  },
+                  "dns_address": "[fd00:1122:3344:2::1]:53",
+                  "gz_address": "fd00:1122:3344:2::2",
+                  "gz_address_index": 1,
+                  "http_address": "[fd00:1122:3344:2::1]:5353"
+                }
+              },
+              {
+                "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+                "underlay_address": "fd00:1122:3344:102::7",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:102::7]:17000"
+                }
+              },
+              {
+                "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
+                "underlay_address": "fd00:1122:3344:102::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "658076ed-fb43-49e7-a813-df3d23497848",
+                "underlay_address": "fd00:1122:3344:102::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
+                  }
+                }
+              },
+              {
+                "id": "6edb836e-b100-4080-bb03-738d6441d571",
+                "underlay_address": "fd00:1122:3344:102::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
+                  }
+                }
+              },
+              {
+                "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                "underlay_address": "fd00:1122:3344:102::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.3",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:102::5]:12221",
+                  "nic": {
+                    "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
+                    "ip": "172.30.2.6",
+                    "kind": {
+                      "type": "service",
+                      "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
+                    },
+                    "mac": "A8:40:25:FF:B2:AF",
+                    "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
+                "underlay_address": "fd00:1122:3344:102::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                  }
+                }
+              },
+              {
+                "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+                "underlay_address": "fd00:1122:3344:102::4",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::4]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                  }
+                }
+              },
+              {
+                "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
+                "underlay_address": "fd00:1122:3344:102::6",
+                "zone_type": {
+                  "type": "clickhouse",
+                  "address": "[fd00:1122:3344:102::6]:8123",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
+                "underlay_address": "fd00:1122:3344:102::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
+                  }
+                }
+              },
+              {
+                "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
+                "underlay_address": "fd00:1122:3344:102::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  }
+                }
+              },
+              {
+                "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                "underlay_address": "fd00:1122:3344:102::12",
+                "zone_type": {
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:102::12]:123",
+                  "dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "nic": {
+                    "id": "9f03c82c-3192-498d-978d-a175049ac389",
+                    "ip": "172.30.3.6",
+                    "kind": {
+                      "type": "service",
+                      "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
+                    },
+                    "mac": "A8:40:25:FF:E4:11",
+                    "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100
+                  },
+                  "ntp_servers": [
+                    "ntp.eng.oxide.computer"
+                  ],
+                  "snat_cfg": {
+                    "ip": "172.20.28.6",
+                    "first_port": 16384,
+                    "last_port": 32767
+                  }
+                }
+              },
+              {
+                "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
+                "underlay_address": "fd00:1122:3344:102::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:102::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
+                  }
+                }
+              }
+            ]
+          }
+        },
+        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+          "time_collected": "2024-03-01T19:18:13.547769Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
+          "zones": {
+            "generation": 5,
+            "zones": [
+              {
+                "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
+                "underlay_address": "fd00:1122:3344:103::a",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::a]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
+                  }
+                }
+              },
+              {
+                "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
+                "underlay_address": "fd00:1122:3344:103::10",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::10]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
+                  }
+                }
+              },
+              {
+                "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
+                "underlay_address": "fd00:1122:3344:3::1",
+                "zone_type": {
+                  "type": "internal_dns",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  },
+                  "dns_address": "[fd00:1122:3344:3::1]:53",
+                  "gz_address": "fd00:1122:3344:3::2",
+                  "gz_address_index": 2,
+                  "http_address": "[fd00:1122:3344:3::1]:5353"
+                }
+              },
+              {
+                "id": "4b74c418-0a08-4489-b652-e97267c1c220",
+                "underlay_address": "fd00:1122:3344:103::7",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::7]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  }
+                }
+              },
+              {
+                "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+                "underlay_address": "fd00:1122:3344:103::5",
+                "zone_type": {
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "172.20.28.4",
+                  "external_tls": true,
+                  "internal_address": "[fd00:1122:3344:103::5]:12221",
+                  "nic": {
+                    "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
+                    "ip": "172.30.2.7",
+                    "kind": {
+                      "type": "service",
+                      "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
+                    },
+                    "mac": "A8:40:25:FF:96:05",
+                    "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
+                "underlay_address": "fd00:1122:3344:103::6",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:103::6]:17000"
+                }
+              },
+              {
+                "id": "921958c0-7761-4294-945d-2a078327bf2c",
+                "underlay_address": "fd00:1122:3344:103::8",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::8]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
+                  }
+                }
+              },
+              {
+                "id": "9df22034-59f6-431f-82e7-258fe983879b",
+                "underlay_address": "fd00:1122:3344:103::4",
+                "zone_type": {
+                  "type": "external_dns",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  },
+                  "dns_address": "172.20.28.1:53",
+                  "http_address": "[fd00:1122:3344:103::4]:5353",
+                  "nic": {
+                    "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
+                    "ip": "172.30.1.5",
+                    "kind": {
+                      "type": "service",
+                      "id": "9df22034-59f6-431f-82e7-258fe983879b"
+                    },
+                    "mac": "A8:40:25:FF:90:12",
+                    "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
+                    "primary": true,
+                    "slot": 0,
+                    "subnet": "172.30.1.0/24",
+                    "vni": 100
+                  }
+                }
+              },
+              {
+                "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
+                "underlay_address": "fd00:1122:3344:103::11",
+                "zone_type": {
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:103::11]:123",
+                  "dns_servers": [
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
+                  ],
+                  "ntp_servers": [
+                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                  ]
+                }
+              },
+              {
+                "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
+                "underlay_address": "fd00:1122:3344:103::9",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::9]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
+                  }
+                }
+              },
+              {
+                "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
+                "underlay_address": "fd00:1122:3344:103::d",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::d]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
+                  }
+                }
+              },
+              {
+                "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
+                "underlay_address": "fd00:1122:3344:103::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
+                  }
+                }
+              },
+              {
+                "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
+                "underlay_address": "fd00:1122:3344:103::e",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::e]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
+                  }
+                }
+              },
+              {
+                "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
+                "underlay_address": "fd00:1122:3344:103::3",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:103::3]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  }
+                }
+              },
+              {
+                "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
+                "underlay_address": "fd00:1122:3344:103::f",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::f]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
+                  }
+                }
+              },
+              {
+                "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
+                "underlay_address": "fd00:1122:3344:103::c",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::c]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                  }
+                }
+              }
+            ]
+          }
+        }
+      }
+    }
+  ],
+  "blueprints": [
+    {
+      "id": "486de160-c8f3-4600-acca-b0c78e33aca4",
+      "omicron_zones": {
+        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+          "generation": 5,
+          "zones": [
+            {
+              "id": "0706860a-3801-42a7-8000-fff741f7665b",
+              "underlay_address": "fd00:1122:3344:101::c",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::c]:32345",
+                "dataset": {
+                  "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
+                }
+              }
+            },
+            {
+              "id": "34a279a2-a025-41ff-a392-1addac400f38",
+              "underlay_address": "fd00:1122:3344:101::3",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:101::3]:32221",
+                "dataset": {
+                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                }
+              }
+            },
+            {
+              "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
+              "underlay_address": "fd00:1122:3344:1::1",
+              "zone_type": {
+                "type": "internal_dns",
+                "dataset": {
+                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                },
+                "dns_address": "[fd00:1122:3344:1::1]:53",
+                "gz_address": "fd00:1122:3344:1::2",
+                "gz_address_index": 0,
+                "http_address": "[fd00:1122:3344:1::1]:5353"
+              }
+            },
+            {
+              "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+              "underlay_address": "fd00:1122:3344:101::5",
+              "zone_type": {
+                "type": "nexus",
+                "external_dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "external_ip": "172.20.28.2",
+                "external_tls": true,
+                "internal_address": "[fd00:1122:3344:101::5]:12221",
+                "nic": {
+                  "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
+                  "ip": "172.30.2.5",
+                  "kind": {
+                    "type": "service",
+                    "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
+                  },
+                  "mac": "A8:40:25:FF:E1:17",
+                  "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.2.0/24",
+                  "vni": 100
+                }
+              }
+            },
+            {
+              "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
+              "underlay_address": "fd00:1122:3344:101::10",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::10]:32345",
+                "dataset": {
+                  "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+                }
+              }
+            },
+            {
+              "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
+              "underlay_address": "fd00:1122:3344:101::f",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::f]:32345",
+                "dataset": {
+                  "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
+                }
+              }
+            },
+            {
+              "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
+              "underlay_address": "fd00:1122:3344:101::9",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::9]:32345",
+                "dataset": {
+                  "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                }
+              }
+            },
+            {
+              "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
+              "underlay_address": "fd00:1122:3344:101::8",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::8]:32345",
+                "dataset": {
+                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                }
+              }
+            },
+            {
+              "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
+              "underlay_address": "fd00:1122:3344:101::4",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:101::4]:32221",
+                "dataset": {
+                  "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                }
+              }
+            },
+            {
+              "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
+              "underlay_address": "fd00:1122:3344:101::d",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::d]:32345",
+                "dataset": {
+                  "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
+                }
+              }
+            },
+            {
+              "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
+              "underlay_address": "fd00:1122:3344:101::a",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::a]:32345",
+                "dataset": {
+                  "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
+                }
+              }
+            },
+            {
+              "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
+              "underlay_address": "fd00:1122:3344:101::6",
+              "zone_type": {
+                "type": "oximeter",
+                "address": "[fd00:1122:3344:101::6]:12223"
+              }
+            },
+            {
+              "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
+              "underlay_address": "fd00:1122:3344:101::11",
+              "zone_type": {
+                "type": "boundary_ntp",
+                "address": "[fd00:1122:3344:101::11]:123",
+                "dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "nic": {
+                  "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
+                  "ip": "172.30.3.5",
+                  "kind": {
+                    "type": "service",
+                    "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
+                  },
+                  "mac": "A8:40:25:FF:E7:A8",
+                  "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.3.0/24",
+                  "vni": 100
+                },
+                "ntp_servers": [
+                  "ntp.eng.oxide.computer"
+                ],
+                "snat_cfg": {
+                  "ip": "172.20.28.5",
+                  "first_port": 0,
+                  "last_port": 16383
+                }
+              }
+            },
+            {
+              "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
+              "underlay_address": "fd00:1122:3344:101::e",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::e]:32345",
+                "dataset": {
+                  "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                }
+              }
+            },
+            {
+              "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
+              "underlay_address": "fd00:1122:3344:101::b",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::b]:32345",
+                "dataset": {
+                  "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
+                }
+              }
+            },
+            {
+              "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
+              "underlay_address": "fd00:1122:3344:101::7",
+              "zone_type": {
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:101::7]:17000"
+              }
+            }
+          ]
+        },
+        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+          "generation": 1,
+          "zones": []
+        },
+        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+          "generation": 5,
+          "zones": [
+            {
+              "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
+              "underlay_address": "fd00:1122:3344:102::11",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::11]:32345",
+                "dataset": {
+                  "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                }
+              }
+            },
+            {
+              "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
+              "underlay_address": "fd00:1122:3344:102::e",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::e]:32345",
+                "dataset": {
+                  "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
+                }
+              }
+            },
+            {
+              "id": "479811e7-6409-45d4-b294-5333534e47ed",
+              "underlay_address": "fd00:1122:3344:102::9",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::9]:32345",
+                "dataset": {
+                  "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                }
+              }
+            },
+            {
+              "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
+              "underlay_address": "fd00:1122:3344:102::a",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::a]:32345",
+                "dataset": {
+                  "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                }
+              }
+            },
+            {
+              "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
+              "underlay_address": "fd00:1122:3344:2::1",
+              "zone_type": {
+                "type": "internal_dns",
+                "dataset": {
+                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                },
+                "dns_address": "[fd00:1122:3344:2::1]:53",
+                "gz_address": "fd00:1122:3344:2::2",
+                "gz_address_index": 1,
+                "http_address": "[fd00:1122:3344:2::1]:5353"
+              }
+            },
+            {
+              "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+              "underlay_address": "fd00:1122:3344:102::7",
+              "zone_type": {
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:102::7]:17000"
+              }
+            },
+            {
+              "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
+              "underlay_address": "fd00:1122:3344:102::8",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::8]:32345",
+                "dataset": {
+                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                }
+              }
+            },
+            {
+              "id": "658076ed-fb43-49e7-a813-df3d23497848",
+              "underlay_address": "fd00:1122:3344:102::10",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::10]:32345",
+                "dataset": {
+                  "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
+                }
+              }
+            },
+            {
+              "id": "6edb836e-b100-4080-bb03-738d6441d571",
+              "underlay_address": "fd00:1122:3344:102::c",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::c]:32345",
+                "dataset": {
+                  "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
+                }
+              }
+            },
+            {
+              "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+              "underlay_address": "fd00:1122:3344:102::5",
+              "zone_type": {
+                "type": "nexus",
+                "external_dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "external_ip": "172.20.28.3",
+                "external_tls": true,
+                "internal_address": "[fd00:1122:3344:102::5]:12221",
+                "nic": {
+                  "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
+                  "ip": "172.30.2.6",
+                  "kind": {
+                    "type": "service",
+                    "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
+                  },
+                  "mac": "A8:40:25:FF:B2:AF",
+                  "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.2.0/24",
+                  "vni": 100
+                }
+              }
+            },
+            {
+              "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
+              "underlay_address": "fd00:1122:3344:102::d",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::d]:32345",
+                "dataset": {
+                  "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                }
+              }
+            },
+            {
+              "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+              "underlay_address": "fd00:1122:3344:102::4",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:102::4]:32221",
+                "dataset": {
+                  "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                }
+              }
+            },
+            {
+              "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
+              "underlay_address": "fd00:1122:3344:102::6",
+              "zone_type": {
+                "type": "clickhouse",
+                "address": "[fd00:1122:3344:102::6]:8123",
+                "dataset": {
+                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                }
+              }
+            },
+            {
+              "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
+              "underlay_address": "fd00:1122:3344:102::f",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::f]:32345",
+                "dataset": {
+                  "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
+                }
+              }
+            },
+            {
+              "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
+              "underlay_address": "fd00:1122:3344:102::3",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:102::3]:32221",
+                "dataset": {
+                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                }
+              }
+            },
+            {
+              "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+              "underlay_address": "fd00:1122:3344:102::12",
+              "zone_type": {
+                "type": "boundary_ntp",
+                "address": "[fd00:1122:3344:102::12]:123",
+                "dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "nic": {
+                  "id": "9f03c82c-3192-498d-978d-a175049ac389",
+                  "ip": "172.30.3.6",
+                  "kind": {
+                    "type": "service",
+                    "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
+                  },
+                  "mac": "A8:40:25:FF:E4:11",
+                  "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.3.0/24",
+                  "vni": 100
+                },
+                "ntp_servers": [
+                  "ntp.eng.oxide.computer"
+                ],
+                "snat_cfg": {
+                  "ip": "172.20.28.6",
+                  "first_port": 16384,
+                  "last_port": 32767
+                }
+              }
+            },
+            {
+              "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
+              "underlay_address": "fd00:1122:3344:102::b",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::b]:32345",
+                "dataset": {
+                  "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
+                }
+              }
+            }
+          ]
+        },
+        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+          "generation": 5,
+          "zones": [
+            {
+              "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
+              "underlay_address": "fd00:1122:3344:103::a",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::a]:32345",
+                "dataset": {
+                  "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
+                }
+              }
+            },
+            {
+              "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
+              "underlay_address": "fd00:1122:3344:103::10",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::10]:32345",
+                "dataset": {
+                  "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
+                }
+              }
+            },
+            {
+              "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
+              "underlay_address": "fd00:1122:3344:3::1",
+              "zone_type": {
+                "type": "internal_dns",
+                "dataset": {
+                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                },
+                "dns_address": "[fd00:1122:3344:3::1]:53",
+                "gz_address": "fd00:1122:3344:3::2",
+                "gz_address_index": 2,
+                "http_address": "[fd00:1122:3344:3::1]:5353"
+              }
+            },
+            {
+              "id": "4b74c418-0a08-4489-b652-e97267c1c220",
+              "underlay_address": "fd00:1122:3344:103::7",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::7]:32345",
+                "dataset": {
+                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                }
+              }
+            },
+            {
+              "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+              "underlay_address": "fd00:1122:3344:103::5",
+              "zone_type": {
+                "type": "nexus",
+                "external_dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "external_ip": "172.20.28.4",
+                "external_tls": true,
+                "internal_address": "[fd00:1122:3344:103::5]:12221",
+                "nic": {
+                  "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
+                  "ip": "172.30.2.7",
+                  "kind": {
+                    "type": "service",
+                    "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
+                  },
+                  "mac": "A8:40:25:FF:96:05",
+                  "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.2.0/24",
+                  "vni": 100
+                }
+              }
+            },
+            {
+              "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
+              "underlay_address": "fd00:1122:3344:103::6",
+              "zone_type": {
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:103::6]:17000"
+              }
+            },
+            {
+              "id": "921958c0-7761-4294-945d-2a078327bf2c",
+              "underlay_address": "fd00:1122:3344:103::8",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::8]:32345",
+                "dataset": {
+                  "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
+                }
+              }
+            },
+            {
+              "id": "9df22034-59f6-431f-82e7-258fe983879b",
+              "underlay_address": "fd00:1122:3344:103::4",
+              "zone_type": {
+                "type": "external_dns",
+                "dataset": {
+                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                },
+                "dns_address": "172.20.28.1:53",
+                "http_address": "[fd00:1122:3344:103::4]:5353",
+                "nic": {
+                  "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
+                  "ip": "172.30.1.5",
+                  "kind": {
+                    "type": "service",
+                    "id": "9df22034-59f6-431f-82e7-258fe983879b"
+                  },
+                  "mac": "A8:40:25:FF:90:12",
+                  "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.1.0/24",
+                  "vni": 100
+                }
+              }
+            },
+            {
+              "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
+              "underlay_address": "fd00:1122:3344:103::11",
+              "zone_type": {
+                "type": "internal_ntp",
+                "address": "[fd00:1122:3344:103::11]:123",
+                "dns_servers": [
+                  "fd00:1122:3344:1::1",
+                  "fd00:1122:3344:2::1",
+                  "fd00:1122:3344:3::1"
+                ],
+                "ntp_servers": [
+                  "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                  "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                ]
+              }
+            },
+            {
+              "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
+              "underlay_address": "fd00:1122:3344:103::9",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::9]:32345",
+                "dataset": {
+                  "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
+                }
+              }
+            },
+            {
+              "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
+              "underlay_address": "fd00:1122:3344:103::d",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::d]:32345",
+                "dataset": {
+                  "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
+                }
+              }
+            },
+            {
+              "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
+              "underlay_address": "fd00:1122:3344:103::b",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::b]:32345",
+                "dataset": {
+                  "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
+                }
+              }
+            },
+            {
+              "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
+              "underlay_address": "fd00:1122:3344:103::e",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::e]:32345",
+                "dataset": {
+                  "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
+                }
+              }
+            },
+            {
+              "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
+              "underlay_address": "fd00:1122:3344:103::3",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:103::3]:32221",
+                "dataset": {
+                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                }
+              }
+            },
+            {
+              "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
+              "underlay_address": "fd00:1122:3344:103::f",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::f]:32345",
+                "dataset": {
+                  "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
+                }
+              }
+            },
+            {
+              "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
+              "underlay_address": "fd00:1122:3344:103::c",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::c]:32345",
+                "dataset": {
+                  "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                }
+              }
+            }
+          ]
+        }
+      },
+      "zones_in_service": [
+        "0706860a-3801-42a7-8000-fff741f7665b",
+        "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
+        "185e6b00-7e18-456c-9dfc-edd0194ac207",
+        "18b99404-d8b2-4463-bc99-638d9b678ab3",
+        "1fc78bc7-feca-4a10-9beb-58b53fda5210",
+        "34a279a2-a025-41ff-a392-1addac400f38",
+        "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
+        "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
+        "479811e7-6409-45d4-b294-5333534e47ed",
+        "4b74c418-0a08-4489-b652-e97267c1c220",
+        "54c947d2-6355-453c-80fc-8f49cc2129ee",
+        "54d1b693-bc4d-4c47-9009-c6f624073801",
+        "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+        "5b743072-9369-45d7-b7c9-dbdf736301f8",
+        "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+        "60d64b70-89f5-4774-9e41-2c79c4173911",
+        "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
+        "658076ed-fb43-49e7-a813-df3d23497848",
+        "67852e14-ce7f-4e50-a61d-4059d0c057b9",
+        "6ace1a69-d55a-45f0-a127-99c90f1b070c",
+        "6edb836e-b100-4080-bb03-738d6441d571",
+        "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+        "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
+        "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
+        "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
+        "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+        "86ee7359-aa35-41d9-8a76-54961bc516a1",
+        "921958c0-7761-4294-945d-2a078327bf2c",
+        "9241b9da-8188-4aae-a9dc-62acb453618f",
+        "925ea436-fad4-4fff-8ae0-24edd725a2b0",
+        "9df22034-59f6-431f-82e7-258fe983879b",
+        "a781a349-62b2-41f0-9dd7-c64d290aef82",
+        "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
+        "be272d1e-4d34-4478-94b4-a76f5fcbef36",
+        "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
+        "c67814bf-9972-4bb0-a666-24778ca70a16",
+        "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+        "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
+        "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
+        "d42fc874-8416-4212-b5a0-febba4c6b0ac",
+        "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
+        "d630cca7-e3f7-47a8-aee8-ad0790895abc",
+        "d8c51c79-46ff-44bd-80be-08d6283f4e21",
+        "da27ee6a-9841-46aa-acda-7cce3e04cb60",
+        "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
+        "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
+        "e09525cf-9c87-4d05-8860-1356dd0dee8a",
+        "f33a2375-212d-42e4-b539-6079aeb4e5b7",
+        "f7264a3e-e8e7-4476-8df3-76c9d00a383b"
+      ],
+      "parent_blueprint_id": null,
+      "internal_dns_version": 1,
+      "time_created": "2024-03-01T19:06:56.467313Z",
+      "creator": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+      "comment": "from collection df8caafd-c444-4f65-a304-b9ceb62a96c2"
+    },
+    {
+      "id": "6c127695-ba15-408d-a992-325a1a888380",
+      "omicron_zones": {
+        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+          "generation": 5,
+          "zones": [
+            {
+              "id": "0706860a-3801-42a7-8000-fff741f7665b",
+              "underlay_address": "fd00:1122:3344:101::c",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::c]:32345",
+                "dataset": {
+                  "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
+                }
+              }
+            },
+            {
+              "id": "34a279a2-a025-41ff-a392-1addac400f38",
+              "underlay_address": "fd00:1122:3344:101::3",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:101::3]:32221",
+                "dataset": {
+                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                }
+              }
+            },
+            {
+              "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
+              "underlay_address": "fd00:1122:3344:1::1",
+              "zone_type": {
+                "type": "internal_dns",
+                "dataset": {
+                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                },
+                "dns_address": "[fd00:1122:3344:1::1]:53",
+                "gz_address": "fd00:1122:3344:1::2",
+                "gz_address_index": 0,
+                "http_address": "[fd00:1122:3344:1::1]:5353"
+              }
+            },
+            {
+              "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+              "underlay_address": "fd00:1122:3344:101::5",
+              "zone_type": {
+                "type": "nexus",
+                "external_dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "external_ip": "172.20.28.2",
+                "external_tls": true,
+                "internal_address": "[fd00:1122:3344:101::5]:12221",
+                "nic": {
+                  "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
+                  "ip": "172.30.2.5",
+                  "kind": {
+                    "type": "service",
+                    "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
+                  },
+                  "mac": "A8:40:25:FF:E1:17",
+                  "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.2.0/24",
+                  "vni": 100
+                }
+              }
+            },
+            {
+              "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
+              "underlay_address": "fd00:1122:3344:101::10",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::10]:32345",
+                "dataset": {
+                  "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+                }
+              }
+            },
+            {
+              "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
+              "underlay_address": "fd00:1122:3344:101::f",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::f]:32345",
+                "dataset": {
+                  "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
+                }
+              }
+            },
+            {
+              "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
+              "underlay_address": "fd00:1122:3344:101::9",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::9]:32345",
+                "dataset": {
+                  "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                }
+              }
+            },
+            {
+              "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
+              "underlay_address": "fd00:1122:3344:101::8",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::8]:32345",
+                "dataset": {
+                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                }
+              }
+            },
+            {
+              "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
+              "underlay_address": "fd00:1122:3344:101::4",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:101::4]:32221",
+                "dataset": {
+                  "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                }
+              }
+            },
+            {
+              "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
+              "underlay_address": "fd00:1122:3344:101::d",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::d]:32345",
+                "dataset": {
+                  "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
+                }
+              }
+            },
+            {
+              "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
+              "underlay_address": "fd00:1122:3344:101::a",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::a]:32345",
+                "dataset": {
+                  "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
+                }
+              }
+            },
+            {
+              "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
+              "underlay_address": "fd00:1122:3344:101::6",
+              "zone_type": {
+                "type": "oximeter",
+                "address": "[fd00:1122:3344:101::6]:12223"
+              }
+            },
+            {
+              "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
+              "underlay_address": "fd00:1122:3344:101::11",
+              "zone_type": {
+                "type": "boundary_ntp",
+                "address": "[fd00:1122:3344:101::11]:123",
+                "dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "nic": {
+                  "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
+                  "ip": "172.30.3.5",
+                  "kind": {
+                    "type": "service",
+                    "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
+                  },
+                  "mac": "A8:40:25:FF:E7:A8",
+                  "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.3.0/24",
+                  "vni": 100
+                },
+                "ntp_servers": [
+                  "ntp.eng.oxide.computer"
+                ],
+                "snat_cfg": {
+                  "ip": "172.20.28.5",
+                  "first_port": 0,
+                  "last_port": 16383
+                }
+              }
+            },
+            {
+              "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
+              "underlay_address": "fd00:1122:3344:101::e",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::e]:32345",
+                "dataset": {
+                  "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                }
+              }
+            },
+            {
+              "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
+              "underlay_address": "fd00:1122:3344:101::b",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::b]:32345",
+                "dataset": {
+                  "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
+                }
+              }
+            },
+            {
+              "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
+              "underlay_address": "fd00:1122:3344:101::7",
+              "zone_type": {
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:101::7]:17000"
+              }
+            }
+          ]
+        },
+        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+          "generation": 2,
+          "zones": [
+            {
+              "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
+              "underlay_address": "fd00:1122:3344:121::21",
+              "zone_type": {
+                "type": "internal_ntp",
+                "address": "[fd00:1122:3344:121::21]:123",
+                "dns_servers": [
+                  "fd00:1122:3344:1::1",
+                  "fd00:1122:3344:2::1",
+                  "fd00:1122:3344:3::1"
+                ],
+                "ntp_servers": [
+                  "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                  "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                ]
+              }
+            }
+          ]
+        },
+        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+          "generation": 5,
+          "zones": [
+            {
+              "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
+              "underlay_address": "fd00:1122:3344:102::11",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::11]:32345",
+                "dataset": {
+                  "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                }
+              }
+            },
+            {
+              "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
+              "underlay_address": "fd00:1122:3344:102::e",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::e]:32345",
+                "dataset": {
+                  "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
+                }
+              }
+            },
+            {
+              "id": "479811e7-6409-45d4-b294-5333534e47ed",
+              "underlay_address": "fd00:1122:3344:102::9",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::9]:32345",
+                "dataset": {
+                  "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                }
+              }
+            },
+            {
+              "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
+              "underlay_address": "fd00:1122:3344:102::a",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::a]:32345",
+                "dataset": {
+                  "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                }
+              }
+            },
+            {
+              "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
+              "underlay_address": "fd00:1122:3344:2::1",
+              "zone_type": {
+                "type": "internal_dns",
+                "dataset": {
+                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                },
+                "dns_address": "[fd00:1122:3344:2::1]:53",
+                "gz_address": "fd00:1122:3344:2::2",
+                "gz_address_index": 1,
+                "http_address": "[fd00:1122:3344:2::1]:5353"
+              }
+            },
+            {
+              "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+              "underlay_address": "fd00:1122:3344:102::7",
+              "zone_type": {
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:102::7]:17000"
+              }
+            },
+            {
+              "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
+              "underlay_address": "fd00:1122:3344:102::8",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::8]:32345",
+                "dataset": {
+                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                }
+              }
+            },
+            {
+              "id": "658076ed-fb43-49e7-a813-df3d23497848",
+              "underlay_address": "fd00:1122:3344:102::10",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::10]:32345",
+                "dataset": {
+                  "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
+                }
+              }
+            },
+            {
+              "id": "6edb836e-b100-4080-bb03-738d6441d571",
+              "underlay_address": "fd00:1122:3344:102::c",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::c]:32345",
+                "dataset": {
+                  "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
+                }
+              }
+            },
+            {
+              "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+              "underlay_address": "fd00:1122:3344:102::5",
+              "zone_type": {
+                "type": "nexus",
+                "external_dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "external_ip": "172.20.28.3",
+                "external_tls": true,
+                "internal_address": "[fd00:1122:3344:102::5]:12221",
+                "nic": {
+                  "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
+                  "ip": "172.30.2.6",
+                  "kind": {
+                    "type": "service",
+                    "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
+                  },
+                  "mac": "A8:40:25:FF:B2:AF",
+                  "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.2.0/24",
+                  "vni": 100
+                }
+              }
+            },
+            {
+              "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
+              "underlay_address": "fd00:1122:3344:102::d",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::d]:32345",
+                "dataset": {
+                  "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                }
+              }
+            },
+            {
+              "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+              "underlay_address": "fd00:1122:3344:102::4",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:102::4]:32221",
+                "dataset": {
+                  "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                }
+              }
+            },
+            {
+              "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
+              "underlay_address": "fd00:1122:3344:102::6",
+              "zone_type": {
+                "type": "clickhouse",
+                "address": "[fd00:1122:3344:102::6]:8123",
+                "dataset": {
+                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                }
+              }
+            },
+            {
+              "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
+              "underlay_address": "fd00:1122:3344:102::f",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::f]:32345",
+                "dataset": {
+                  "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
+                }
+              }
+            },
+            {
+              "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
+              "underlay_address": "fd00:1122:3344:102::3",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:102::3]:32221",
+                "dataset": {
+                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                }
+              }
+            },
+            {
+              "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+              "underlay_address": "fd00:1122:3344:102::12",
+              "zone_type": {
+                "type": "boundary_ntp",
+                "address": "[fd00:1122:3344:102::12]:123",
+                "dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "nic": {
+                  "id": "9f03c82c-3192-498d-978d-a175049ac389",
+                  "ip": "172.30.3.6",
+                  "kind": {
+                    "type": "service",
+                    "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
+                  },
+                  "mac": "A8:40:25:FF:E4:11",
+                  "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.3.0/24",
+                  "vni": 100
+                },
+                "ntp_servers": [
+                  "ntp.eng.oxide.computer"
+                ],
+                "snat_cfg": {
+                  "ip": "172.20.28.6",
+                  "first_port": 16384,
+                  "last_port": 32767
+                }
+              }
+            },
+            {
+              "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
+              "underlay_address": "fd00:1122:3344:102::b",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::b]:32345",
+                "dataset": {
+                  "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
+                }
+              }
+            }
+          ]
+        },
+        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+          "generation": 5,
+          "zones": [
+            {
+              "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
+              "underlay_address": "fd00:1122:3344:103::a",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::a]:32345",
+                "dataset": {
+                  "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
+                }
+              }
+            },
+            {
+              "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
+              "underlay_address": "fd00:1122:3344:103::10",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::10]:32345",
+                "dataset": {
+                  "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
+                }
+              }
+            },
+            {
+              "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
+              "underlay_address": "fd00:1122:3344:3::1",
+              "zone_type": {
+                "type": "internal_dns",
+                "dataset": {
+                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                },
+                "dns_address": "[fd00:1122:3344:3::1]:53",
+                "gz_address": "fd00:1122:3344:3::2",
+                "gz_address_index": 2,
+                "http_address": "[fd00:1122:3344:3::1]:5353"
+              }
+            },
+            {
+              "id": "4b74c418-0a08-4489-b652-e97267c1c220",
+              "underlay_address": "fd00:1122:3344:103::7",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::7]:32345",
+                "dataset": {
+                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                }
+              }
+            },
+            {
+              "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+              "underlay_address": "fd00:1122:3344:103::5",
+              "zone_type": {
+                "type": "nexus",
+                "external_dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "external_ip": "172.20.28.4",
+                "external_tls": true,
+                "internal_address": "[fd00:1122:3344:103::5]:12221",
+                "nic": {
+                  "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
+                  "ip": "172.30.2.7",
+                  "kind": {
+                    "type": "service",
+                    "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
+                  },
+                  "mac": "A8:40:25:FF:96:05",
+                  "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.2.0/24",
+                  "vni": 100
+                }
+              }
+            },
+            {
+              "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
+              "underlay_address": "fd00:1122:3344:103::6",
+              "zone_type": {
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:103::6]:17000"
+              }
+            },
+            {
+              "id": "921958c0-7761-4294-945d-2a078327bf2c",
+              "underlay_address": "fd00:1122:3344:103::8",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::8]:32345",
+                "dataset": {
+                  "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
+                }
+              }
+            },
+            {
+              "id": "9df22034-59f6-431f-82e7-258fe983879b",
+              "underlay_address": "fd00:1122:3344:103::4",
+              "zone_type": {
+                "type": "external_dns",
+                "dataset": {
+                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                },
+                "dns_address": "172.20.28.1:53",
+                "http_address": "[fd00:1122:3344:103::4]:5353",
+                "nic": {
+                  "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
+                  "ip": "172.30.1.5",
+                  "kind": {
+                    "type": "service",
+                    "id": "9df22034-59f6-431f-82e7-258fe983879b"
+                  },
+                  "mac": "A8:40:25:FF:90:12",
+                  "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.1.0/24",
+                  "vni": 100
+                }
+              }
+            },
+            {
+              "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
+              "underlay_address": "fd00:1122:3344:103::11",
+              "zone_type": {
+                "type": "internal_ntp",
+                "address": "[fd00:1122:3344:103::11]:123",
+                "dns_servers": [
+                  "fd00:1122:3344:1::1",
+                  "fd00:1122:3344:2::1",
+                  "fd00:1122:3344:3::1"
+                ],
+                "ntp_servers": [
+                  "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                  "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                ]
+              }
+            },
+            {
+              "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
+              "underlay_address": "fd00:1122:3344:103::9",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::9]:32345",
+                "dataset": {
+                  "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
+                }
+              }
+            },
+            {
+              "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
+              "underlay_address": "fd00:1122:3344:103::d",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::d]:32345",
+                "dataset": {
+                  "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
+                }
+              }
+            },
+            {
+              "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
+              "underlay_address": "fd00:1122:3344:103::b",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::b]:32345",
+                "dataset": {
+                  "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
+                }
+              }
+            },
+            {
+              "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
+              "underlay_address": "fd00:1122:3344:103::e",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::e]:32345",
+                "dataset": {
+                  "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
+                }
+              }
+            },
+            {
+              "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
+              "underlay_address": "fd00:1122:3344:103::3",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:103::3]:32221",
+                "dataset": {
+                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                }
+              }
+            },
+            {
+              "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
+              "underlay_address": "fd00:1122:3344:103::f",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::f]:32345",
+                "dataset": {
+                  "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
+                }
+              }
+            },
+            {
+              "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
+              "underlay_address": "fd00:1122:3344:103::c",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::c]:32345",
+                "dataset": {
+                  "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                }
+              }
+            }
+          ]
+        }
+      },
+      "zones_in_service": [
+        "0706860a-3801-42a7-8000-fff741f7665b",
+        "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
+        "185e6b00-7e18-456c-9dfc-edd0194ac207",
+        "18b99404-d8b2-4463-bc99-638d9b678ab3",
+        "1fc78bc7-feca-4a10-9beb-58b53fda5210",
+        "2d5c29f7-4915-4876-82f8-e388b15c5067",
+        "34a279a2-a025-41ff-a392-1addac400f38",
+        "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
+        "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
+        "479811e7-6409-45d4-b294-5333534e47ed",
+        "4b74c418-0a08-4489-b652-e97267c1c220",
+        "54c947d2-6355-453c-80fc-8f49cc2129ee",
+        "54d1b693-bc4d-4c47-9009-c6f624073801",
+        "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+        "5b743072-9369-45d7-b7c9-dbdf736301f8",
+        "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+        "60d64b70-89f5-4774-9e41-2c79c4173911",
+        "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
+        "658076ed-fb43-49e7-a813-df3d23497848",
+        "67852e14-ce7f-4e50-a61d-4059d0c057b9",
+        "6ace1a69-d55a-45f0-a127-99c90f1b070c",
+        "6edb836e-b100-4080-bb03-738d6441d571",
+        "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+        "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
+        "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
+        "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
+        "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+        "86ee7359-aa35-41d9-8a76-54961bc516a1",
+        "921958c0-7761-4294-945d-2a078327bf2c",
+        "9241b9da-8188-4aae-a9dc-62acb453618f",
+        "925ea436-fad4-4fff-8ae0-24edd725a2b0",
+        "9df22034-59f6-431f-82e7-258fe983879b",
+        "a781a349-62b2-41f0-9dd7-c64d290aef82",
+        "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
+        "be272d1e-4d34-4478-94b4-a76f5fcbef36",
+        "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
+        "c67814bf-9972-4bb0-a666-24778ca70a16",
+        "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+        "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
+        "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
+        "d42fc874-8416-4212-b5a0-febba4c6b0ac",
+        "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
+        "d630cca7-e3f7-47a8-aee8-ad0790895abc",
+        "d8c51c79-46ff-44bd-80be-08d6283f4e21",
+        "da27ee6a-9841-46aa-acda-7cce3e04cb60",
+        "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
+        "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
+        "e09525cf-9c87-4d05-8860-1356dd0dee8a",
+        "f33a2375-212d-42e4-b539-6079aeb4e5b7",
+        "f7264a3e-e8e7-4476-8df3-76c9d00a383b"
+      ],
+      "parent_blueprint_id": "486de160-c8f3-4600-acca-b0c78e33aca4",
+      "internal_dns_version": 1,
+      "time_created": "2024-03-01T19:07:58.105708Z",
+      "creator": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+      "comment": "sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f: add NTP zone"
+    },
+    {
+      "id": "cfa03594-f438-4736-a416-6307f7bf8f2e",
+      "omicron_zones": {
+        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+          "generation": 5,
+          "zones": [
+            {
+              "id": "0706860a-3801-42a7-8000-fff741f7665b",
+              "underlay_address": "fd00:1122:3344:101::c",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::c]:32345",
+                "dataset": {
+                  "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
+                }
+              }
+            },
+            {
+              "id": "34a279a2-a025-41ff-a392-1addac400f38",
+              "underlay_address": "fd00:1122:3344:101::3",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:101::3]:32221",
+                "dataset": {
+                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                }
+              }
+            },
+            {
+              "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
+              "underlay_address": "fd00:1122:3344:1::1",
+              "zone_type": {
+                "type": "internal_dns",
+                "dataset": {
+                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                },
+                "dns_address": "[fd00:1122:3344:1::1]:53",
+                "gz_address": "fd00:1122:3344:1::2",
+                "gz_address_index": 0,
+                "http_address": "[fd00:1122:3344:1::1]:5353"
+              }
+            },
+            {
+              "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+              "underlay_address": "fd00:1122:3344:101::5",
+              "zone_type": {
+                "type": "nexus",
+                "external_dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "external_ip": "172.20.28.2",
+                "external_tls": true,
+                "internal_address": "[fd00:1122:3344:101::5]:12221",
+                "nic": {
+                  "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
+                  "ip": "172.30.2.5",
+                  "kind": {
+                    "type": "service",
+                    "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
+                  },
+                  "mac": "A8:40:25:FF:E1:17",
+                  "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.2.0/24",
+                  "vni": 100
+                }
+              }
+            },
+            {
+              "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
+              "underlay_address": "fd00:1122:3344:101::10",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::10]:32345",
+                "dataset": {
+                  "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+                }
+              }
+            },
+            {
+              "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
+              "underlay_address": "fd00:1122:3344:101::f",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::f]:32345",
+                "dataset": {
+                  "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
+                }
+              }
+            },
+            {
+              "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
+              "underlay_address": "fd00:1122:3344:101::9",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::9]:32345",
+                "dataset": {
+                  "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                }
+              }
+            },
+            {
+              "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
+              "underlay_address": "fd00:1122:3344:101::8",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::8]:32345",
+                "dataset": {
+                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                }
+              }
+            },
+            {
+              "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
+              "underlay_address": "fd00:1122:3344:101::4",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:101::4]:32221",
+                "dataset": {
+                  "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                }
+              }
+            },
+            {
+              "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
+              "underlay_address": "fd00:1122:3344:101::d",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::d]:32345",
+                "dataset": {
+                  "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
+                }
+              }
+            },
+            {
+              "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
+              "underlay_address": "fd00:1122:3344:101::a",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::a]:32345",
+                "dataset": {
+                  "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
+                }
+              }
+            },
+            {
+              "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
+              "underlay_address": "fd00:1122:3344:101::6",
+              "zone_type": {
+                "type": "oximeter",
+                "address": "[fd00:1122:3344:101::6]:12223"
+              }
+            },
+            {
+              "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
+              "underlay_address": "fd00:1122:3344:101::11",
+              "zone_type": {
+                "type": "boundary_ntp",
+                "address": "[fd00:1122:3344:101::11]:123",
+                "dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "nic": {
+                  "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
+                  "ip": "172.30.3.5",
+                  "kind": {
+                    "type": "service",
+                    "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
+                  },
+                  "mac": "A8:40:25:FF:E7:A8",
+                  "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.3.0/24",
+                  "vni": 100
+                },
+                "ntp_servers": [
+                  "ntp.eng.oxide.computer"
+                ],
+                "snat_cfg": {
+                  "ip": "172.20.28.5",
+                  "first_port": 0,
+                  "last_port": 16383
+                }
+              }
+            },
+            {
+              "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
+              "underlay_address": "fd00:1122:3344:101::e",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::e]:32345",
+                "dataset": {
+                  "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                }
+              }
+            },
+            {
+              "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
+              "underlay_address": "fd00:1122:3344:101::b",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::b]:32345",
+                "dataset": {
+                  "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
+                }
+              }
+            },
+            {
+              "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
+              "underlay_address": "fd00:1122:3344:101::7",
+              "zone_type": {
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:101::7]:17000"
+              }
+            }
+          ]
+        },
+        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+          "generation": 3,
+          "zones": [
+            {
+              "id": "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
+              "underlay_address": "fd00:1122:3344:121::28",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:121::28]:32345",
+                "dataset": {
+                  "pool_name": "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3"
+                }
+              }
+            },
+            {
+              "id": "12431d3e-1516-4822-98b3-3711ff912e69",
+              "underlay_address": "fd00:1122:3344:121::22",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:121::22]:32345",
+                "dataset": {
+                  "pool_name": "oxp_0622b2cf-297e-4356-b94c-d74309f443ba"
+                }
+              }
+            },
+            {
+              "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
+              "underlay_address": "fd00:1122:3344:121::21",
+              "zone_type": {
+                "type": "internal_ntp",
+                "address": "[fd00:1122:3344:121::21]:123",
+                "dns_servers": [
+                  "fd00:1122:3344:1::1",
+                  "fd00:1122:3344:2::1",
+                  "fd00:1122:3344:3::1"
+                ],
+                "ntp_servers": [
+                  "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                  "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                ]
+              }
+            },
+            {
+              "id": "56414d76-9b8c-4e15-acf1-cd269c2d8638",
+              "underlay_address": "fd00:1122:3344:121::25",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:121::25]:32345",
+                "dataset": {
+                  "pool_name": "oxp_574728ca-1559-48f3-a46c-1e608966e6a0"
+                }
+              }
+            },
+            {
+              "id": "891cb9da-301d-40cc-b93d-5fcb350fa968",
+              "underlay_address": "fd00:1122:3344:121::29",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:121::29]:32345",
+                "dataset": {
+                  "pool_name": "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
+                }
+              }
+            },
+            {
+              "id": "d9014907-24c0-4c33-a475-db15942f4f8f",
+              "underlay_address": "fd00:1122:3344:121::23",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:121::23]:32345",
+                "dataset": {
+                  "pool_name": "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37"
+                }
+              }
+            },
+            {
+              "id": "e950de25-7306-4cf7-a6f4-6db11adaa310",
+              "underlay_address": "fd00:1122:3344:121::27",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:121::27]:32345",
+                "dataset": {
+                  "pool_name": "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c"
+                }
+              }
+            },
+            {
+              "id": "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
+              "underlay_address": "fd00:1122:3344:121::26",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:121::26]:32345",
+                "dataset": {
+                  "pool_name": "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945"
+                }
+              }
+            },
+            {
+              "id": "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00",
+              "underlay_address": "fd00:1122:3344:121::24",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:121::24]:32345",
+                "dataset": {
+                  "pool_name": "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1"
+                }
+              }
+            }
+          ]
+        },
+        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+          "generation": 5,
+          "zones": [
+            {
+              "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
+              "underlay_address": "fd00:1122:3344:102::11",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::11]:32345",
+                "dataset": {
+                  "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                }
+              }
+            },
+            {
+              "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
+              "underlay_address": "fd00:1122:3344:102::e",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::e]:32345",
+                "dataset": {
+                  "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
+                }
+              }
+            },
+            {
+              "id": "479811e7-6409-45d4-b294-5333534e47ed",
+              "underlay_address": "fd00:1122:3344:102::9",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::9]:32345",
+                "dataset": {
+                  "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                }
+              }
+            },
+            {
+              "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
+              "underlay_address": "fd00:1122:3344:102::a",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::a]:32345",
+                "dataset": {
+                  "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                }
+              }
+            },
+            {
+              "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
+              "underlay_address": "fd00:1122:3344:2::1",
+              "zone_type": {
+                "type": "internal_dns",
+                "dataset": {
+                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                },
+                "dns_address": "[fd00:1122:3344:2::1]:53",
+                "gz_address": "fd00:1122:3344:2::2",
+                "gz_address_index": 1,
+                "http_address": "[fd00:1122:3344:2::1]:5353"
+              }
+            },
+            {
+              "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+              "underlay_address": "fd00:1122:3344:102::7",
+              "zone_type": {
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:102::7]:17000"
+              }
+            },
+            {
+              "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
+              "underlay_address": "fd00:1122:3344:102::8",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::8]:32345",
+                "dataset": {
+                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                }
+              }
+            },
+            {
+              "id": "658076ed-fb43-49e7-a813-df3d23497848",
+              "underlay_address": "fd00:1122:3344:102::10",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::10]:32345",
+                "dataset": {
+                  "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
+                }
+              }
+            },
+            {
+              "id": "6edb836e-b100-4080-bb03-738d6441d571",
+              "underlay_address": "fd00:1122:3344:102::c",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::c]:32345",
+                "dataset": {
+                  "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
+                }
+              }
+            },
+            {
+              "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+              "underlay_address": "fd00:1122:3344:102::5",
+              "zone_type": {
+                "type": "nexus",
+                "external_dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "external_ip": "172.20.28.3",
+                "external_tls": true,
+                "internal_address": "[fd00:1122:3344:102::5]:12221",
+                "nic": {
+                  "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
+                  "ip": "172.30.2.6",
+                  "kind": {
+                    "type": "service",
+                    "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
+                  },
+                  "mac": "A8:40:25:FF:B2:AF",
+                  "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.2.0/24",
+                  "vni": 100
+                }
+              }
+            },
+            {
+              "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
+              "underlay_address": "fd00:1122:3344:102::d",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::d]:32345",
+                "dataset": {
+                  "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                }
+              }
+            },
+            {
+              "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+              "underlay_address": "fd00:1122:3344:102::4",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:102::4]:32221",
+                "dataset": {
+                  "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                }
+              }
+            },
+            {
+              "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
+              "underlay_address": "fd00:1122:3344:102::6",
+              "zone_type": {
+                "type": "clickhouse",
+                "address": "[fd00:1122:3344:102::6]:8123",
+                "dataset": {
+                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                }
+              }
+            },
+            {
+              "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
+              "underlay_address": "fd00:1122:3344:102::f",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::f]:32345",
+                "dataset": {
+                  "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
+                }
+              }
+            },
+            {
+              "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
+              "underlay_address": "fd00:1122:3344:102::3",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:102::3]:32221",
+                "dataset": {
+                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                }
+              }
+            },
+            {
+              "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+              "underlay_address": "fd00:1122:3344:102::12",
+              "zone_type": {
+                "type": "boundary_ntp",
+                "address": "[fd00:1122:3344:102::12]:123",
+                "dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "nic": {
+                  "id": "9f03c82c-3192-498d-978d-a175049ac389",
+                  "ip": "172.30.3.6",
+                  "kind": {
+                    "type": "service",
+                    "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
+                  },
+                  "mac": "A8:40:25:FF:E4:11",
+                  "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.3.0/24",
+                  "vni": 100
+                },
+                "ntp_servers": [
+                  "ntp.eng.oxide.computer"
+                ],
+                "snat_cfg": {
+                  "ip": "172.20.28.6",
+                  "first_port": 16384,
+                  "last_port": 32767
+                }
+              }
+            },
+            {
+              "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
+              "underlay_address": "fd00:1122:3344:102::b",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::b]:32345",
+                "dataset": {
+                  "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
+                }
+              }
+            }
+          ]
+        },
+        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+          "generation": 5,
+          "zones": [
+            {
+              "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
+              "underlay_address": "fd00:1122:3344:103::a",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::a]:32345",
+                "dataset": {
+                  "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
+                }
+              }
+            },
+            {
+              "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
+              "underlay_address": "fd00:1122:3344:103::10",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::10]:32345",
+                "dataset": {
+                  "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
+                }
+              }
+            },
+            {
+              "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
+              "underlay_address": "fd00:1122:3344:3::1",
+              "zone_type": {
+                "type": "internal_dns",
+                "dataset": {
+                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                },
+                "dns_address": "[fd00:1122:3344:3::1]:53",
+                "gz_address": "fd00:1122:3344:3::2",
+                "gz_address_index": 2,
+                "http_address": "[fd00:1122:3344:3::1]:5353"
+              }
+            },
+            {
+              "id": "4b74c418-0a08-4489-b652-e97267c1c220",
+              "underlay_address": "fd00:1122:3344:103::7",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::7]:32345",
+                "dataset": {
+                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                }
+              }
+            },
+            {
+              "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+              "underlay_address": "fd00:1122:3344:103::5",
+              "zone_type": {
+                "type": "nexus",
+                "external_dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "external_ip": "172.20.28.4",
+                "external_tls": true,
+                "internal_address": "[fd00:1122:3344:103::5]:12221",
+                "nic": {
+                  "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
+                  "ip": "172.30.2.7",
+                  "kind": {
+                    "type": "service",
+                    "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
+                  },
+                  "mac": "A8:40:25:FF:96:05",
+                  "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.2.0/24",
+                  "vni": 100
+                }
+              }
+            },
+            {
+              "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
+              "underlay_address": "fd00:1122:3344:103::6",
+              "zone_type": {
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:103::6]:17000"
+              }
+            },
+            {
+              "id": "921958c0-7761-4294-945d-2a078327bf2c",
+              "underlay_address": "fd00:1122:3344:103::8",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::8]:32345",
+                "dataset": {
+                  "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
+                }
+              }
+            },
+            {
+              "id": "9df22034-59f6-431f-82e7-258fe983879b",
+              "underlay_address": "fd00:1122:3344:103::4",
+              "zone_type": {
+                "type": "external_dns",
+                "dataset": {
+                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                },
+                "dns_address": "172.20.28.1:53",
+                "http_address": "[fd00:1122:3344:103::4]:5353",
+                "nic": {
+                  "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
+                  "ip": "172.30.1.5",
+                  "kind": {
+                    "type": "service",
+                    "id": "9df22034-59f6-431f-82e7-258fe983879b"
+                  },
+                  "mac": "A8:40:25:FF:90:12",
+                  "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
+                  "primary": true,
+                  "slot": 0,
+                  "subnet": "172.30.1.0/24",
+                  "vni": 100
+                }
+              }
+            },
+            {
+              "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
+              "underlay_address": "fd00:1122:3344:103::11",
+              "zone_type": {
+                "type": "internal_ntp",
+                "address": "[fd00:1122:3344:103::11]:123",
+                "dns_servers": [
+                  "fd00:1122:3344:1::1",
+                  "fd00:1122:3344:2::1",
+                  "fd00:1122:3344:3::1"
+                ],
+                "ntp_servers": [
+                  "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
+                  "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                ]
+              }
+            },
+            {
+              "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
+              "underlay_address": "fd00:1122:3344:103::9",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::9]:32345",
+                "dataset": {
+                  "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
+                }
+              }
+            },
+            {
+              "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
+              "underlay_address": "fd00:1122:3344:103::d",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::d]:32345",
+                "dataset": {
+                  "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
+                }
+              }
+            },
+            {
+              "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
+              "underlay_address": "fd00:1122:3344:103::b",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::b]:32345",
+                "dataset": {
+                  "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
+                }
+              }
+            },
+            {
+              "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
+              "underlay_address": "fd00:1122:3344:103::e",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::e]:32345",
+                "dataset": {
+                  "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
+                }
+              }
+            },
+            {
+              "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
+              "underlay_address": "fd00:1122:3344:103::3",
+              "zone_type": {
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:103::3]:32221",
+                "dataset": {
+                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                }
+              }
+            },
+            {
+              "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
+              "underlay_address": "fd00:1122:3344:103::f",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::f]:32345",
+                "dataset": {
+                  "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
+                }
+              }
+            },
+            {
+              "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
+              "underlay_address": "fd00:1122:3344:103::c",
+              "zone_type": {
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::c]:32345",
+                "dataset": {
+                  "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                }
+              }
+            }
+          ]
+        }
+      },
+      "zones_in_service": [
+        "0706860a-3801-42a7-8000-fff741f7665b",
+        "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
+        "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
+        "12431d3e-1516-4822-98b3-3711ff912e69",
+        "185e6b00-7e18-456c-9dfc-edd0194ac207",
+        "18b99404-d8b2-4463-bc99-638d9b678ab3",
+        "1fc78bc7-feca-4a10-9beb-58b53fda5210",
+        "2d5c29f7-4915-4876-82f8-e388b15c5067",
+        "34a279a2-a025-41ff-a392-1addac400f38",
+        "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
+        "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
+        "479811e7-6409-45d4-b294-5333534e47ed",
+        "4b74c418-0a08-4489-b652-e97267c1c220",
+        "54c947d2-6355-453c-80fc-8f49cc2129ee",
+        "54d1b693-bc4d-4c47-9009-c6f624073801",
+        "56414d76-9b8c-4e15-acf1-cd269c2d8638",
+        "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+        "5b743072-9369-45d7-b7c9-dbdf736301f8",
+        "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+        "60d64b70-89f5-4774-9e41-2c79c4173911",
+        "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
+        "658076ed-fb43-49e7-a813-df3d23497848",
+        "67852e14-ce7f-4e50-a61d-4059d0c057b9",
+        "6ace1a69-d55a-45f0-a127-99c90f1b070c",
+        "6edb836e-b100-4080-bb03-738d6441d571",
+        "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+        "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
+        "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
+        "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
+        "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+        "86ee7359-aa35-41d9-8a76-54961bc516a1",
+        "891cb9da-301d-40cc-b93d-5fcb350fa968",
+        "921958c0-7761-4294-945d-2a078327bf2c",
+        "9241b9da-8188-4aae-a9dc-62acb453618f",
+        "925ea436-fad4-4fff-8ae0-24edd725a2b0",
+        "9df22034-59f6-431f-82e7-258fe983879b",
+        "a781a349-62b2-41f0-9dd7-c64d290aef82",
+        "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
+        "be272d1e-4d34-4478-94b4-a76f5fcbef36",
+        "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
+        "c67814bf-9972-4bb0-a666-24778ca70a16",
+        "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+        "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
+        "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
+        "d42fc874-8416-4212-b5a0-febba4c6b0ac",
+        "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
+        "d630cca7-e3f7-47a8-aee8-ad0790895abc",
+        "d8c51c79-46ff-44bd-80be-08d6283f4e21",
+        "d9014907-24c0-4c33-a475-db15942f4f8f",
+        "da27ee6a-9841-46aa-acda-7cce3e04cb60",
+        "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
+        "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
+        "e09525cf-9c87-4d05-8860-1356dd0dee8a",
+        "e950de25-7306-4cf7-a6f4-6db11adaa310",
+        "f33a2375-212d-42e4-b539-6079aeb4e5b7",
+        "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
+        "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
+        "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00"
+      ],
+      "parent_blueprint_id": "6c127695-ba15-408d-a992-325a1a888380",
+      "internal_dns_version": 2,
+      "time_created": "2024-03-01T19:08:52.730520Z",
+      "creator": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+      "comment": "sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f: add zones"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-complex-stdout b/dev-tools/reconfigurator-cli/tests/output/cmd-complex-stdout
new file mode 100644
index 0000000000..0cc5fe01ba
--- /dev/null
+++ b/dev-tools/reconfigurator-cli/tests/output/cmd-complex-stdout
@@ -0,0 +1,385 @@
+> sled-list
+ID NZPOOLS SUBNET 
+
+> 
+
+> inventory-list
+ID NERRORS TIME_DONE 
+
+> 
+
+> blueprint-list
+ID 
+
+> 
+
+> file-contents tests/input/complex.json
+sled: 1762267a-a0cb-4c9a-88b4-8ce828cbc021 (subnet: fd00:1122:3344:101::/64, zpools: 9)
+sled: a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f (subnet: fd00:1122:3344:121::/64, zpools: 8)
+sled: a6634f64-f6fb-426c-b00b-9b30aed9f53a (subnet: fd00:1122:3344:102::/64, zpools: 10)
+sled: fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89 (subnet: fd00:1122:3344:103::/64, zpools: 10)
+collection: 83ed3949-d221-44f3-a901-02e9ff16d2a5 (errors: 8, completed at: 2024-03-01T19:17:10.594Z)
+collection: 0a8dc8ff-9013-43aa-9f06-9540b24ea902 (errors: 8, completed at: 2024-03-01T19:17:11.153Z)
+collection: cd3547d7-f172-47d2-9bbf-9730e88b0559 (errors: 8, completed at: 2024-03-01T19:18:07.486Z)
+collection: 623f7e48-6c47-4e17-8aa3-cebe98ca4287 (errors: 8, completed at: 2024-03-01T19:18:12.110Z)
+collection: 4e9fbfc2-eb1d-4d45-bdb9-64d965a490c5 (errors: 8, completed at: 2024-03-01T19:18:13.548Z)
+blueprint:  486de160-c8f3-4600-acca-b0c78e33aca4 (created at: 2024-03-01 19:06:56.467313 UTC)
+blueprint:  6c127695-ba15-408d-a992-325a1a888380 (created at: 2024-03-01 19:07:58.105708 UTC)
+blueprint:  cfa03594-f438-4736-a416-6307f7bf8f2e (created at: 2024-03-01 19:08:52.730520 UTC)
+
+
+> 
+
+> load tests/input/complex.json 83ed3949-d221-44f3-a901-02e9ff16d2a5
+using collection 83ed3949-d221-44f3-a901-02e9ff16d2a5 as source of sled inventory data
+sled 1762267a-a0cb-4c9a-88b4-8ce828cbc021 loaded
+sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f loaded
+sled a6634f64-f6fb-426c-b00b-9b30aed9f53a loaded
+sled fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89 loaded
+collection 83ed3949-d221-44f3-a901-02e9ff16d2a5 loaded
+collection 0a8dc8ff-9013-43aa-9f06-9540b24ea902 loaded
+collection cd3547d7-f172-47d2-9bbf-9730e88b0559 loaded
+collection 623f7e48-6c47-4e17-8aa3-cebe98ca4287 loaded
+collection 4e9fbfc2-eb1d-4d45-bdb9-64d965a490c5 loaded
+blueprint 486de160-c8f3-4600-acca-b0c78e33aca4 loaded
+blueprint 6c127695-ba15-408d-a992-325a1a888380 loaded
+blueprint cfa03594-f438-4736-a416-6307f7bf8f2e loaded
+loaded data from "tests/input/complex.json"
+
+
+> 
+
+> sled-list
+ID                                   NZPOOLS SUBNET                  
+1762267a-a0cb-4c9a-88b4-8ce828cbc021 9       fd00:1122:3344:101::/64 
+a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f 8       fd00:1122:3344:121::/64 
+a6634f64-f6fb-426c-b00b-9b30aed9f53a 10      fd00:1122:3344:102::/64 
+fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89 10      fd00:1122:3344:103::/64 
+
+> 
+
+> inventory-list
+ID                                   NERRORS TIME_DONE                
+83ed3949-d221-44f3-a901-02e9ff16d2a5 8       2024-03-01T19:17:10.594Z 
+0a8dc8ff-9013-43aa-9f06-9540b24ea902 8       2024-03-01T19:17:11.153Z 
+cd3547d7-f172-47d2-9bbf-9730e88b0559 8       2024-03-01T19:18:07.486Z 
+623f7e48-6c47-4e17-8aa3-cebe98ca4287 8       2024-03-01T19:18:12.110Z 
+4e9fbfc2-eb1d-4d45-bdb9-64d965a490c5 8       2024-03-01T19:18:13.548Z 
+
+> 
+
+> blueprint-list
+ID                                   
+486de160-c8f3-4600-acca-b0c78e33aca4 
+6c127695-ba15-408d-a992-325a1a888380 
+cfa03594-f438-4736-a416-6307f7bf8f2e 
+
+> 
+
+> blueprint-show cfa03594-f438-4736-a416-6307f7bf8f2e
+blueprint  cfa03594-f438-4736-a416-6307f7bf8f2e
+parent:    6c127695-ba15-408d-a992-325a1a888380
+created by 54c947d2-6355-453c-80fc-8f49cc2129ee (likely a Nexus instance)
+created at 2024-03-01T19:08:52.730Z
+internal DNS version: 2
+comment: sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f: add zones
+zones:
+
+  sled 1762267a-a0cb-4c9a-88b4-8ce828cbc021: Omicron zones at generation 5
+    0706860a-3801-42a7-8000-fff741f7665b in service     crucible
+    34a279a2-a025-41ff-a392-1addac400f38 in service     cockroach_db
+    3d344935-6f48-405f-8794-2c6e8fa4b1c2 in service     internal_dns
+    59c3ed4f-3670-44b1-9ea0-9a6162a08629 in service     nexus
+    67852e14-ce7f-4e50-a61d-4059d0c057b9 in service     crucible
+    6ace1a69-d55a-45f0-a127-99c90f1b070c in service     crucible
+    84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe in service     crucible
+    854c1544-d0ea-4d65-9cf4-776e91a4fcb5 in service     crucible
+    86ee7359-aa35-41d9-8a76-54961bc516a1 in service     cockroach_db
+    925ea436-fad4-4fff-8ae0-24edd725a2b0 in service     crucible
+    be272d1e-4d34-4478-94b4-a76f5fcbef36 in service     crucible
+    bebd754d-d06c-45f4-99bf-e1aab6253ab8 in service     oximeter
+    d630cca7-e3f7-47a8-aee8-ad0790895abc in service     boundary_ntp
+    d8c51c79-46ff-44bd-80be-08d6283f4e21 in service     crucible
+    dbbc0422-2b23-4d9e-b1db-8003cfc29ad0 in service     crucible
+    f33a2375-212d-42e4-b539-6079aeb4e5b7 in service     crucible_pantry
+  sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f: Omicron zones at generation 3
+    0eca39c6-62e2-4a1f-8ea2-e9332a774a26 in service     crucible
+    12431d3e-1516-4822-98b3-3711ff912e69 in service     crucible
+    2d5c29f7-4915-4876-82f8-e388b15c5067 in service     internal_ntp
+    56414d76-9b8c-4e15-acf1-cd269c2d8638 in service     crucible
+    891cb9da-301d-40cc-b93d-5fcb350fa968 in service     crucible
+    d9014907-24c0-4c33-a475-db15942f4f8f in service     crucible
+    e950de25-7306-4cf7-a6f4-6db11adaa310 in service     crucible
+    fe3dc092-ea23-42d7-9f1d-36eed72b539c in service     crucible
+    ff8cce49-a721-4ad1-b09e-2bd6e4e42d00 in service     crucible
+  sled a6634f64-f6fb-426c-b00b-9b30aed9f53a: Omicron zones at generation 5
+    0ebd2b5b-2dec-4d17-8e23-e468c04dacb6 in service     crucible
+    18b99404-d8b2-4463-bc99-638d9b678ab3 in service     crucible
+    479811e7-6409-45d4-b294-5333534e47ed in service     crucible
+    54d1b693-bc4d-4c47-9009-c6f624073801 in service     crucible
+    5b743072-9369-45d7-b7c9-dbdf736301f8 in service     internal_dns
+    5c4163fa-d6a6-4a0c-9476-3dda135bc9a4 in service     crucible_pantry
+    61a39188-dd82-43f6-a87c-ebb4c2d54ac4 in service     crucible
+    658076ed-fb43-49e7-a813-df3d23497848 in service     crucible
+    6edb836e-b100-4080-bb03-738d6441d571 in service     crucible
+    7f83d784-9c5c-498f-b559-4176bbafbc4b in service     nexus
+    8052938f-73bc-4c54-8ed0-c62f6d77b9fd in service     crucible
+    86bc0a2a-e24a-4bc1-8222-89207a0937f9 in service     cockroach_db
+    9241b9da-8188-4aae-a9dc-62acb453618f in service     clickhouse
+    a781a349-62b2-41f0-9dd7-c64d290aef82 in service     crucible
+    c67814bf-9972-4bb0-a666-24778ca70a16 in service     cockroach_db
+    c74ba7d7-3d5f-4a29-aadc-effe33a90909 in service     boundary_ntp
+    dfc27183-ed85-44c3-ac3c-4427c4cb03e3 in service     crucible
+  sled fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89: Omicron zones at generation 5
+    185e6b00-7e18-456c-9dfc-edd0194ac207 in service     crucible
+    1fc78bc7-feca-4a10-9beb-58b53fda5210 in service     crucible
+    45dccfd6-4674-43c6-a0e7-4dde5f24d90d in service     internal_dns
+    4b74c418-0a08-4489-b652-e97267c1c220 in service     crucible
+    54c947d2-6355-453c-80fc-8f49cc2129ee in service     nexus
+    60d64b70-89f5-4774-9e41-2c79c4173911 in service     crucible_pantry
+    921958c0-7761-4294-945d-2a078327bf2c in service     crucible
+    9df22034-59f6-431f-82e7-258fe983879b in service     external_dns
+    bb21c0c4-c43f-4913-9bf9-707bd7f1769a in service     internal_ntp
+    ccb0d612-9d7c-41ba-bd51-661c3c75ec91 in service     crucible
+    ccef3e95-cae5-4072-9b7d-49e56fa9dff4 in service     crucible
+    d42fc874-8416-4212-b5a0-febba4c6b0ac in service     crucible
+    d5ba7d45-26d7-45ca-8c0a-a560f243b5fa in service     crucible
+    da27ee6a-9841-46aa-acda-7cce3e04cb60 in service     cockroach_db
+    e09525cf-9c87-4d05-8860-1356dd0dee8a in service     crucible
+    f7264a3e-e8e7-4476-8df3-76c9d00a383b in service     crucible
+
+
+> 
+
+> blueprint-diff-inventory 83ed3949-d221-44f3-a901-02e9ff16d2a5 cfa03594-f438-4736-a416-6307f7bf8f2e
+diff collection 83ed3949-d221-44f3-a901-02e9ff16d2a5 blueprint cfa03594-f438-4736-a416-6307f7bf8f2e
+--- collection 83ed3949-d221-44f3-a901-02e9ff16d2a5
++++ blueprint cfa03594-f438-4736-a416-6307f7bf8f2e
+  sled 1762267a-a0cb-4c9a-88b4-8ce828cbc021
+      zone config generation 5
+         zone 0706860a-3801-42a7-8000-fff741f7665b type crucible underlay IP fd00:1122:3344:101::c (unchanged)
+         zone 34a279a2-a025-41ff-a392-1addac400f38 type cockroach_db underlay IP fd00:1122:3344:101::3 (unchanged)
+         zone 3d344935-6f48-405f-8794-2c6e8fa4b1c2 type internal_dns underlay IP fd00:1122:3344:1::1 (unchanged)
+         zone 59c3ed4f-3670-44b1-9ea0-9a6162a08629 type nexus underlay IP fd00:1122:3344:101::5 (unchanged)
+         zone 67852e14-ce7f-4e50-a61d-4059d0c057b9 type crucible underlay IP fd00:1122:3344:101::10 (unchanged)
+         zone 6ace1a69-d55a-45f0-a127-99c90f1b070c type crucible underlay IP fd00:1122:3344:101::f (unchanged)
+         zone 84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe type crucible underlay IP fd00:1122:3344:101::9 (unchanged)
+         zone 854c1544-d0ea-4d65-9cf4-776e91a4fcb5 type crucible underlay IP fd00:1122:3344:101::8 (unchanged)
+         zone 86ee7359-aa35-41d9-8a76-54961bc516a1 type cockroach_db underlay IP fd00:1122:3344:101::4 (unchanged)
+         zone 925ea436-fad4-4fff-8ae0-24edd725a2b0 type crucible underlay IP fd00:1122:3344:101::d (unchanged)
+         zone be272d1e-4d34-4478-94b4-a76f5fcbef36 type crucible underlay IP fd00:1122:3344:101::a (unchanged)
+         zone bebd754d-d06c-45f4-99bf-e1aab6253ab8 type oximeter underlay IP fd00:1122:3344:101::6 (unchanged)
+         zone d630cca7-e3f7-47a8-aee8-ad0790895abc type boundary_ntp underlay IP fd00:1122:3344:101::11 (unchanged)
+         zone d8c51c79-46ff-44bd-80be-08d6283f4e21 type crucible underlay IP fd00:1122:3344:101::e (unchanged)
+         zone dbbc0422-2b23-4d9e-b1db-8003cfc29ad0 type crucible underlay IP fd00:1122:3344:101::b (unchanged)
+         zone f33a2375-212d-42e4-b539-6079aeb4e5b7 type crucible_pantry underlay IP fd00:1122:3344:101::7 (unchanged)
+  sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f
+      zone config generation 3
+         zone 0eca39c6-62e2-4a1f-8ea2-e9332a774a26 type crucible underlay IP fd00:1122:3344:121::28 (unchanged)
+         zone 12431d3e-1516-4822-98b3-3711ff912e69 type crucible underlay IP fd00:1122:3344:121::22 (unchanged)
+         zone 2d5c29f7-4915-4876-82f8-e388b15c5067 type internal_ntp underlay IP fd00:1122:3344:121::21 (unchanged)
+         zone 56414d76-9b8c-4e15-acf1-cd269c2d8638 type crucible underlay IP fd00:1122:3344:121::25 (unchanged)
+         zone 891cb9da-301d-40cc-b93d-5fcb350fa968 type crucible underlay IP fd00:1122:3344:121::29 (unchanged)
+         zone d9014907-24c0-4c33-a475-db15942f4f8f type crucible underlay IP fd00:1122:3344:121::23 (unchanged)
+         zone e950de25-7306-4cf7-a6f4-6db11adaa310 type crucible underlay IP fd00:1122:3344:121::27 (unchanged)
+         zone fe3dc092-ea23-42d7-9f1d-36eed72b539c type crucible underlay IP fd00:1122:3344:121::26 (unchanged)
+         zone ff8cce49-a721-4ad1-b09e-2bd6e4e42d00 type crucible underlay IP fd00:1122:3344:121::24 (unchanged)
+  sled a6634f64-f6fb-426c-b00b-9b30aed9f53a
+      zone config generation 5
+         zone 0ebd2b5b-2dec-4d17-8e23-e468c04dacb6 type crucible underlay IP fd00:1122:3344:102::11 (unchanged)
+         zone 18b99404-d8b2-4463-bc99-638d9b678ab3 type crucible underlay IP fd00:1122:3344:102::e (unchanged)
+         zone 479811e7-6409-45d4-b294-5333534e47ed type crucible underlay IP fd00:1122:3344:102::9 (unchanged)
+         zone 54d1b693-bc4d-4c47-9009-c6f624073801 type crucible underlay IP fd00:1122:3344:102::a (unchanged)
+         zone 5b743072-9369-45d7-b7c9-dbdf736301f8 type internal_dns underlay IP fd00:1122:3344:2::1 (unchanged)
+         zone 5c4163fa-d6a6-4a0c-9476-3dda135bc9a4 type crucible_pantry underlay IP fd00:1122:3344:102::7 (unchanged)
+         zone 61a39188-dd82-43f6-a87c-ebb4c2d54ac4 type crucible underlay IP fd00:1122:3344:102::8 (unchanged)
+         zone 658076ed-fb43-49e7-a813-df3d23497848 type crucible underlay IP fd00:1122:3344:102::10 (unchanged)
+         zone 6edb836e-b100-4080-bb03-738d6441d571 type crucible underlay IP fd00:1122:3344:102::c (unchanged)
+         zone 7f83d784-9c5c-498f-b559-4176bbafbc4b type nexus underlay IP fd00:1122:3344:102::5 (unchanged)
+         zone 8052938f-73bc-4c54-8ed0-c62f6d77b9fd type crucible underlay IP fd00:1122:3344:102::d (unchanged)
+         zone 86bc0a2a-e24a-4bc1-8222-89207a0937f9 type cockroach_db underlay IP fd00:1122:3344:102::4 (unchanged)
+         zone 9241b9da-8188-4aae-a9dc-62acb453618f type clickhouse underlay IP fd00:1122:3344:102::6 (unchanged)
+         zone a781a349-62b2-41f0-9dd7-c64d290aef82 type crucible underlay IP fd00:1122:3344:102::f (unchanged)
+         zone c67814bf-9972-4bb0-a666-24778ca70a16 type cockroach_db underlay IP fd00:1122:3344:102::3 (unchanged)
+         zone c74ba7d7-3d5f-4a29-aadc-effe33a90909 type boundary_ntp underlay IP fd00:1122:3344:102::12 (unchanged)
+         zone dfc27183-ed85-44c3-ac3c-4427c4cb03e3 type crucible underlay IP fd00:1122:3344:102::b (unchanged)
+  sled fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89
+      zone config generation 5
+         zone 185e6b00-7e18-456c-9dfc-edd0194ac207 type crucible underlay IP fd00:1122:3344:103::a (unchanged)
+         zone 1fc78bc7-feca-4a10-9beb-58b53fda5210 type crucible underlay IP fd00:1122:3344:103::10 (unchanged)
+         zone 45dccfd6-4674-43c6-a0e7-4dde5f24d90d type internal_dns underlay IP fd00:1122:3344:3::1 (unchanged)
+         zone 4b74c418-0a08-4489-b652-e97267c1c220 type crucible underlay IP fd00:1122:3344:103::7 (unchanged)
+         zone 54c947d2-6355-453c-80fc-8f49cc2129ee type nexus underlay IP fd00:1122:3344:103::5 (unchanged)
+         zone 60d64b70-89f5-4774-9e41-2c79c4173911 type crucible_pantry underlay IP fd00:1122:3344:103::6 (unchanged)
+         zone 921958c0-7761-4294-945d-2a078327bf2c type crucible underlay IP fd00:1122:3344:103::8 (unchanged)
+         zone 9df22034-59f6-431f-82e7-258fe983879b type external_dns underlay IP fd00:1122:3344:103::4 (unchanged)
+         zone bb21c0c4-c43f-4913-9bf9-707bd7f1769a type internal_ntp underlay IP fd00:1122:3344:103::11 (unchanged)
+         zone ccb0d612-9d7c-41ba-bd51-661c3c75ec91 type crucible underlay IP fd00:1122:3344:103::9 (unchanged)
+         zone ccef3e95-cae5-4072-9b7d-49e56fa9dff4 type crucible underlay IP fd00:1122:3344:103::d (unchanged)
+         zone d42fc874-8416-4212-b5a0-febba4c6b0ac type crucible underlay IP fd00:1122:3344:103::b (unchanged)
+         zone d5ba7d45-26d7-45ca-8c0a-a560f243b5fa type crucible underlay IP fd00:1122:3344:103::e (unchanged)
+         zone da27ee6a-9841-46aa-acda-7cce3e04cb60 type cockroach_db underlay IP fd00:1122:3344:103::3 (unchanged)
+         zone e09525cf-9c87-4d05-8860-1356dd0dee8a type crucible underlay IP fd00:1122:3344:103::f (unchanged)
+         zone f7264a3e-e8e7-4476-8df3-76c9d00a383b type crucible underlay IP fd00:1122:3344:103::c (unchanged)
+
+
+> 
+
+> blueprint-diff 6c127695-ba15-408d-a992-325a1a888380 cfa03594-f438-4736-a416-6307f7bf8f2e
+diff blueprint 6c127695-ba15-408d-a992-325a1a888380 blueprint cfa03594-f438-4736-a416-6307f7bf8f2e
+--- blueprint 6c127695-ba15-408d-a992-325a1a888380
++++ blueprint cfa03594-f438-4736-a416-6307f7bf8f2e
+  sled 1762267a-a0cb-4c9a-88b4-8ce828cbc021
+      zone config generation 5
+         zone 0706860a-3801-42a7-8000-fff741f7665b type crucible underlay IP fd00:1122:3344:101::c (unchanged)
+         zone 34a279a2-a025-41ff-a392-1addac400f38 type cockroach_db underlay IP fd00:1122:3344:101::3 (unchanged)
+         zone 3d344935-6f48-405f-8794-2c6e8fa4b1c2 type internal_dns underlay IP fd00:1122:3344:1::1 (unchanged)
+         zone 59c3ed4f-3670-44b1-9ea0-9a6162a08629 type nexus underlay IP fd00:1122:3344:101::5 (unchanged)
+         zone 67852e14-ce7f-4e50-a61d-4059d0c057b9 type crucible underlay IP fd00:1122:3344:101::10 (unchanged)
+         zone 6ace1a69-d55a-45f0-a127-99c90f1b070c type crucible underlay IP fd00:1122:3344:101::f (unchanged)
+         zone 84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe type crucible underlay IP fd00:1122:3344:101::9 (unchanged)
+         zone 854c1544-d0ea-4d65-9cf4-776e91a4fcb5 type crucible underlay IP fd00:1122:3344:101::8 (unchanged)
+         zone 86ee7359-aa35-41d9-8a76-54961bc516a1 type cockroach_db underlay IP fd00:1122:3344:101::4 (unchanged)
+         zone 925ea436-fad4-4fff-8ae0-24edd725a2b0 type crucible underlay IP fd00:1122:3344:101::d (unchanged)
+         zone be272d1e-4d34-4478-94b4-a76f5fcbef36 type crucible underlay IP fd00:1122:3344:101::a (unchanged)
+         zone bebd754d-d06c-45f4-99bf-e1aab6253ab8 type oximeter underlay IP fd00:1122:3344:101::6 (unchanged)
+         zone d630cca7-e3f7-47a8-aee8-ad0790895abc type boundary_ntp underlay IP fd00:1122:3344:101::11 (unchanged)
+         zone d8c51c79-46ff-44bd-80be-08d6283f4e21 type crucible underlay IP fd00:1122:3344:101::e (unchanged)
+         zone dbbc0422-2b23-4d9e-b1db-8003cfc29ad0 type crucible underlay IP fd00:1122:3344:101::b (unchanged)
+         zone f33a2375-212d-42e4-b539-6079aeb4e5b7 type crucible_pantry underlay IP fd00:1122:3344:101::7 (unchanged)
+  sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f
+-     zone config generation 2
++     zone config generation 3
+         zone 2d5c29f7-4915-4876-82f8-e388b15c5067 type internal_ntp underlay IP fd00:1122:3344:121::21 (unchanged)
++        zone 0eca39c6-62e2-4a1f-8ea2-e9332a774a26 type crucible underlay IP fd00:1122:3344:121::28 (added)
++        zone 12431d3e-1516-4822-98b3-3711ff912e69 type crucible underlay IP fd00:1122:3344:121::22 (added)
++        zone 56414d76-9b8c-4e15-acf1-cd269c2d8638 type crucible underlay IP fd00:1122:3344:121::25 (added)
++        zone 891cb9da-301d-40cc-b93d-5fcb350fa968 type crucible underlay IP fd00:1122:3344:121::29 (added)
++        zone d9014907-24c0-4c33-a475-db15942f4f8f type crucible underlay IP fd00:1122:3344:121::23 (added)
++        zone e950de25-7306-4cf7-a6f4-6db11adaa310 type crucible underlay IP fd00:1122:3344:121::27 (added)
++        zone fe3dc092-ea23-42d7-9f1d-36eed72b539c type crucible underlay IP fd00:1122:3344:121::26 (added)
++        zone ff8cce49-a721-4ad1-b09e-2bd6e4e42d00 type crucible underlay IP fd00:1122:3344:121::24 (added)
+  sled a6634f64-f6fb-426c-b00b-9b30aed9f53a
+      zone config generation 5
+         zone 0ebd2b5b-2dec-4d17-8e23-e468c04dacb6 type crucible underlay IP fd00:1122:3344:102::11 (unchanged)
+         zone 18b99404-d8b2-4463-bc99-638d9b678ab3 type crucible underlay IP fd00:1122:3344:102::e (unchanged)
+         zone 479811e7-6409-45d4-b294-5333534e47ed type crucible underlay IP fd00:1122:3344:102::9 (unchanged)
+         zone 54d1b693-bc4d-4c47-9009-c6f624073801 type crucible underlay IP fd00:1122:3344:102::a (unchanged)
+         zone 5b743072-9369-45d7-b7c9-dbdf736301f8 type internal_dns underlay IP fd00:1122:3344:2::1 (unchanged)
+         zone 5c4163fa-d6a6-4a0c-9476-3dda135bc9a4 type crucible_pantry underlay IP fd00:1122:3344:102::7 (unchanged)
+         zone 61a39188-dd82-43f6-a87c-ebb4c2d54ac4 type crucible underlay IP fd00:1122:3344:102::8 (unchanged)
+         zone 658076ed-fb43-49e7-a813-df3d23497848 type crucible underlay IP fd00:1122:3344:102::10 (unchanged)
+         zone 6edb836e-b100-4080-bb03-738d6441d571 type crucible underlay IP fd00:1122:3344:102::c (unchanged)
+         zone 7f83d784-9c5c-498f-b559-4176bbafbc4b type nexus underlay IP fd00:1122:3344:102::5 (unchanged)
+         zone 8052938f-73bc-4c54-8ed0-c62f6d77b9fd type crucible underlay IP fd00:1122:3344:102::d (unchanged)
+         zone 86bc0a2a-e24a-4bc1-8222-89207a0937f9 type cockroach_db underlay IP fd00:1122:3344:102::4 (unchanged)
+         zone 9241b9da-8188-4aae-a9dc-62acb453618f type clickhouse underlay IP fd00:1122:3344:102::6 (unchanged)
+         zone a781a349-62b2-41f0-9dd7-c64d290aef82 type crucible underlay IP fd00:1122:3344:102::f (unchanged)
+         zone c67814bf-9972-4bb0-a666-24778ca70a16 type cockroach_db underlay IP fd00:1122:3344:102::3 (unchanged)
+         zone c74ba7d7-3d5f-4a29-aadc-effe33a90909 type boundary_ntp underlay IP fd00:1122:3344:102::12 (unchanged)
+         zone dfc27183-ed85-44c3-ac3c-4427c4cb03e3 type crucible underlay IP fd00:1122:3344:102::b (unchanged)
+  sled fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89
+      zone config generation 5
+         zone 185e6b00-7e18-456c-9dfc-edd0194ac207 type crucible underlay IP fd00:1122:3344:103::a (unchanged)
+         zone 1fc78bc7-feca-4a10-9beb-58b53fda5210 type crucible underlay IP fd00:1122:3344:103::10 (unchanged)
+         zone 45dccfd6-4674-43c6-a0e7-4dde5f24d90d type internal_dns underlay IP fd00:1122:3344:3::1 (unchanged)
+         zone 4b74c418-0a08-4489-b652-e97267c1c220 type crucible underlay IP fd00:1122:3344:103::7 (unchanged)
+         zone 54c947d2-6355-453c-80fc-8f49cc2129ee type nexus underlay IP fd00:1122:3344:103::5 (unchanged)
+         zone 60d64b70-89f5-4774-9e41-2c79c4173911 type crucible_pantry underlay IP fd00:1122:3344:103::6 (unchanged)
+         zone 921958c0-7761-4294-945d-2a078327bf2c type crucible underlay IP fd00:1122:3344:103::8 (unchanged)
+         zone 9df22034-59f6-431f-82e7-258fe983879b type external_dns underlay IP fd00:1122:3344:103::4 (unchanged)
+         zone bb21c0c4-c43f-4913-9bf9-707bd7f1769a type internal_ntp underlay IP fd00:1122:3344:103::11 (unchanged)
+         zone ccb0d612-9d7c-41ba-bd51-661c3c75ec91 type crucible underlay IP fd00:1122:3344:103::9 (unchanged)
+         zone ccef3e95-cae5-4072-9b7d-49e56fa9dff4 type crucible underlay IP fd00:1122:3344:103::d (unchanged)
+         zone d42fc874-8416-4212-b5a0-febba4c6b0ac type crucible underlay IP fd00:1122:3344:103::b (unchanged)
+         zone d5ba7d45-26d7-45ca-8c0a-a560f243b5fa type crucible underlay IP fd00:1122:3344:103::e (unchanged)
+         zone da27ee6a-9841-46aa-acda-7cce3e04cb60 type cockroach_db underlay IP fd00:1122:3344:103::3 (unchanged)
+         zone e09525cf-9c87-4d05-8860-1356dd0dee8a type crucible underlay IP fd00:1122:3344:103::f (unchanged)
+         zone f7264a3e-e8e7-4476-8df3-76c9d00a383b type crucible underlay IP fd00:1122:3344:103::c (unchanged)
+
+
+> 
+
+> blueprint-diff cfa03594-f438-4736-a416-6307f7bf8f2e 6c127695-ba15-408d-a992-325a1a888380
+diff blueprint cfa03594-f438-4736-a416-6307f7bf8f2e blueprint 6c127695-ba15-408d-a992-325a1a888380
+--- blueprint cfa03594-f438-4736-a416-6307f7bf8f2e
++++ blueprint 6c127695-ba15-408d-a992-325a1a888380
+  sled 1762267a-a0cb-4c9a-88b4-8ce828cbc021
+      zone config generation 5
+         zone 0706860a-3801-42a7-8000-fff741f7665b type crucible underlay IP fd00:1122:3344:101::c (unchanged)
+         zone 34a279a2-a025-41ff-a392-1addac400f38 type cockroach_db underlay IP fd00:1122:3344:101::3 (unchanged)
+         zone 3d344935-6f48-405f-8794-2c6e8fa4b1c2 type internal_dns underlay IP fd00:1122:3344:1::1 (unchanged)
+         zone 59c3ed4f-3670-44b1-9ea0-9a6162a08629 type nexus underlay IP fd00:1122:3344:101::5 (unchanged)
+         zone 67852e14-ce7f-4e50-a61d-4059d0c057b9 type crucible underlay IP fd00:1122:3344:101::10 (unchanged)
+         zone 6ace1a69-d55a-45f0-a127-99c90f1b070c type crucible underlay IP fd00:1122:3344:101::f (unchanged)
+         zone 84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe type crucible underlay IP fd00:1122:3344:101::9 (unchanged)
+         zone 854c1544-d0ea-4d65-9cf4-776e91a4fcb5 type crucible underlay IP fd00:1122:3344:101::8 (unchanged)
+         zone 86ee7359-aa35-41d9-8a76-54961bc516a1 type cockroach_db underlay IP fd00:1122:3344:101::4 (unchanged)
+         zone 925ea436-fad4-4fff-8ae0-24edd725a2b0 type crucible underlay IP fd00:1122:3344:101::d (unchanged)
+         zone be272d1e-4d34-4478-94b4-a76f5fcbef36 type crucible underlay IP fd00:1122:3344:101::a (unchanged)
+         zone bebd754d-d06c-45f4-99bf-e1aab6253ab8 type oximeter underlay IP fd00:1122:3344:101::6 (unchanged)
+         zone d630cca7-e3f7-47a8-aee8-ad0790895abc type boundary_ntp underlay IP fd00:1122:3344:101::11 (unchanged)
+         zone d8c51c79-46ff-44bd-80be-08d6283f4e21 type crucible underlay IP fd00:1122:3344:101::e (unchanged)
+         zone dbbc0422-2b23-4d9e-b1db-8003cfc29ad0 type crucible underlay IP fd00:1122:3344:101::b (unchanged)
+         zone f33a2375-212d-42e4-b539-6079aeb4e5b7 type crucible_pantry underlay IP fd00:1122:3344:101::7 (unchanged)
+  sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f
+-     zone config generation 3
++     zone config generation 2
+-        zone 0eca39c6-62e2-4a1f-8ea2-e9332a774a26 type crucible (removed)
+-        zone 12431d3e-1516-4822-98b3-3711ff912e69 type crucible (removed)
+-        zone 56414d76-9b8c-4e15-acf1-cd269c2d8638 type crucible (removed)
+-        zone 891cb9da-301d-40cc-b93d-5fcb350fa968 type crucible (removed)
+-        zone d9014907-24c0-4c33-a475-db15942f4f8f type crucible (removed)
+-        zone e950de25-7306-4cf7-a6f4-6db11adaa310 type crucible (removed)
+-        zone fe3dc092-ea23-42d7-9f1d-36eed72b539c type crucible (removed)
+-        zone ff8cce49-a721-4ad1-b09e-2bd6e4e42d00 type crucible (removed)
+         zone 2d5c29f7-4915-4876-82f8-e388b15c5067 type internal_ntp underlay IP fd00:1122:3344:121::21 (unchanged)
+  sled a6634f64-f6fb-426c-b00b-9b30aed9f53a
+      zone config generation 5
+         zone 0ebd2b5b-2dec-4d17-8e23-e468c04dacb6 type crucible underlay IP fd00:1122:3344:102::11 (unchanged)
+         zone 18b99404-d8b2-4463-bc99-638d9b678ab3 type crucible underlay IP fd00:1122:3344:102::e (unchanged)
+         zone 479811e7-6409-45d4-b294-5333534e47ed type crucible underlay IP fd00:1122:3344:102::9 (unchanged)
+         zone 54d1b693-bc4d-4c47-9009-c6f624073801 type crucible underlay IP fd00:1122:3344:102::a (unchanged)
+         zone 5b743072-9369-45d7-b7c9-dbdf736301f8 type internal_dns underlay IP fd00:1122:3344:2::1 (unchanged)
+         zone 5c4163fa-d6a6-4a0c-9476-3dda135bc9a4 type crucible_pantry underlay IP fd00:1122:3344:102::7 (unchanged)
+         zone 61a39188-dd82-43f6-a87c-ebb4c2d54ac4 type crucible underlay IP fd00:1122:3344:102::8 (unchanged)
+         zone 658076ed-fb43-49e7-a813-df3d23497848 type crucible underlay IP fd00:1122:3344:102::10 (unchanged)
+         zone 6edb836e-b100-4080-bb03-738d6441d571 type crucible underlay IP fd00:1122:3344:102::c (unchanged)
+         zone 7f83d784-9c5c-498f-b559-4176bbafbc4b type nexus underlay IP fd00:1122:3344:102::5 (unchanged)
+         zone 8052938f-73bc-4c54-8ed0-c62f6d77b9fd type crucible underlay IP fd00:1122:3344:102::d (unchanged)
+         zone 86bc0a2a-e24a-4bc1-8222-89207a0937f9 type cockroach_db underlay IP fd00:1122:3344:102::4 (unchanged)
+         zone 9241b9da-8188-4aae-a9dc-62acb453618f type clickhouse underlay IP fd00:1122:3344:102::6 (unchanged)
+         zone a781a349-62b2-41f0-9dd7-c64d290aef82 type crucible underlay IP fd00:1122:3344:102::f (unchanged)
+         zone c67814bf-9972-4bb0-a666-24778ca70a16 type cockroach_db underlay IP fd00:1122:3344:102::3 (unchanged)
+         zone c74ba7d7-3d5f-4a29-aadc-effe33a90909 type boundary_ntp underlay IP fd00:1122:3344:102::12 (unchanged)
+         zone dfc27183-ed85-44c3-ac3c-4427c4cb03e3 type crucible underlay IP fd00:1122:3344:102::b (unchanged)
+  sled fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89
+      zone config generation 5
+         zone 185e6b00-7e18-456c-9dfc-edd0194ac207 type crucible underlay IP fd00:1122:3344:103::a (unchanged)
+         zone 1fc78bc7-feca-4a10-9beb-58b53fda5210 type crucible underlay IP fd00:1122:3344:103::10 (unchanged)
+         zone 45dccfd6-4674-43c6-a0e7-4dde5f24d90d type internal_dns underlay IP fd00:1122:3344:3::1 (unchanged)
+         zone 4b74c418-0a08-4489-b652-e97267c1c220 type crucible underlay IP fd00:1122:3344:103::7 (unchanged)
+         zone 54c947d2-6355-453c-80fc-8f49cc2129ee type nexus underlay IP fd00:1122:3344:103::5 (unchanged)
+         zone 60d64b70-89f5-4774-9e41-2c79c4173911 type crucible_pantry underlay IP fd00:1122:3344:103::6 (unchanged)
+         zone 921958c0-7761-4294-945d-2a078327bf2c type crucible underlay IP fd00:1122:3344:103::8 (unchanged)
+         zone 9df22034-59f6-431f-82e7-258fe983879b type external_dns underlay IP fd00:1122:3344:103::4 (unchanged)
+         zone bb21c0c4-c43f-4913-9bf9-707bd7f1769a type internal_ntp underlay IP fd00:1122:3344:103::11 (unchanged)
+         zone ccb0d612-9d7c-41ba-bd51-661c3c75ec91 type crucible underlay IP fd00:1122:3344:103::9 (unchanged)
+         zone ccef3e95-cae5-4072-9b7d-49e56fa9dff4 type crucible underlay IP fd00:1122:3344:103::d (unchanged)
+         zone d42fc874-8416-4212-b5a0-febba4c6b0ac type crucible underlay IP fd00:1122:3344:103::b (unchanged)
+         zone d5ba7d45-26d7-45ca-8c0a-a560f243b5fa type crucible underlay IP fd00:1122:3344:103::e (unchanged)
+         zone da27ee6a-9841-46aa-acda-7cce3e04cb60 type cockroach_db underlay IP fd00:1122:3344:103::3 (unchanged)
+         zone e09525cf-9c87-4d05-8860-1356dd0dee8a type crucible underlay IP fd00:1122:3344:103::f (unchanged)
+         zone f7264a3e-e8e7-4476-8df3-76c9d00a383b type crucible underlay IP fd00:1122:3344:103::c (unchanged)
+
+
+> 
+
+> sled-add dde1c0e2-b10d-4621-b420-f179f7a7a00a
+added sled
+
+> 
+
+> blueprint-plan cfa03594-f438-4736-a416-6307f7bf8f2e 83ed3949-d221-44f3-a901-02e9ff16d2a5
+generated blueprint REDACTED_UUID based on parent blueprint cfa03594-f438-4736-a416-6307f7bf8f2e
+
diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-stderr b/dev-tools/reconfigurator-cli/tests/output/cmd-stderr
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-stdout b/dev-tools/reconfigurator-cli/tests/output/cmd-stdout
new file mode 100644
index 0000000000..10b158f218
--- /dev/null
+++ b/dev-tools/reconfigurator-cli/tests/output/cmd-stdout
@@ -0,0 +1,58 @@
+> sled-list
+ID NZPOOLS SUBNET 
+
+> inventory-list
+ID NERRORS TIME_DONE 
+
+> blueprint-list
+ID 
+
+> 
+
+> sled-show REDACTED_UUID_REDACTED_UUID_REDACTED
+error: no sled with id REDACTED_UUID_REDACTED_UUID_REDACTED
+
+> sled-add REDACTED_UUID_REDACTED_UUID_REDACTED
+added sled
+
+> sled-list
+ID                                   NZPOOLS SUBNET                  
+REDACTED_UUID_REDACTED_UUID_REDACTED 10      fd00:1122:3344:101::/64 
+
+> sled-show REDACTED_UUID_REDACTED_UUID_REDACTED
+sled REDACTED_UUID_REDACTED_UUID_REDACTED
+subnet fd00:1122:3344:101::/64
+zpools (10):
+    ZpoolName("oxp_REDACTED_UUID_REDACTED_UUID_REDACTED")
+    ZpoolName("oxp_REDACTED_UUID_REDACTED_UUID_REDACTED")
+    ZpoolName("oxp_REDACTED_UUID_REDACTED_UUID_REDACTED")
+    ZpoolName("oxp_REDACTED_UUID_REDACTED_UUID_REDACTED")
+    ZpoolName("oxp_REDACTED_UUID_REDACTED_UUID_REDACTED")
+    ZpoolName("oxp_REDACTED_UUID_REDACTED_UUID_REDACTED")
+    ZpoolName("oxp_REDACTED_UUID_REDACTED_UUID_REDACTED")
+    ZpoolName("oxp_REDACTED_UUID_REDACTED_UUID_REDACTED")
+    ZpoolName("oxp_REDACTED_UUID_REDACTED_UUID_REDACTED")
+    ZpoolName("oxp_REDACTED_UUID_REDACTED_UUID_REDACTED")
+
+
+> sled-add REDACTED_UUID_REDACTED_UUID_REDACTED
+added sled
+
+> sled-add REDACTED_UUID_REDACTED_UUID_REDACTED
+added sled
+
+> sled-list
+ID                                   NZPOOLS SUBNET                  
+REDACTED_UUID_REDACTED_UUID_REDACTED 10      fd00:1122:3344:103::/64 
+REDACTED_UUID_REDACTED_UUID_REDACTED 10      fd00:1122:3344:102::/64 
+REDACTED_UUID_REDACTED_UUID_REDACTED 10      fd00:1122:3344:101::/64 
+
+> 
+
+> inventory-generate
+generated inventory collection REDACTED_UUID_REDACTED_UUID_REDACTED from configured sleds
+
+> inventory-list
+ID                                   NERRORS TIME_DONE                
+REDACTED_UUID_REDACTED_UUID_REDACTED 0       <REDACTED     TIMESTAMP> 
+
diff --git a/dev-tools/reconfigurator-cli/tests/test_basic.rs b/dev-tools/reconfigurator-cli/tests/test_basic.rs
new file mode 100644
index 0000000000..f70fcb1366
--- /dev/null
+++ b/dev-tools/reconfigurator-cli/tests/test_basic.rs
@@ -0,0 +1,115 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use anyhow::Context;
+use camino::Utf8Path;
+use expectorate::assert_contents;
+use nexus_types::deployment::UnstableReconfiguratorState;
+use omicron_test_utils::dev::test_cmds::assert_exit_code;
+use omicron_test_utils::dev::test_cmds::path_to_executable;
+use omicron_test_utils::dev::test_cmds::redact_variable;
+use omicron_test_utils::dev::test_cmds::run_command;
+use omicron_test_utils::dev::test_cmds::EXIT_SUCCESS;
+use std::path::PathBuf;
+use subprocess::Exec;
+
+fn path_to_cli() -> PathBuf {
+    path_to_executable(env!("CARGO_BIN_EXE_reconfigurator-cli"))
+}
+
+// Run a battery of simple commands and make sure things basically seem to work.
+#[test]
+fn test_basic() {
+    let exec = Exec::cmd(path_to_cli()).arg("tests/input/cmds.txt");
+    let (exit_status, stdout_text, stderr_text) = run_command(exec);
+    assert_exit_code(exit_status, EXIT_SUCCESS, &stderr_text);
+    let stdout_text = redact_variable(&stdout_text, &[]);
+    assert_contents("tests/output/cmd-stdout", &stdout_text);
+    assert_contents("tests/output/cmd-stderr", &stderr_text);
+}
+
+// To do more interesting testing, we need more interesting data than we can
+// currently cons up in the REPL environment.  This test uses a file saved from
+// the "madrid" rig using `omdb db reconfigurator-save`.
+//
+// This test may be broken if the format of this file changes (see the
+// UnstableReconfiguratorState struct).  To fix it, generate a new file from a
+// real system with at least one inventory collection and at least two
+// blueprints.  You will also need to set EXPECTORATE=overwrite when running
+// this for the first time with the new input file in order to generate the new
+// output file.  Check it.
+//
+// If this test becomes too burdensome, we could drop it.  But for now it's
+// useful to have a smoke test for the command.
+#[test]
+fn test_complex() {
+    let input_path = Utf8Path::new("tests/input/complex.json");
+    let input_file = std::fs::File::open(input_path)
+        .with_context(|| format!("open {:?}", input_path))
+        .unwrap();
+    let input_data: UnstableReconfiguratorState =
+        serde_json::from_reader(&input_file)
+            .with_context(|| format!("read {:?}", input_path))
+            .unwrap();
+    assert!(
+        input_data.collections.len() > 0,
+        "input file must have at least one inventory collection"
+    );
+    assert!(
+        input_data.blueprints.len() > 1,
+        "input file must have at least two blueprints"
+    );
+
+    let collection = input_data.collections.iter().next().unwrap().id;
+    let mut blueprints = input_data.blueprints.iter().rev();
+    let blueprint2 = blueprints.next().unwrap().id;
+    let blueprint1 = blueprints.next().unwrap().id;
+    let tmpdir = camino_tempfile::tempdir().expect("failed to create tmpdir");
+    let tmpfile = tmpdir.path().join("cmds.txt");
+
+    // We construct the list of commands dynamically to avoid having to re-do it
+    // when the input file has to be regenerated.
+    let input_cmds = [
+        "sled-list",
+        "inventory-list",
+        "blueprint-list",
+        &format!("file-contents {}", input_path),
+        &format!("load {} {}", input_path, collection),
+        "sled-list",
+        "inventory-list",
+        "blueprint-list",
+        &format!("blueprint-show {}", blueprint2),
+        &format!("blueprint-diff-inventory {} {}", collection, blueprint2),
+        &format!("blueprint-diff {} {}", blueprint1, blueprint2),
+        &format!("blueprint-diff {} {}", blueprint2, blueprint1),
+        "sled-add dde1c0e2-b10d-4621-b420-f179f7a7a00a",
+        &format!("blueprint-plan {} {}", blueprint2, collection),
+    ]
+    .into_iter()
+    .map(|s| format!("{}\n", s))
+    .collect::<Vec<_>>()
+    .join("\n");
+
+    println!("will execute commands:\n{}", input_cmds);
+
+    std::fs::write(&tmpfile, &input_cmds)
+        .with_context(|| format!("write {:?}", &tmpfile))
+        .unwrap();
+
+    let exec = Exec::cmd(path_to_cli()).arg(&tmpfile);
+    let (exit_status, stdout_text, stderr_text) = run_command(exec);
+    assert_exit_code(exit_status, EXIT_SUCCESS, &stderr_text);
+
+    // This is a much lighter form of redaction than `redact_variable()` does.
+    let stdout_text =
+        regex::Regex::new(r"generated blueprint .* based on parent blueprint")
+            .unwrap()
+            .replace_all(
+                &stdout_text,
+                "generated blueprint REDACTED_UUID based on parent blueprint",
+            )
+            .to_string();
+
+    assert_contents("tests/output/cmd-complex-stdout", &stdout_text);
+}
diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml
index de79f3429d..45d1c34382 100644
--- a/nexus/Cargo.toml
+++ b/nexus/Cargo.toml
@@ -84,6 +84,7 @@ nexus-db-queries.workspace = true
 nexus-inventory.workspace = true
 nexus-reconfigurator-execution.workspace = true
 nexus-reconfigurator-planning.workspace = true
+nexus-reconfigurator-preparation.workspace = true
 nexus-types.workspace = true
 omicron-common.workspace = true
 omicron-passwords.workspace = true
diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs
index a00318c9dc..1dae1030f0 100644
--- a/nexus/db-queries/src/db/datastore/deployment.rs
+++ b/nexus/db-queries/src/db/datastore/deployment.rs
@@ -7,6 +7,7 @@ use crate::authz;
 use crate::authz::ApiResource;
 use crate::context::OpContext;
 use crate::db;
+use crate::db::datastore::SQL_BATCH_SIZE;
 use crate::db::error::public_error_from_diesel;
 use crate::db::error::ErrorHandler;
 use crate::db::pagination::paginated;
@@ -49,18 +50,8 @@ use omicron_common::api::external::ResourceType;
 use omicron_common::bail_unless;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
-use std::num::NonZeroU32;
 use uuid::Uuid;
 
-/// "limit" used in SQL queries that paginate through all sleds, omicron
-/// zones, etc.
-///
-/// While we always load an entire blueprint in one operation, we use a
-/// [`Paginator`] to guard against single queries returning an unchecked number
-/// of rows.
-// unsafe: `new_unchecked` is only unsound if the argument is 0.
-const SQL_BATCH_SIZE: NonZeroU32 = unsafe { NonZeroU32::new_unchecked(1000) };
-
 impl DataStore {
     /// List blueprints
     pub async fn blueprints_list(
diff --git a/nexus/db-queries/src/db/datastore/ip_pool.rs b/nexus/db-queries/src/db/datastore/ip_pool.rs
index 4a37efd612..a9062e6ee3 100644
--- a/nexus/db-queries/src/db/datastore/ip_pool.rs
+++ b/nexus/db-queries/src/db/datastore/ip_pool.rs
@@ -5,6 +5,7 @@
 //! [`DataStore`] methods on [`IpPool`]s.
 
 use super::DataStore;
+use super::SQL_BATCH_SIZE;
 use crate::authz;
 use crate::context::OpContext;
 use crate::db;
@@ -25,6 +26,7 @@ use crate::db::model::IpPoolResourceType;
 use crate::db::model::IpPoolUpdate;
 use crate::db::model::Name;
 use crate::db::pagination::paginated;
+use crate::db::pagination::Paginator;
 use crate::db::pool::DbConnection;
 use crate::db::queries::ip_pool::FilterOverlappingIpRanges;
 use crate::db::TransactionError;
@@ -669,6 +671,33 @@ impl DataStore {
             })
     }
 
+    /// List all IP pool ranges for a given pool, making as many queries as
+    /// needed to get them all
+    ///
+    /// This should generally not be used in API handlers or other
+    /// latency-sensitive contexts, but it can make sense in saga actions or
+    /// background tasks.
+    pub async fn ip_pool_list_ranges_batched(
+        &self,
+        opctx: &OpContext,
+        authz_pool: &authz::IpPool,
+    ) -> ListResultVec<IpPoolRange> {
+        opctx.check_complex_operations_allowed()?;
+        let mut ip_ranges = Vec::new();
+        let mut paginator = Paginator::new(SQL_BATCH_SIZE);
+        while let Some(p) = paginator.next() {
+            let batch = self
+                .ip_pool_list_ranges(opctx, &authz_pool, &p.current_pagparams())
+                .await?;
+            // The use of `last_address` here assumes `paginator` is sorting
+            // in Ascending order (which it does - see the implementation of
+            // `current_pagparams()`).
+            paginator = p.found_batch(&batch, &|r| r.last_address);
+            ip_ranges.extend(batch);
+        }
+        Ok(ip_ranges)
+    }
+
     pub async fn ip_pool_add_range(
         &self,
         opctx: &OpContext,
diff --git a/nexus/db-queries/src/db/datastore/zpool.rs b/nexus/db-queries/src/db/datastore/zpool.rs
index 79e5f5a55a..5424c126eb 100644
--- a/nexus/db-queries/src/db/datastore/zpool.rs
+++ b/nexus/db-queries/src/db/datastore/zpool.rs
@@ -5,6 +5,7 @@
 //! [`DataStore`] methods on [`Zpool`]s.
 
 use super::DataStore;
+use super::SQL_BATCH_SIZE;
 use crate::authz;
 use crate::db;
 use crate::db::collection_insert::AsyncInsertError;
@@ -16,6 +17,7 @@ use crate::db::identity::Asset;
 use crate::db::model::Sled;
 use crate::db::model::Zpool;
 use crate::db::pagination::paginated;
+use crate::db::pagination::Paginator;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use chrono::Utc;
 use diesel::prelude::*;
@@ -66,8 +68,8 @@ impl DataStore {
         })
     }
 
-    /// Paginates through all zpools on U.2 disks in all sleds
-    pub async fn zpool_list_all_external(
+    /// Fetches a page of the list of all zpools on U.2 disks in all sleds
+    async fn zpool_list_all_external(
         &self,
         opctx: &OpContext,
         pagparams: &DataPageParams<'_, Uuid>,
@@ -90,4 +92,30 @@ impl DataStore {
             .await
             .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
     }
+
+    /// List all zpools on U.2 disks in all sleds, making as many queries as
+    /// needed to get them all
+    ///
+    /// This should generally not be used in API handlers or other
+    /// latency-sensitive contexts, but it can make sense in saga actions or
+    /// background tasks.
+    pub async fn zpool_list_all_external_batched(
+        &self,
+        opctx: &OpContext,
+    ) -> ListResultVec<Zpool> {
+        opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?;
+        opctx.check_complex_operations_allowed()?;
+        let mut zpools = Vec::new();
+        let mut paginator = Paginator::new(SQL_BATCH_SIZE);
+        while let Some(p) = paginator.next() {
+            let batch = self
+                .zpool_list_all_external(opctx, &p.current_pagparams())
+                .await?;
+            paginator =
+                p.found_batch(&batch, &|z: &nexus_db_model::Zpool| z.id());
+            zpools.extend(batch);
+        }
+
+        Ok(zpools)
+    }
 }
diff --git a/nexus/inventory/src/builder.rs b/nexus/inventory/src/builder.rs
index 08a905143c..602890a440 100644
--- a/nexus/inventory/src/builder.rs
+++ b/nexus/inventory/src/builder.rs
@@ -93,11 +93,14 @@ impl CollectionBuilder {
     /// `collector` is an arbitrary string describing the agent that collected
     /// this data.  It's generally a Nexus instance uuid but it can be anything.
     /// It's just for debugging.
-    pub fn new(collector: &str) -> Self {
+    pub fn new<S>(collector: S) -> Self
+    where
+        String: From<S>,
+    {
         CollectionBuilder {
             errors: vec![],
             time_started: now_db_precision(),
-            collector: collector.to_owned(),
+            collector: String::from(collector),
             baseboards: BTreeSet::new(),
             cabooses: BTreeSet::new(),
             rot_pages: BTreeSet::new(),
diff --git a/nexus/reconfigurator/planning/Cargo.toml b/nexus/reconfigurator/planning/Cargo.toml
index 3ec616168c..41ea17d56e 100644
--- a/nexus/reconfigurator/planning/Cargo.toml
+++ b/nexus/reconfigurator/planning/Cargo.toml
@@ -6,6 +6,8 @@ edition = "2021"
 [dependencies]
 anyhow.workspace = true
 chrono.workspace = true
+gateway-client.workspace = true
+indexmap.workspace = true
 internal-dns.workspace = true
 ipnet.workspace = true
 ipnetwork.workspace = true
@@ -13,6 +15,7 @@ nexus-config.workspace = true
 nexus-inventory.workspace = true
 nexus-types.workspace = true
 omicron-common.workspace = true
+sled-agent-client.workspace = true
 slog.workspace = true
 thiserror.workspace = true
 uuid.workspace = true
diff --git a/nexus/reconfigurator/planning/src/blueprint_builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder.rs
index ebb64c36f9..d541e112d5 100644
--- a/nexus/reconfigurator/planning/src/blueprint_builder.rs
+++ b/nexus/reconfigurator/planning/src/blueprint_builder.rs
@@ -462,21 +462,6 @@ impl<'a> BlueprintBuilder<'a> {
         sled_id: Uuid,
         desired_zone_count: usize,
     ) -> Result<EnsureMultiple, Error> {
-        // How many Nexus zones do we need to add?
-        let nexus_count = self.sled_num_nexus_zones(sled_id);
-        let num_nexus_to_add = match desired_zone_count.checked_sub(nexus_count)
-        {
-            Some(0) => return Ok(EnsureMultiple::NotNeeded),
-            Some(n) => n,
-            None => {
-                return Err(Error::Planner(anyhow!(
-                    "removing a Nexus zone not yet supported \
-                     (sled {sled_id} has {nexus_count}; \
-                     planner wants {desired_zone_count})"
-                )));
-            }
-        };
-
         // Whether Nexus should use TLS and what the external DNS servers it
         // should use are currently provided at rack-setup time, and should be
         // consistent across all Nexus instances. We'll assume we can copy them
@@ -503,6 +488,35 @@ impl<'a> BlueprintBuilder<'a> {
                 })
             })
             .ok_or(Error::NoNexusZonesInParentBlueprint)?;
+        self.sled_ensure_zone_multiple_nexus_with_config(
+            sled_id,
+            desired_zone_count,
+            external_tls,
+            external_dns_servers,
+        )
+    }
+
+    pub fn sled_ensure_zone_multiple_nexus_with_config(
+        &mut self,
+        sled_id: Uuid,
+        desired_zone_count: usize,
+        external_tls: bool,
+        external_dns_servers: Vec<IpAddr>,
+    ) -> Result<EnsureMultiple, Error> {
+        // How many Nexus zones do we need to add?
+        let nexus_count = self.sled_num_nexus_zones(sled_id);
+        let num_nexus_to_add = match desired_zone_count.checked_sub(nexus_count)
+        {
+            Some(0) => return Ok(EnsureMultiple::NotNeeded),
+            Some(n) => n,
+            None => {
+                return Err(Error::Planner(anyhow!(
+                    "removing a Nexus zone not yet supported \
+                     (sled {sled_id} has {nexus_count}; \
+                     planner wants {desired_zone_count})"
+                )));
+            }
+        };
 
         for _ in 0..num_nexus_to_add {
             let nexus_id = Uuid::new_v4();
@@ -736,203 +750,124 @@ impl<'a> BlueprintZones<'a> {
 #[cfg(test)]
 pub mod test {
     use super::*;
-    use nexus_types::external_api::views::SledPolicy;
-    use nexus_types::external_api::views::SledState;
+    use crate::system::SledBuilder;
+    use crate::system::SystemDescription;
     use omicron_common::address::IpRange;
-    use omicron_common::address::Ipv4Range;
-    use omicron_common::address::Ipv6Subnet;
-    use omicron_common::address::SLED_PREFIX;
-    use omicron_common::api::external::ByteCount;
     use omicron_test_utils::dev::test_setup_log;
     use sled_agent_client::types::{
-        Baseboard, Inventory, OmicronZoneConfig, OmicronZoneDataset,
-        OmicronZoneType, OmicronZonesConfig, SledRole,
+        OmicronZoneConfig, OmicronZoneType, OmicronZonesConfig,
     };
-    use std::str::FromStr;
 
     pub const DEFAULT_N_SLEDS: usize = 3;
 
-    /// Returns a collection and policy describing a pretty simple system.
-    ///
-    /// `n_sleds` is the number of sleds supported. Currently, this value can
-    /// be anywhere between 0 and 5. (More can be added in the future if
-    /// necessary.)
-    pub fn example(n_sleds: usize) -> (Collection, Policy) {
-        let mut builder = nexus_inventory::CollectionBuilder::new("test-suite");
-
-        if n_sleds > 5 {
-            panic!("example() only supports up to 5 sleds, but got {n_sleds}");
-        }
+    pub struct ExampleSystem {
+        pub system: SystemDescription,
+        pub policy: Policy,
+        pub collection: Collection,
+        pub blueprint: Blueprint,
+    }
 
-        let sled_ids = [
-            "72443b6c-b8bb-4ffa-ab3a-aeaa428ed79b",
-            "a5f3db3a-61aa-4f90-ad3e-02833c253bf5",
-            "0d168386-2551-44e8-98dd-ae7a7570f8a0",
-            "aaaaa1a1-0c3f-4928-aba7-6ec5c1db05f7",
-            "85e88acb-7b86-45ff-9c88-734e1da71c3d",
-        ];
-        let mut policy = Policy {
-            sleds: BTreeMap::new(),
-            // IPs from TEST-NET-1 (RFC 5737)
-            service_ip_pool_ranges: vec![Ipv4Range::new(
-                "192.0.2.2".parse().unwrap(),
-                "192.0.2.20".parse().unwrap(),
-            )
-            .unwrap()
-            .into()],
-            target_nexus_zone_count: 3,
-        };
-        let mut service_ip_pool_range = policy.service_ip_pool_ranges[0].iter();
-        let mut nexus_nic_ips = NEXUS_OPTE_IPV4_SUBNET
-            .iter()
-            .skip(NUM_INITIAL_RESERVED_IP_ADDRESSES);
-        let mut nexus_nic_macs = {
-            let mut used = HashSet::new();
-            std::iter::from_fn(move || {
-                let mut mac = MacAddr::random_system();
-                while !used.insert(mac) {
-                    mac = MacAddr::random_system();
-                }
-                Some(mac)
-            })
-        };
+    impl ExampleSystem {
+        pub fn new(log: &slog::Logger, nsleds: usize) -> ExampleSystem {
+            let mut system = SystemDescription::new();
+            let sled_ids: Vec<_> =
+                (0..nsleds).map(|_| Uuid::new_v4()).collect();
+            for sled_id in &sled_ids {
+                let _ = system.sled(SledBuilder::new().id(*sled_id)).unwrap();
+            }
 
-        for sled_id_str in sled_ids.iter().take(n_sleds) {
-            let sled_id: Uuid = sled_id_str.parse().unwrap();
-            let sled_ip = policy_add_sled(&mut policy, sled_id);
-            let serial_number = format!("s{}", policy.sleds.len());
-            builder
-                .found_sled_inventory(
-                    "test-suite",
-                    Inventory {
-                        baseboard: Baseboard::Gimlet {
-                            identifier: serial_number,
-                            model: String::from("model1"),
-                            revision: 0,
+            let policy = system.to_policy().expect("failed to make policy");
+            let mut inventory_builder = system
+                .to_collection_builder()
+                .expect("failed to build collection");
+
+            // For each sled, have it report 0 zones in the initial inventory.
+            // This will enable us to build a blueprint from the initial
+            // inventory, which we can then use to build new blueprints.
+            for sled_id in &sled_ids {
+                inventory_builder
+                    .found_sled_omicron_zones(
+                        "fake sled agent",
+                        *sled_id,
+                        OmicronZonesConfig {
+                            generation: Generation::new(),
+                            zones: vec![],
                         },
-                        reservoir_size: ByteCount::from(1024),
-                        sled_role: SledRole::Gimlet,
-                        sled_agent_address: SocketAddrV6::new(
-                            sled_ip, 12345, 0, 0,
-                        )
-                        .to_string(),
-                        sled_id,
-                        usable_hardware_threads: 10,
-                        usable_physical_ram: ByteCount::from(1024 * 1024),
-                    },
+                    )
+                    .expect("recording Omicron zones");
+            }
+
+            let empty_zone_inventory = inventory_builder.build();
+            let initial_blueprint =
+                BlueprintBuilder::build_initial_from_collection(
+                    &empty_zone_inventory,
+                    Generation::new(),
+                    &policy,
+                    "test suite",
                 )
                 .unwrap();
 
-            let zpools = &policy.sleds.get(&sled_id).unwrap().zpools;
-            let mut sled_ips =
-                std::iter::successors(Some(sled_ip.saturating_add(1)), |ip| {
-                    println!("sled_ips iterator: currently {ip:?}");
-                    Some(ip.saturating_add(1))
-                });
-            let zones: Vec<_> = std::iter::once({
-                let ip = sled_ips.next().unwrap();
-                OmicronZoneConfig {
-                    id: Uuid::new_v4(),
-                    underlay_address: ip,
-                    zone_type: OmicronZoneType::InternalNtp {
-                        address: SocketAddrV6::new(ip, 12345, 0, 0).to_string(),
-                        dns_servers: vec![],
-                        domain: None,
-                        ntp_servers: vec![],
-                    },
-                }
-            })
-            .chain(std::iter::once({
-                let id = Uuid::new_v4();
-                let ip = sled_ips.next().unwrap();
-                let external_ip =
-                    service_ip_pool_range.next().expect("no service IPs left");
-                let nic_ip =
-                    nexus_nic_ips.next().expect("no nexus nic IPs left");
-                OmicronZoneConfig {
-                    id,
-                    underlay_address: ip,
-                    zone_type: OmicronZoneType::Nexus {
-                        internal_address: SocketAddrV6::new(ip, 12346, 0, 0)
-                            .to_string(),
-                        external_ip,
-                        nic: NetworkInterface {
-                            id: Uuid::new_v4(),
-                            kind: NetworkInterfaceKind::Service(id),
-                            name: format!("nexus-{id}").parse().unwrap(),
-                            ip: nic_ip.into(),
-                            mac: nexus_nic_macs
-                                .next()
-                                .expect("no nexus nic MACs left"),
-                            subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET).into(),
-                            vni: Vni::SERVICES_VNI,
-                            primary: true,
-                            slot: 0,
-                        },
-                        external_tls: false,
-                        external_dns_servers: Vec::new(),
-                    },
-                }
-            }))
-            .chain(zpools.iter().map(|zpool_name| {
-                let ip = sled_ips.next().unwrap();
-                OmicronZoneConfig {
-                    id: Uuid::new_v4(),
-                    underlay_address: ip,
-                    zone_type: OmicronZoneType::Crucible {
-                        address: String::from("[::1]:12345"),
-                        dataset: OmicronZoneDataset {
-                            pool_name: zpool_name.clone(),
-                        },
-                    },
+            // Now make a blueprint and collection with some zones on each sled.
+            let mut builder = BlueprintBuilder::new_based_on(
+                &log,
+                &initial_blueprint,
+                Generation::new(),
+                &policy,
+                "test suite",
+            )
+            .unwrap();
+            for (sled_id, sled_resources) in &policy.sleds {
+                let _ = builder.sled_ensure_zone_ntp(*sled_id).unwrap();
+                let _ = builder
+                    .sled_ensure_zone_multiple_nexus_with_config(
+                        *sled_id,
+                        1,
+                        false,
+                        vec![],
+                    )
+                    .unwrap();
+                for pool_name in &sled_resources.zpools {
+                    let _ = builder
+                        .sled_ensure_zone_crucible(*sled_id, pool_name.clone())
+                        .unwrap();
                 }
-            }))
-            .collect();
+            }
 
-            builder
-                .found_sled_omicron_zones(
-                    "test-suite",
-                    sled_id,
-                    OmicronZonesConfig {
-                        generation: Generation::new().next(),
-                        zones,
-                    },
-                )
-                .unwrap();
-        }
+            let blueprint = builder.build();
+            let mut builder = system
+                .to_collection_builder()
+                .expect("failed to build collection");
 
-        let collection = builder.build();
+            for sled_id in blueprint.sleds() {
+                let Some(zones) = blueprint.omicron_zones.get(&sled_id) else {
+                    continue;
+                };
+                builder
+                    .found_sled_omicron_zones(
+                        "fake sled agent",
+                        sled_id,
+                        zones.clone(),
+                    )
+                    .unwrap();
+            }
 
-        (collection, policy)
+            ExampleSystem {
+                system,
+                policy,
+                collection: builder.build(),
+                blueprint,
+            }
+        }
     }
 
-    pub fn policy_add_sled(policy: &mut Policy, sled_id: Uuid) -> Ipv6Addr {
-        let i = policy.sleds.len() + 1;
-        let sled_ip: Ipv6Addr =
-            format!("fd00:1122:3344:{}::1", i + 1).parse().unwrap();
-
-        let zpools: BTreeSet<ZpoolName> = [
-            "oxp_be776cf5-4cba-4b7d-8109-3dfd020f22ee",
-            "oxp_aee23a17-b2ce-43f2-9302-c738d92cca28",
-            "oxp_f7940a6b-c865-41cf-ad61-1b831d594286",
-        ]
-        .iter()
-        .map(|name_str| {
-            ZpoolName::from_str(name_str).expect("not a valid zpool name")
-        })
-        .collect();
-
-        let subnet = Ipv6Subnet::<SLED_PREFIX>::new(sled_ip);
-        policy.sleds.insert(
-            sled_id,
-            SledResources {
-                policy: SledPolicy::provisionable(),
-                state: SledState::Active,
-                zpools,
-                subnet,
-            },
-        );
-        sled_ip
+    /// Returns a collection and policy describing a pretty simple system.
+    ///
+    /// `n_sleds` is the number of sleds supported. Currently, this value can
+    /// be anywhere between 0 and 5. (More can be added in the future if
+    /// necessary.)
+    pub fn example(log: &slog::Logger, nsleds: usize) -> (Collection, Policy) {
+        let example = ExampleSystem::new(log, nsleds);
+        (example.collection, example.policy)
     }
 
     /// Checks various conditions that should be true for all blueprints
@@ -959,7 +894,7 @@ pub mod test {
         // Test creating a blueprint from a collection and verifying that it
         // describes no changes.
         let logctx = test_setup_log("blueprint_builder_test_initial");
-        let (collection, policy) = example(DEFAULT_N_SLEDS);
+        let (collection, policy) = example(&logctx.log, DEFAULT_N_SLEDS);
         let blueprint_initial =
             BlueprintBuilder::build_initial_from_collection(
                 &collection,
@@ -1009,29 +944,23 @@ pub mod test {
     #[test]
     fn test_basic() {
         let logctx = test_setup_log("blueprint_builder_test_basic");
-        let (collection, mut policy) = example(DEFAULT_N_SLEDS);
-        let blueprint1 = BlueprintBuilder::build_initial_from_collection(
-            &collection,
-            Generation::new(),
-            &policy,
-            "the_test",
-        )
-        .expect("failed to create initial blueprint");
-        verify_blueprint(&blueprint1);
+        let mut example = ExampleSystem::new(&logctx.log, DEFAULT_N_SLEDS);
+        let blueprint1 = &example.blueprint;
+        verify_blueprint(blueprint1);
 
         let mut builder = BlueprintBuilder::new_based_on(
             &logctx.log,
-            &blueprint1,
+            blueprint1,
             Generation::new(),
-            &policy,
+            &example.policy,
             "test_basic",
         )
         .expect("failed to create builder");
 
-        // The initial blueprint should have internal NTP zones on all the
+        // The example blueprint should have internal NTP zones on all the
         // existing sleds, plus Crucible zones on all pools.  So if we ensure
         // all these zones exist, we should see no change.
-        for (sled_id, sled_resources) in &policy.sleds {
+        for (sled_id, sled_resources) in &example.policy.sleds {
             builder.sled_ensure_zone_ntp(*sled_id).unwrap();
             for pool_name in &sled_resources.zpools {
                 builder
@@ -1053,7 +982,9 @@ pub mod test {
 
         // The next step is adding these zones to a new sled.
         let new_sled_id = Uuid::new_v4();
-        let _ = policy_add_sled(&mut policy, new_sled_id);
+        let _ =
+            example.system.sled(SledBuilder::new().id(new_sled_id)).unwrap();
+        let policy = example.system.to_policy().unwrap();
         let mut builder = BlueprintBuilder::new_based_on(
             &logctx.log,
             &blueprint2,
@@ -1135,7 +1066,7 @@ pub mod test {
         let logctx = test_setup_log(
             "blueprint_builder_test_add_nexus_with_no_existing_nexus_zones",
         );
-        let (mut collection, policy) = example(DEFAULT_N_SLEDS);
+        let (mut collection, policy) = example(&logctx.log, DEFAULT_N_SLEDS);
 
         // We don't care about the internal DNS version here.
         let internal_dns_version = Generation::new();
@@ -1191,7 +1122,7 @@ pub mod test {
     fn test_add_nexus_error_cases() {
         let logctx =
             test_setup_log("blueprint_builder_test_add_nexus_error_cases");
-        let (mut collection, policy) = example(DEFAULT_N_SLEDS);
+        let (mut collection, policy) = example(&logctx.log, DEFAULT_N_SLEDS);
 
         // We don't care about the internal DNS version here.
         let internal_dns_version = Generation::new();
@@ -1312,7 +1243,7 @@ pub mod test {
             "blueprint_builder_test_invalid_parent_blueprint_\
              two_zones_with_same_external_ip",
         );
-        let (mut collection, policy) = example(DEFAULT_N_SLEDS);
+        let (mut collection, policy) = example(&logctx.log, DEFAULT_N_SLEDS);
 
         // We should fail if the parent blueprint claims to contain two
         // zones with the same external IP. Skim through the zones, copy the
@@ -1370,7 +1301,7 @@ pub mod test {
             "blueprint_builder_test_invalid_parent_blueprint_\
              two_nexus_zones_with_same_nic_ip",
         );
-        let (mut collection, policy) = example(DEFAULT_N_SLEDS);
+        let (mut collection, policy) = example(&logctx.log, DEFAULT_N_SLEDS);
 
         // We should fail if the parent blueprint claims to contain two
         // Nexus zones with the same NIC IP. Skim through the zones, copy
@@ -1426,7 +1357,7 @@ pub mod test {
             "blueprint_builder_test_invalid_parent_blueprint_\
              two_zones_with_same_vnic_mac",
         );
-        let (mut collection, policy) = example(DEFAULT_N_SLEDS);
+        let (mut collection, policy) = example(&logctx.log, DEFAULT_N_SLEDS);
 
         // We should fail if the parent blueprint claims to contain two
         // zones with the same service vNIC MAC address. Skim through the
diff --git a/nexus/reconfigurator/planning/src/lib.rs b/nexus/reconfigurator/planning/src/lib.rs
index 546f2c1dc1..e0a61826f0 100644
--- a/nexus/reconfigurator/planning/src/lib.rs
+++ b/nexus/reconfigurator/planning/src/lib.rs
@@ -118,3 +118,4 @@
 pub mod blueprint_builder;
 mod ip_allocator;
 pub mod planner;
+pub mod system;
diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs
index 9f4b653507..bae3dfd68a 100644
--- a/nexus/reconfigurator/planning/src/planner.rs
+++ b/nexus/reconfigurator/planning/src/planner.rs
@@ -319,10 +319,11 @@ impl<'a> Planner<'a> {
 mod test {
     use super::Planner;
     use crate::blueprint_builder::test::example;
-    use crate::blueprint_builder::test::policy_add_sled;
     use crate::blueprint_builder::test::verify_blueprint;
+    use crate::blueprint_builder::test::ExampleSystem;
     use crate::blueprint_builder::test::DEFAULT_N_SLEDS;
     use crate::blueprint_builder::BlueprintBuilder;
+    use crate::system::SledBuilder;
     use nexus_inventory::now_db_precision;
     use nexus_types::external_api::views::SledPolicy;
     use nexus_types::external_api::views::SledProvisionPolicy;
@@ -341,14 +342,14 @@ mod test {
         let internal_dns_version = Generation::new();
 
         // Use our example inventory collection.
-        let (mut collection, mut policy) = example(DEFAULT_N_SLEDS);
+        let mut example = ExampleSystem::new(&logctx.log, DEFAULT_N_SLEDS);
 
         // Build the initial blueprint.  We don't bother verifying it here
         // because there's a separate test for that.
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
-            &collection,
+            &example.collection,
             internal_dns_version,
-            &policy,
+            &example.policy,
             "the_test",
         )
         .expect("failed to create initial blueprint");
@@ -361,9 +362,9 @@ mod test {
             logctx.log.clone(),
             &blueprint1,
             internal_dns_version,
-            &policy,
+            &example.policy,
             "no-op?",
-            &collection,
+            &example.collection,
         )
         .expect("failed to create planner")
         .plan()
@@ -379,7 +380,9 @@ mod test {
         // Now add a new sled.
         let new_sled_id =
             "7097f5b3-5896-4fff-bd97-63a9a69563a9".parse().unwrap();
-        let _ = policy_add_sled(&mut policy, new_sled_id);
+        let _ =
+            example.system.sled(SledBuilder::new().id(new_sled_id)).unwrap();
+        let policy = example.system.to_policy().unwrap();
 
         // Check that the first step is to add an NTP zone
         let blueprint3 = Planner::new_based_on(
@@ -388,7 +391,7 @@ mod test {
             internal_dns_version,
             &policy,
             "test: add NTP?",
-            &collection,
+            &example.collection,
         )
         .expect("failed to create planner")
         .plan()
@@ -421,7 +424,7 @@ mod test {
             internal_dns_version,
             &policy,
             "test: add nothing more",
-            &collection,
+            &example.collection,
         )
         .expect("failed to create planner")
         .plan()
@@ -434,6 +437,7 @@ mod test {
         verify_blueprint(&blueprint4);
 
         // Now update the inventory to have the requested NTP zone.
+        let mut collection = example.collection.clone();
         assert!(collection
             .omicron_zones
             .insert(
@@ -479,7 +483,7 @@ mod test {
         assert_eq!(sled_changes.zones_removed().count(), 0);
         assert_eq!(sled_changes.zones_changed().count(), 0);
         let zones = sled_changes.zones_added().collect::<Vec<_>>();
-        assert_eq!(zones.len(), 3);
+        assert_eq!(zones.len(), 10);
         for zone in &zones {
             let OmicronZoneType::Crucible { .. } = zone.zone_type else {
                 panic!("unexpectedly added a non-Crucible zone: {zone:?}");
@@ -522,7 +526,8 @@ mod test {
         // Use our example inventory collection as a starting point, but strip
         // it down to just one sled.
         let (sled_id, collection, mut policy) = {
-            let (mut collection, mut policy) = example(DEFAULT_N_SLEDS);
+            let (mut collection, mut policy) =
+                example(&logctx.log, DEFAULT_N_SLEDS);
 
             // Pick one sled ID to keep and remove the rest.
             let keep_sled_id =
@@ -606,7 +611,7 @@ mod test {
         );
 
         // Use our example inventory collection as a starting point.
-        let (collection, mut policy) = example(DEFAULT_N_SLEDS);
+        let (collection, mut policy) = example(&logctx.log, DEFAULT_N_SLEDS);
 
         // Build the initial blueprint.
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
@@ -692,7 +697,7 @@ mod test {
         // and decommissioned sleds. (When we add more kinds of
         // non-provisionable states in the future, we'll have to add more
         // sleds.)
-        let (collection, mut policy) = example(5);
+        let (collection, mut policy) = example(&logctx.log, 5);
 
         // Build the initial blueprint.
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs
new file mode 100644
index 0000000000..5aff89df0d
--- /dev/null
+++ b/nexus/reconfigurator/planning/src/system.rs
@@ -0,0 +1,578 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Builders for constructing descriptions of systems (real or synthetic) and
+//! associated inventory collections and blueprints
+
+use anyhow::{anyhow, bail, ensure, Context};
+use gateway_client::types::RotState;
+use gateway_client::types::SpState;
+use indexmap::IndexMap;
+use nexus_inventory::CollectionBuilder;
+use nexus_types::deployment::Policy;
+use nexus_types::deployment::SledResources;
+use nexus_types::external_api::views::SledPolicy;
+use nexus_types::external_api::views::SledProvisionPolicy;
+use nexus_types::external_api::views::SledState;
+use nexus_types::inventory::BaseboardId;
+use nexus_types::inventory::PowerState;
+use nexus_types::inventory::RotSlot;
+use nexus_types::inventory::SledRole;
+use nexus_types::inventory::SpType;
+use nexus_types::inventory::ZpoolName;
+use omicron_common::address::get_sled_address;
+use omicron_common::address::IpRange;
+use omicron_common::address::Ipv6Subnet;
+use omicron_common::address::NEXUS_REDUNDANCY;
+use omicron_common::address::RACK_PREFIX;
+use omicron_common::address::SLED_PREFIX;
+use omicron_common::api::external::ByteCount;
+use std::collections::BTreeSet;
+use std::fmt::Debug;
+use std::net::Ipv4Addr;
+use std::net::Ipv6Addr;
+use uuid::Uuid;
+
+trait SubnetIterator: Iterator<Item = Ipv6Subnet<SLED_PREFIX>> + Debug {}
+impl<T> SubnetIterator for T where
+    T: Iterator<Item = Ipv6Subnet<SLED_PREFIX>> + Debug
+{
+}
+
+/// Describes an actual or synthetic Oxide rack for planning and testing
+///
+/// From this description, you can extract a `Policy` or inventory `Collection`.
+/// There are a few intended purposes here:
+///
+/// 1. to easily construct fake racks in automated tests for the Planner and
+///    other parts of Reconfigurator
+///
+/// 2. to explore the Planner's behavior via the `reconfigurator-cli` tool
+///
+/// 3. eventually: to commonize code between Reconfigurator and RSS.  This is
+///    more speculative at this point, but the idea here is that RSS itself
+///    could construct a `SystemDescription` and then use the facilities here to
+///    assign subnets and maybe even lay out the initial set of zones (which
+///    does not exist here yet).  This way Reconfigurator and RSS are using the
+///    same code to do this.
+#[derive(Debug)]
+pub struct SystemDescription {
+    collector: Option<String>,
+    sleds: IndexMap<Uuid, Sled>,
+    sled_subnets: Box<dyn SubnetIterator>,
+    available_non_scrimlet_slots: BTreeSet<u16>,
+    available_scrimlet_slots: BTreeSet<u16>,
+    target_nexus_zone_count: usize,
+    service_ip_pool_ranges: Vec<IpRange>,
+}
+
+impl SystemDescription {
+    pub fn new() -> Self {
+        // Prepare sets of available slots (cubby numbers) for (1) all
+        // non-Scrimlet sleds, and (2) Scrimlets in particular.  These do not
+        // overlap.
+        //
+        // We will use these in two places:
+        //
+        // (1) when the caller specifies what slot a sled should go into, to
+        //     validate that the slot is available and make sure we don't use it
+        //     again
+        //
+        // (2) when the caller adds a sled but leaves the slot unspecified, so
+        //     that we can assign it a slot number
+        //
+        // We use `BTreeSet` because it efficiently expresses what we want,
+        // though the set sizes are small enough that it doesn't much matter.
+        let available_scrimlet_slots: BTreeSet<u16> = BTreeSet::from([14, 16]);
+        let available_non_scrimlet_slots: BTreeSet<u16> = (0..=31)
+            .collect::<BTreeSet<_>>()
+            .difference(&available_scrimlet_slots)
+            .copied()
+            .collect();
+
+        // Prepare an iterator to allow us to assign sled subnets.
+        let rack_subnet_base: Ipv6Addr =
+            "fd00:1122:3344:0100::".parse().unwrap();
+        let rack_subnet =
+            ipnet::Ipv6Net::new(rack_subnet_base, RACK_PREFIX).unwrap();
+        // Skip the initial DNS subnet.
+        // (The same behavior is replicated in RSS in `Plan::create()` in
+        // sled-agent/src/rack_setup/plan/sled.rs.)
+        let sled_subnets = Box::new(
+            rack_subnet
+                .subnets(SLED_PREFIX)
+                .unwrap()
+                .skip(1)
+                .map(|s| Ipv6Subnet::new(s.network())),
+        );
+
+        // Policy defaults
+        let target_nexus_zone_count = NEXUS_REDUNDANCY;
+        // IPs from TEST-NET-1 (RFC 5737)
+        let service_ip_pool_ranges = vec![IpRange::try_from((
+            "192.0.2.2".parse::<Ipv4Addr>().unwrap(),
+            "192.0.2.20".parse::<Ipv4Addr>().unwrap(),
+        ))
+        .unwrap()];
+
+        SystemDescription {
+            sleds: IndexMap::new(),
+            collector: None,
+            sled_subnets,
+            available_non_scrimlet_slots,
+            available_scrimlet_slots,
+            target_nexus_zone_count,
+            service_ip_pool_ranges,
+        }
+    }
+
+    /// Returns a complete system deployed on a single Sled
+    pub fn single_sled() -> anyhow::Result<Self> {
+        let mut builder = SystemDescription::new();
+        let sled = SledBuilder::new();
+        builder.sled(sled)?;
+        Ok(builder)
+    }
+
+    /// Returns a complete system resembling a full rack
+    pub fn full_rack() -> anyhow::Result<Self> {
+        let mut builder = SystemDescription::new();
+        for slot_number in 1..32 {
+            let mut sled = SledBuilder::new();
+            if slot_number == 14 || slot_number == 16 {
+                sled = sled.sled_role(SledRole::Scrimlet);
+            }
+            builder.sled(sled)?;
+        }
+        Ok(builder)
+    }
+
+    pub fn collector_label<S>(&mut self, collector_label: S) -> &mut Self
+    where
+        String: From<S>,
+    {
+        self.collector = Some(String::from(collector_label));
+        self
+    }
+
+    pub fn target_nexus_zone_count(&mut self, count: usize) -> &mut Self {
+        self.target_nexus_zone_count = count;
+        self
+    }
+
+    pub fn service_ip_pool_ranges(
+        &mut self,
+        ranges: Vec<IpRange>,
+    ) -> &mut Self {
+        self.service_ip_pool_ranges = ranges;
+        self
+    }
+
+    /// Add a sled to the system, as described by a SledBuilder
+    pub fn sled(&mut self, sled: SledBuilder) -> anyhow::Result<&mut Self> {
+        let sled_id = sled.id.unwrap_or_else(Uuid::new_v4);
+        ensure!(
+            !self.sleds.contains_key(&sled_id),
+            "attempted to add sled with the same id as an existing one: {}",
+            sled_id
+        );
+        let sled_subnet = self
+            .sled_subnets
+            .next()
+            .ok_or_else(|| anyhow!("ran out of IPv6 subnets for sleds"))?;
+        let hardware_slot = if let Some(slot) = sled.hardware_slot {
+            // If the caller specified a slot number, use that.
+            // Make sure it's still available, though.
+            if !self.available_scrimlet_slots.remove(&slot)
+                && !self.available_non_scrimlet_slots.remove(&slot)
+            {
+                bail!("sled slot {} was used twice", slot);
+            }
+            slot
+        } else if sled.sled_role == SledRole::Scrimlet {
+            // Otherwise, if this is a Scrimlet, it must be in one of the
+            // allowed Scrimlet slots.
+            self.available_scrimlet_slots
+                .pop_first()
+                .ok_or_else(|| anyhow!("ran out of slots for Scrimlets"))?
+        } else {
+            // Otherwise, prefer a non-Scrimlet slots, but allow a Scrimlet slot
+            // to be used if we run out of non-Scrimlet slots.
+            self.available_non_scrimlet_slots
+                .pop_first()
+                .or_else(|| self.available_scrimlet_slots.pop_first())
+                .ok_or_else(|| anyhow!("ran out of slots for non-Scrimlets"))?
+        };
+
+        let sled = Sled::new_simulated(
+            sled_id,
+            sled_subnet,
+            sled.sled_role,
+            sled.unique,
+            sled.hardware,
+            hardware_slot,
+            sled.npools,
+        );
+        self.sleds.insert(sled_id, sled);
+        Ok(self)
+    }
+
+    /// Add a sled to the system based on information that came from the
+    /// database of an existing system
+    pub fn sled_full(
+        &mut self,
+        sled_id: Uuid,
+        sled_resources: SledResources,
+        inventory_sp: Option<SledHwInventory<'_>>,
+        inventory_sled_agent: &nexus_types::inventory::SledAgent,
+    ) -> anyhow::Result<&mut Self> {
+        ensure!(
+            !self.sleds.contains_key(&sled_id),
+            "attempted to add sled with the same id as an existing one: {}",
+            sled_id
+        );
+        self.sleds.insert(
+            sled_id,
+            Sled::new_full(
+                sled_id,
+                sled_resources,
+                inventory_sp,
+                inventory_sled_agent,
+            ),
+        );
+        Ok(self)
+    }
+
+    pub fn to_collection_builder(&self) -> anyhow::Result<CollectionBuilder> {
+        let collector_label = self
+            .collector
+            .as_ref()
+            .cloned()
+            .unwrap_or_else(|| String::from("example"));
+        let mut builder = CollectionBuilder::new(collector_label);
+
+        for s in self.sleds.values() {
+            if let Some((slot, sp_state)) = s.sp_state() {
+                builder
+                    .found_sp_state(
+                        "fake MGS 1",
+                        SpType::Sled,
+                        u32::from(*slot),
+                        sp_state.clone(),
+                    )
+                    .context("recording SP state")?;
+            }
+
+            builder
+                .found_sled_inventory(
+                    "fake sled agent",
+                    s.sled_agent_inventory().clone(),
+                )
+                .context("recording sled agent")?;
+        }
+
+        Ok(builder)
+    }
+
+    pub fn to_policy(&self) -> anyhow::Result<Policy> {
+        let sleds = self
+            .sleds
+            .values()
+            .map(|sled| {
+                let sled_resources = SledResources {
+                    policy: sled.policy,
+                    state: SledState::Active,
+                    zpools: sled.zpools.iter().cloned().collect(),
+                    subnet: sled.sled_subnet,
+                };
+                (sled.sled_id, sled_resources)
+            })
+            .collect();
+
+        Ok(Policy {
+            sleds,
+            service_ip_pool_ranges: self.service_ip_pool_ranges.clone(),
+            target_nexus_zone_count: self.target_nexus_zone_count,
+        })
+    }
+}
+
+#[derive(Clone, Debug)]
+pub enum SledHardware {
+    Gimlet,
+    Pc,
+    Unknown,
+    Empty,
+}
+
+#[derive(Clone, Debug)]
+pub struct SledBuilder {
+    id: Option<Uuid>,
+    unique: Option<String>,
+    hardware: SledHardware,
+    hardware_slot: Option<u16>,
+    sled_role: SledRole,
+    npools: u8,
+}
+
+impl SledBuilder {
+    /// Begin describing a sled to be added to a `SystemDescription`
+    pub fn new() -> Self {
+        SledBuilder {
+            id: None,
+            unique: None,
+            hardware: SledHardware::Gimlet,
+            hardware_slot: None,
+            sled_role: SledRole::Gimlet,
+            npools: 10,
+        }
+    }
+
+    /// Set the id of the sled
+    ///
+    /// Default: randomly generated
+    pub fn id(mut self, id: Uuid) -> Self {
+        self.id = Some(id);
+        self
+    }
+
+    /// Set a unique string used to generate the serial number and other
+    /// identifiers
+    ///
+    /// Default: randomly generated
+    pub fn unique<S>(mut self, unique: S) -> Self
+    where
+        String: From<S>,
+    {
+        self.unique = Some(String::from(unique));
+        self
+    }
+
+    /// Set the number of U.2 (external) pools this sled should have
+    ///
+    /// Default is currently `10` based on the typical value for a Gimlet
+    pub fn npools(mut self, npools: u8) -> Self {
+        self.npools = npools;
+        self
+    }
+
+    /// Sets what type of hardware this sled uses
+    ///
+    /// Default: `SledHarware::Gimlet`
+    pub fn hardware(mut self, hardware: SledHardware) -> Self {
+        self.hardware = hardware;
+        self
+    }
+
+    /// Sets which cubby in the rack the sled is in
+    ///
+    /// Default: determined based on sled role and unused slots
+    pub fn hardware_slot(mut self, hardware_slot: u16) -> Self {
+        self.hardware_slot = Some(hardware_slot);
+        self
+    }
+
+    /// Sets whether this sled is attached to a switch (`SledRole::Scrimlet`) or
+    /// not (`SledRole::Gimlet`)
+    pub fn sled_role(mut self, sled_role: SledRole) -> Self {
+        self.sled_role = sled_role;
+        self
+    }
+}
+
+/// Convenience structure summarizing `Sled` inputs that come from inventory
+#[derive(Debug)]
+pub struct SledHwInventory<'a> {
+    pub baseboard_id: &'a BaseboardId,
+    pub sp: &'a nexus_types::inventory::ServiceProcessor,
+    pub rot: &'a nexus_types::inventory::RotState,
+}
+
+/// Our abstract description of a `Sled`
+///
+/// This needs to be rich enough to generate a Policy and inventory Collection.
+#[derive(Clone, Debug)]
+struct Sled {
+    sled_id: Uuid,
+    sled_subnet: Ipv6Subnet<SLED_PREFIX>,
+    inventory_sp: Option<(u16, SpState)>,
+    inventory_sled_agent: sled_agent_client::types::Inventory,
+    zpools: Vec<ZpoolName>,
+    policy: SledPolicy,
+}
+
+impl Sled {
+    /// Create a `Sled` using faked-up information based on a `SledBuilder`
+    fn new_simulated(
+        sled_id: Uuid,
+        sled_subnet: Ipv6Subnet<SLED_PREFIX>,
+        sled_role: SledRole,
+        unique: Option<String>,
+        hardware: SledHardware,
+        hardware_slot: u16,
+        nzpools: u8,
+    ) -> Sled {
+        let unique = unique.unwrap_or_else(|| hardware_slot.to_string());
+        let model = format!("model{}", unique);
+        let serial = format!("serial{}", unique);
+        let revision = 0;
+        let zpools = (0..nzpools)
+            .map(|_| format!("oxp_{}", Uuid::new_v4()).parse().unwrap())
+            .collect();
+        let inventory_sp = match hardware {
+            SledHardware::Empty => None,
+            SledHardware::Gimlet | SledHardware::Pc | SledHardware::Unknown => {
+                Some((
+                    hardware_slot,
+                    SpState {
+                        base_mac_address: [0; 6],
+                        hubris_archive_id: format!("hubris{}", unique),
+                        model: model.clone(),
+                        power_state: PowerState::A2,
+                        revision,
+                        rot: RotState::Enabled {
+                            active: RotSlot::A,
+                            pending_persistent_boot_preference: None,
+                            persistent_boot_preference: RotSlot::A,
+                            slot_a_sha3_256_digest: Some(String::from(
+                                "slotAdigest1",
+                            )),
+                            slot_b_sha3_256_digest: Some(String::from(
+                                "slotBdigest1",
+                            )),
+                            transient_boot_preference: None,
+                        },
+                        serial_number: serial.clone(),
+                    },
+                ))
+            }
+        };
+
+        let inventory_sled_agent = {
+            let baseboard = match hardware {
+                SledHardware::Gimlet => {
+                    sled_agent_client::types::Baseboard::Gimlet {
+                        identifier: serial.clone(),
+                        model: model.clone(),
+                        revision: i64::from(revision),
+                    }
+                }
+                SledHardware::Pc => sled_agent_client::types::Baseboard::Pc {
+                    identifier: serial.clone(),
+                    model: model.clone(),
+                },
+                SledHardware::Unknown | SledHardware::Empty => {
+                    sled_agent_client::types::Baseboard::Unknown
+                }
+            };
+            let sled_agent_address = get_sled_address(sled_subnet).to_string();
+            sled_agent_client::types::Inventory {
+                baseboard,
+                reservoir_size: ByteCount::from(1024),
+                sled_role,
+                sled_agent_address,
+                sled_id: sled_id,
+                usable_hardware_threads: 10,
+                usable_physical_ram: ByteCount::from(1024 * 1024),
+            }
+        };
+
+        Sled {
+            sled_id,
+            sled_subnet,
+            inventory_sp,
+            inventory_sled_agent,
+            zpools,
+            policy: SledPolicy::InService {
+                provision_policy: SledProvisionPolicy::Provisionable,
+            },
+        }
+    }
+
+    /// Create a `Sled` based on real information from another `Policy` and
+    /// inventory `Collection`
+    fn new_full(
+        sled_id: Uuid,
+        sled_resources: SledResources,
+        inventory_sp: Option<SledHwInventory<'_>>,
+        inv_sled_agent: &nexus_types::inventory::SledAgent,
+    ) -> Sled {
+        // Elsewhere, the user gives us some rough parameters (like a unique
+        // string) that we use to construct fake `sled_agent_client` types that
+        // we can provide to the inventory builder so that _it_ can construct
+        // the corresponding inventory types.  Here, we're working backwards,
+        // which is a little weird: we're given inventory types and we construct
+        // the fake `sled_agent_client` types, again so that we can later pass
+        // them to the inventory builder so that it can construct the same
+        // inventory types again.  This is a little goofy.
+        let baseboard = inventory_sp
+            .as_ref()
+            .map(|sledhw| sled_agent_client::types::Baseboard::Gimlet {
+                identifier: sledhw.baseboard_id.serial_number.clone(),
+                model: sledhw.baseboard_id.part_number.clone(),
+                revision: i64::from(sledhw.sp.baseboard_revision),
+            })
+            .unwrap_or(sled_agent_client::types::Baseboard::Unknown);
+
+        let inventory_sp = inventory_sp.map(|sledhw| {
+            let sp_state = SpState {
+                base_mac_address: [0; 6],
+                hubris_archive_id: sledhw.sp.hubris_archive.clone(),
+                model: sledhw.baseboard_id.part_number.clone(),
+                power_state: sledhw.sp.power_state,
+                revision: sledhw.sp.baseboard_revision,
+                rot: RotState::Enabled {
+                    active: sledhw.rot.active_slot,
+                    pending_persistent_boot_preference: sledhw
+                        .rot
+                        .pending_persistent_boot_preference,
+                    persistent_boot_preference: sledhw
+                        .rot
+                        .persistent_boot_preference,
+                    slot_a_sha3_256_digest: sledhw
+                        .rot
+                        .slot_a_sha3_256_digest
+                        .clone(),
+                    slot_b_sha3_256_digest: sledhw
+                        .rot
+                        .slot_b_sha3_256_digest
+                        .clone(),
+                    transient_boot_preference: sledhw
+                        .rot
+                        .transient_boot_preference,
+                },
+                serial_number: sledhw.baseboard_id.serial_number.clone(),
+            };
+
+            (sledhw.sp.sp_slot, sp_state)
+        });
+
+        let inventory_sled_agent = sled_agent_client::types::Inventory {
+            baseboard,
+            reservoir_size: inv_sled_agent.reservoir_size,
+            sled_role: inv_sled_agent.sled_role,
+            sled_agent_address: inv_sled_agent.sled_agent_address.to_string(),
+            sled_id,
+            usable_hardware_threads: inv_sled_agent.usable_hardware_threads,
+            usable_physical_ram: inv_sled_agent.usable_physical_ram,
+        };
+
+        Sled {
+            sled_id,
+            sled_subnet: sled_resources.subnet,
+            zpools: sled_resources.zpools.into_iter().collect(),
+            inventory_sp,
+            inventory_sled_agent,
+            policy: sled_resources.policy,
+        }
+    }
+
+    fn sp_state(&self) -> Option<&(u16, SpState)> {
+        self.inventory_sp.as_ref()
+    }
+
+    fn sled_agent_inventory(&self) -> &sled_agent_client::types::Inventory {
+        &self.inventory_sled_agent
+    }
+}
diff --git a/nexus/reconfigurator/preparation/Cargo.toml b/nexus/reconfigurator/preparation/Cargo.toml
new file mode 100644
index 0000000000..f95f9c4afe
--- /dev/null
+++ b/nexus/reconfigurator/preparation/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "nexus-reconfigurator-preparation"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+illumos-utils.workspace = true
+nexus-db-model.workspace = true
+nexus-types.workspace = true
+omicron-common.workspace = true
+
+omicron-workspace-hack.workspace = true
diff --git a/nexus/reconfigurator/preparation/src/lib.rs b/nexus/reconfigurator/preparation/src/lib.rs
new file mode 100644
index 0000000000..77d4532023
--- /dev/null
+++ b/nexus/reconfigurator/preparation/src/lib.rs
@@ -0,0 +1,74 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Common facilities for assembling inputs to the planner
+
+use nexus_types::deployment::Policy;
+use nexus_types::deployment::SledResources;
+use nexus_types::deployment::ZpoolName;
+use nexus_types::identity::Asset;
+use omicron_common::address::IpRange;
+use omicron_common::address::Ipv6Subnet;
+use omicron_common::address::SLED_PREFIX;
+use omicron_common::api::external::Error;
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
+use std::str::FromStr;
+
+/// Given various pieces of database state that go into the blueprint planning
+/// process, produce a `Policy` object encapsulating what the planner needs to
+/// generate a blueprint
+pub fn policy_from_db(
+    sled_rows: &[nexus_db_model::Sled],
+    zpool_rows: &[nexus_db_model::Zpool],
+    ip_pool_range_rows: &[nexus_db_model::IpPoolRange],
+    target_nexus_zone_count: usize,
+) -> Result<Policy, Error> {
+    let mut zpools_by_sled_id = {
+        let mut zpools = BTreeMap::new();
+        for z in zpool_rows {
+            let sled_zpool_names =
+                zpools.entry(z.sled_id).or_insert_with(BTreeSet::new);
+            // It's unfortunate that Nexus knows how Sled Agent
+            // constructs zpool names, but there's not currently an
+            // alternative.
+            let zpool_name_generated =
+                illumos_utils::zpool::ZpoolName::new_external(z.id())
+                    .to_string();
+            let zpool_name = ZpoolName::from_str(&zpool_name_generated)
+                .map_err(|e| {
+                    Error::internal_error(&format!(
+                        "unexpectedly failed to parse generated \
+                                zpool name: {}: {}",
+                        zpool_name_generated, e
+                    ))
+                })?;
+            sled_zpool_names.insert(zpool_name);
+        }
+        zpools
+    };
+
+    let sleds = sled_rows
+        .into_iter()
+        .map(|sled_row| {
+            let sled_id = sled_row.id();
+            let subnet = Ipv6Subnet::<SLED_PREFIX>::new(sled_row.ip());
+            let zpools = zpools_by_sled_id
+                .remove(&sled_id)
+                .unwrap_or_else(BTreeSet::new);
+            let sled_info = SledResources {
+                policy: sled_row.policy(),
+                state: sled_row.state().into(),
+                subnet,
+                zpools,
+            };
+            (sled_id, sled_info)
+        })
+        .collect();
+
+    let service_ip_pool_ranges =
+        ip_pool_range_rows.iter().map(IpRange::from).collect();
+
+    Ok(Policy { sleds, service_ip_pool_ranges, target_nexus_zone_count })
+}
diff --git a/nexus/src/app/deployment.rs b/nexus/src/app/deployment.rs
index 31ba9fe065..778e768878 100644
--- a/nexus/src/app/deployment.rs
+++ b/nexus/src/app/deployment.rs
@@ -7,23 +7,16 @@
 use nexus_db_model::DnsGroup;
 use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
-use nexus_db_queries::db::datastore::SQL_BATCH_SIZE;
-use nexus_db_queries::db::pagination::Paginator;
 use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder;
 use nexus_reconfigurator_planning::planner::Planner;
+use nexus_reconfigurator_preparation::policy_from_db;
 use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::BlueprintMetadata;
 use nexus_types::deployment::BlueprintTarget;
 use nexus_types::deployment::BlueprintTargetSet;
 use nexus_types::deployment::Policy;
-use nexus_types::deployment::SledResources;
-use nexus_types::deployment::ZpoolName;
-use nexus_types::identity::Asset;
 use nexus_types::inventory::Collection;
-use omicron_common::address::IpRange;
-use omicron_common::address::Ipv6Subnet;
 use omicron_common::address::NEXUS_REDUNDANCY;
-use omicron_common::address::SLED_PREFIX;
 use omicron_common::api::external::CreateResult;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::DeleteResult;
@@ -34,9 +27,6 @@ use omicron_common::api::external::ListResultVec;
 use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::LookupType;
 use slog_error_chain::InlineErrorChain;
-use std::collections::BTreeMap;
-use std::collections::BTreeSet;
-use std::str::FromStr;
 use uuid::Uuid;
 
 /// Common structure for collecting information that the planner needs
@@ -118,81 +108,23 @@ impl super::Nexus {
         let datastore = self.datastore();
 
         let sled_rows = datastore.sled_list_all_batched(opctx).await?;
-
-        let mut zpools_by_sled_id = {
-            let mut zpools = BTreeMap::new();
-            let mut paginator = Paginator::new(SQL_BATCH_SIZE);
-            while let Some(p) = paginator.next() {
-                let batch = datastore
-                    .zpool_list_all_external(opctx, &p.current_pagparams())
-                    .await?;
-                paginator =
-                    p.found_batch(&batch, &|z: &nexus_db_model::Zpool| z.id());
-                for z in batch {
-                    let sled_zpool_names =
-                        zpools.entry(z.sled_id).or_insert_with(BTreeSet::new);
-                    // It's unfortunate that Nexus knows how Sled Agent
-                    // constructs zpool names, but there's not currently an
-                    // alternative.
-                    let zpool_name_generated =
-                        illumos_utils::zpool::ZpoolName::new_external(z.id())
-                            .to_string();
-                    let zpool_name = ZpoolName::from_str(&zpool_name_generated)
-                        .map_err(|e| {
-                            Error::internal_error(&format!(
-                                "unexpectedly failed to parse generated \
-                                zpool name: {}: {}",
-                                zpool_name_generated, e
-                            ))
-                        })?;
-                    sled_zpool_names.insert(zpool_name);
-                }
-            }
-            zpools
-        };
-
-        let sleds = sled_rows
-            .into_iter()
-            .map(|sled_row| {
-                let sled_id = sled_row.id();
-                let subnet = Ipv6Subnet::<SLED_PREFIX>::new(sled_row.ip());
-                let zpools = zpools_by_sled_id
-                    .remove(&sled_id)
-                    .unwrap_or_else(BTreeSet::new);
-                let sled_info = SledResources {
-                    policy: sled_row.policy(),
-                    state: sled_row.state().into(),
-                    subnet,
-                    zpools,
-                };
-                (sled_id, sled_info)
-            })
-            .collect();
-
-        let service_ip_pool_ranges = {
+        let zpool_rows =
+            datastore.zpool_list_all_external_batched(opctx).await?;
+        let ip_pool_range_rows = {
             let (authz_service_ip_pool, _) =
                 datastore.ip_pools_service_lookup(opctx).await?;
-
-            let mut ip_ranges = Vec::new();
-            let mut paginator = Paginator::new(SQL_BATCH_SIZE);
-            while let Some(p) = paginator.next() {
-                let batch = datastore
-                    .ip_pool_list_ranges(
-                        opctx,
-                        &authz_service_ip_pool,
-                        &p.current_pagparams(),
-                    )
-                    .await?;
-                // The use of `last_address` here assumes `paginator` is sorting
-                // in Ascending order (which it does - see the implementation of
-                // `current_pagparams()`).
-                paginator = p.found_batch(&batch, &|r| r.last_address);
-                ip_ranges.extend(batch.iter().map(IpRange::from));
-            }
-
-            ip_ranges
+            datastore
+                .ip_pool_list_ranges_batched(opctx, &authz_service_ip_pool)
+                .await?
         };
 
+        let policy = policy_from_db(
+            &sled_rows,
+            &zpool_rows,
+            &ip_pool_range_rows,
+            NEXUS_REDUNDANCY,
+        )?;
+
         // The choice of which inventory collection to use here is not
         // necessarily trivial.  Inventory collections may be incomplete due to
         // transient (or even persistent) errors.  It's not yet clear what
@@ -223,11 +155,7 @@ impl super::Nexus {
 
         Ok(PlanningContext {
             creator,
-            policy: Policy {
-                sleds,
-                service_ip_pool_ranges,
-                target_nexus_zone_count: NEXUS_REDUNDANCY,
-            },
+            policy,
             inventory,
             internal_dns_version: *dns_version.version,
         })
diff --git a/nexus/types/Cargo.toml b/nexus/types/Cargo.toml
index 93716d4804..ecc180b6db 100644
--- a/nexus/types/Cargo.toml
+++ b/nexus/types/Cargo.toml
@@ -9,12 +9,14 @@ anyhow.workspace = true
 chrono.workspace = true
 base64.workspace = true
 futures.workspace = true
+humantime.workspace = true
 omicron-uuid-kinds.workspace = true
 openssl.workspace = true
 parse-display.workspace = true
 schemars = { workspace = true, features = ["chrono", "uuid1"] }
 serde.workspace = true
 serde_json.workspace = true
+serde_with.workspace = true
 steno.workspace = true
 strum.workspace = true
 thiserror.workspace = true
diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs
index ef3c03a302..ac950d2ca3 100644
--- a/nexus/types/src/deployment.rs
+++ b/nexus/types/src/deployment.rs
@@ -32,6 +32,7 @@ use serde::Deserialize;
 use serde::Serialize;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
+use std::fmt;
 use uuid::Uuid;
 
 /// Fleet-wide deployment policy
@@ -49,7 +50,7 @@ use uuid::Uuid;
 ///
 /// The current policy is pretty limited.  It's aimed primarily at supporting
 /// the add/remove sled use case.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Policy {
     /// set of sleds that are supposed to be part of the control plane, along
     /// with information about resources available to the planner
@@ -64,7 +65,7 @@ pub struct Policy {
 }
 
 /// Describes the resources available on each sled for the planner
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SledResources {
     /// current sled policy
     pub policy: SledPolicy,
@@ -134,7 +135,7 @@ impl SledResources {
 // zones deployed on each host and some supporting configuration (e.g., DNS).
 // This is aimed at supporting add/remove sleds.  The plan is to grow this to
 // include more of the system as we support more use cases.
-#[derive(Debug, Clone, Eq, PartialEq, JsonSchema, Deserialize, Serialize)]
+#[derive(Clone, Eq, PartialEq, JsonSchema, Deserialize, Serialize)]
 pub struct Blueprint {
     /// unique identifier for this blueprint
     pub id: Uuid,
@@ -225,6 +226,59 @@ impl Blueprint {
     }
 }
 
+impl fmt::Debug for Blueprint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        writeln!(f, "blueprint  {}", self.id)?;
+        writeln!(
+            f,
+            "parent:    {}",
+            self.parent_blueprint_id
+                .map(|u| u.to_string())
+                .unwrap_or_else(|| String::from("<none>"))
+        )?;
+        writeln!(
+            f,
+            "created by {}{}",
+            self.creator,
+            if self.creator.parse::<Uuid>().is_ok() {
+                " (likely a Nexus instance)"
+            } else {
+                ""
+            }
+        )?;
+        writeln!(
+            f,
+            "created at {}",
+            humantime::format_rfc3339_millis(self.time_created.into(),)
+        )?;
+        writeln!(f, "internal DNS version: {}", self.internal_dns_version)?;
+        writeln!(f, "comment: {}", self.comment)?;
+        writeln!(f, "zones:\n")?;
+        for (sled_id, sled_zones) in &self.omicron_zones {
+            writeln!(
+                f,
+                "  sled {}: Omicron zones at generation {}",
+                sled_id, sled_zones.generation
+            )?;
+            for z in &sled_zones.zones {
+                writeln!(
+                    f,
+                    "    {} {} {}",
+                    z.id,
+                    if self.zones_in_service.contains(&z.id) {
+                        "in service    "
+                    } else {
+                        "not in service"
+                    },
+                    z.zone_type.label(),
+                )?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
 /// Describe high-level metadata about a blueprint
 // These fields are a subset of [`Blueprint`], and include only the data we can
 // quickly fetch from the main blueprint table (e.g., when listing all
@@ -494,12 +548,12 @@ impl<'a> OmicronZonesDiff<'a> {
 
     fn print_whole_sled(
         &self,
-        f: &mut std::fmt::Formatter<'_>,
+        f: &mut fmt::Formatter<'_>,
         prefix: char,
         label: &str,
         bbsledzones: &OmicronZonesConfig,
         sled_id: Uuid,
-    ) -> std::fmt::Result {
+    ) -> fmt::Result {
         writeln!(f, "{} sled {} ({})", prefix, sled_id, label)?;
         writeln!(
             f,
@@ -523,8 +577,8 @@ impl<'a> OmicronZonesDiff<'a> {
 }
 
 /// Implements diff(1)-like output for diff'ing two blueprints
-impl<'a> std::fmt::Display for OmicronZonesDiff<'a> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+impl<'a> fmt::Display for OmicronZonesDiff<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         writeln!(f, "diff {} {}", self.before_label, self.after_label)?;
         writeln!(f, "--- {}", self.before_label)?;
         writeln!(f, "+++ {}", self.after_label)?;
@@ -655,3 +709,17 @@ impl<'a> std::fmt::Display for OmicronZonesDiff<'a> {
         Ok(())
     }
 }
+
+/// Encapsulates Reconfigurator state
+///
+/// This serialized from is intended for saving state from hand-constructed or
+/// real, deployed systems and loading it back into a simulator or test suite
+///
+/// **This format is not stable.  It may change at any time without
+/// backwards-compatibility guarantees.**
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct UnstableReconfiguratorState {
+    pub policy: Policy,
+    pub collections: Vec<Collection>,
+    pub blueprints: Vec<Blueprint>,
+}
diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs
index 71e8e64d97..57fc7bd647 100644
--- a/nexus/types/src/inventory.rs
+++ b/nexus/types/src/inventory.rs
@@ -18,6 +18,8 @@ pub use gateway_client::types::RotSlot;
 pub use gateway_client::types::SpType;
 use omicron_common::api::external::ByteCount;
 pub use omicron_common::api::internal::shared::SourceNatConfig;
+use serde::{Deserialize, Serialize};
+use serde_with::serde_as;
 pub use sled_agent_client::types::NetworkInterface;
 pub use sled_agent_client::types::NetworkInterfaceKind;
 pub use sled_agent_client::types::OmicronZoneConfig;
@@ -48,7 +50,8 @@ use uuid::Uuid;
 /// database.
 ///
 /// See the documentation in the database schema for more background.
-#[derive(Debug, Eq, PartialEq, Clone)]
+#[serde_as]
+#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)]
 pub struct Collection {
     /// unique identifier for this collection
     pub id: Uuid,
@@ -79,16 +82,19 @@ pub struct Collection {
     ///
     /// In practice, these will be inserted into the `inv_service_processor`
     /// table.
+    #[serde_as(as = "Vec<(_, _)>")]
     pub sps: BTreeMap<Arc<BaseboardId>, ServiceProcessor>,
     /// all roots of trust, keyed by baseboard id
     ///
     /// In practice, these will be inserted into the `inv_root_of_trust` table.
+    #[serde_as(as = "Vec<(_, _)>")]
     pub rots: BTreeMap<Arc<BaseboardId>, RotState>,
     /// all caboose contents found, keyed first by the kind of caboose
     /// (`CabooseWhich`), then the baseboard id of the sled where they were
     /// found
     ///
     /// In practice, these will be inserted into the `inv_caboose` table.
+    #[serde_as(as = "BTreeMap<_, Vec<(_, _)>>")]
     pub cabooses_found:
         BTreeMap<CabooseWhich, BTreeMap<Arc<BaseboardId>, CabooseFound>>,
     /// all root of trust page contents found, keyed first by the kind of page
@@ -97,6 +103,7 @@ pub struct Collection {
     ///
     /// In practice, these will be inserted into the `inv_root_of_trust_page`
     /// table.
+    #[serde_as(as = "BTreeMap<_, Vec<(_, _)>>")]
     pub rot_pages_found:
         BTreeMap<RotPageWhich, BTreeMap<Arc<BaseboardId>, RotPageFound>>,
 
@@ -157,7 +164,9 @@ impl Collection {
 /// number.  We do not include that here.  If we ever did find a baseboard with
 /// the same part number and serial number but a new revision number, we'd want
 /// to treat that as the same baseboard as one with a different revision number.
-#[derive(Clone, Debug, Ord, Eq, PartialOrd, PartialEq)]
+#[derive(
+    Clone, Debug, Ord, Eq, PartialOrd, PartialEq, Deserialize, Serialize,
+)]
 pub struct BaseboardId {
     /// Oxide Part Number
     pub part_number: String,
@@ -181,7 +190,9 @@ impl From<UninitializedSledId> for BaseboardId {
 ///
 /// These are normalized in the database.  Each distinct `Caboose` is assigned a
 /// uuid and shared across many possible collections that reference it.
-#[derive(Clone, Debug, Ord, Eq, PartialOrd, PartialEq)]
+#[derive(
+    Clone, Debug, Ord, Eq, PartialOrd, PartialEq, Deserialize, Serialize,
+)]
 pub struct Caboose {
     pub board: String,
     pub git_commit: String,
@@ -202,7 +213,9 @@ impl From<gateway_client::types::SpComponentCaboose> for Caboose {
 
 /// Indicates that a particular `Caboose` was found (at a particular time from a
 /// particular source, but these are only for debugging)
-#[derive(Clone, Debug, Ord, Eq, PartialOrd, PartialEq)]
+#[derive(
+    Clone, Debug, Ord, Eq, PartialOrd, PartialEq, Deserialize, Serialize,
+)]
 pub struct CabooseFound {
     pub time_collected: DateTime<Utc>,
     pub source: String,
@@ -210,7 +223,9 @@ pub struct CabooseFound {
 }
 
 /// Describes a service processor found during collection
-#[derive(Clone, Debug, Ord, Eq, PartialOrd, PartialEq)]
+#[derive(
+    Clone, Debug, Ord, Eq, PartialOrd, PartialEq, Deserialize, Serialize,
+)]
 pub struct ServiceProcessor {
     pub time_collected: DateTime<Utc>,
     pub source: String,
@@ -225,7 +240,9 @@ pub struct ServiceProcessor {
 
 /// Describes the root of trust state found (from a service processor) during
 /// collection
-#[derive(Clone, Debug, Ord, Eq, PartialOrd, PartialEq)]
+#[derive(
+    Clone, Debug, Ord, Eq, PartialOrd, PartialEq, Deserialize, Serialize,
+)]
 pub struct RotState {
     pub time_collected: DateTime<Utc>,
     pub source: String,
@@ -239,7 +256,18 @@ pub struct RotState {
 }
 
 /// Describes which caboose this is (which component, which slot)
-#[derive(Clone, Copy, Debug, EnumIter, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    EnumIter,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Deserialize,
+    Serialize,
+)]
 pub enum CabooseWhich {
     SpSlot0,
     SpSlot1,
@@ -251,14 +279,18 @@ pub enum CabooseWhich {
 ///
 /// These are normalized in the database.  Each distinct `RotPage` is assigned a
 /// uuid and shared across many possible collections that reference it.
-#[derive(Clone, Debug, Ord, Eq, PartialOrd, PartialEq)]
+#[derive(
+    Clone, Debug, Ord, Eq, PartialOrd, PartialEq, Deserialize, Serialize,
+)]
 pub struct RotPage {
     pub data_base64: String,
 }
 
 /// Indicates that a particular `RotPage` was found (at a particular time from a
 /// particular source, but these are only for debugging)
-#[derive(Clone, Debug, Ord, Eq, PartialOrd, PartialEq)]
+#[derive(
+    Clone, Debug, Ord, Eq, PartialOrd, PartialEq, Deserialize, Serialize,
+)]
 pub struct RotPageFound {
     pub time_collected: DateTime<Utc>,
     pub source: String,
@@ -266,7 +298,18 @@ pub struct RotPageFound {
 }
 
 /// Describes which root of trust page this is
-#[derive(Clone, Copy, Debug, EnumIter, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    EnumIter,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Deserialize,
+    Serialize,
+)]
 pub enum RotPageWhich {
     Cmpa,
     CfpaActive,
@@ -307,7 +350,7 @@ impl IntoRotPage for gateway_client::types::RotCfpa {
 /// A sled may be on a PC (in dev/test environments) and have no associated
 /// baseboard.  There might also be baseboards with no associated sled (if
 /// they have not been formally added to the control plane).
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
 pub struct SledAgent {
     pub time_collected: DateTime<Utc>,
     pub source: String,
@@ -320,7 +363,7 @@ pub struct SledAgent {
     pub reservoir_size: ByteCount,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
 pub struct OmicronZonesFound {
     pub time_collected: DateTime<Utc>,
     pub source: String,
diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml
index 0a607d5115..49bdc030c7 100644
--- a/test-utils/Cargo.toml
+++ b/test-utils/Cargo.toml
@@ -18,6 +18,7 @@ libc.workspace = true
 nexus-config.workspace = true
 omicron-common.workspace = true
 pem.workspace = true
+regex.workspace = true
 ring.workspace = true
 rustls.workspace = true
 slog.workspace = true
@@ -29,7 +30,6 @@ tokio = { workspace = true, features = ["full"] }
 tokio-postgres.workspace = true
 usdt.workspace = true
 rcgen.workspace = true
-regex.workspace = true
 reqwest.workspace = true
 walkdir.workspace = true
 omicron-workspace-hack.workspace = true
diff --git a/test-utils/src/dev/test_cmds.rs b/test-utils/src/dev/test_cmds.rs
index 15c94554c8..89bf022369 100644
--- a/test-utils/src/dev/test_cmds.rs
+++ b/test-utils/src/dev/test_cmds.rs
@@ -120,3 +120,73 @@ pub fn temp_file_path(label: &str) -> PathBuf {
 pub fn error_for_enoent() -> String {
     io::Error::from_raw_os_error(libc::ENOENT).to_string()
 }
+
+/// Redacts text from a string (usually stdout/stderr) that may change from
+/// invocation to invocation (e.g., assigned TCP port numbers, timestamps)
+///
+/// This allows use to use expectorate to verify the shape of the CLI output.
+pub fn redact_variable(input: &str, extra_redactions: &[&str]) -> String {
+    // Replace TCP port numbers.  We include the localhost characters to avoid
+    // catching any random sequence of numbers.
+    let s = regex::Regex::new(r"\[::1\]:\d{4,5}")
+        .unwrap()
+        .replace_all(input, "[::1]:REDACTED_PORT")
+        .to_string();
+    let s = regex::Regex::new(r"\[::ffff:127.0.0.1\]:\d{4,5}")
+        .unwrap()
+        .replace_all(&s, "[::ffff:127.0.0.1]:REDACTED_PORT")
+        .to_string();
+    let s = regex::Regex::new(r"127\.0\.0\.1:\d{4,5}")
+        .unwrap()
+        .replace_all(&s, "127.0.0.1:REDACTED_PORT")
+        .to_string();
+
+    // Replace uuids.
+    let s = regex::Regex::new(
+        "[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-\
+        [a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}",
+    )
+    .unwrap()
+    .replace_all(&s, "REDACTED_UUID_REDACTED_UUID_REDACTED")
+    .to_string();
+
+    // Replace timestamps.
+    let s = regex::Regex::new(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z")
+        .unwrap()
+        .replace_all(&s, "<REDACTED_TIMESTAMP>")
+        .to_string();
+
+    let s = regex::Regex::new(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z")
+        .unwrap()
+        .replace_all(&s, "<REDACTED     TIMESTAMP>")
+        .to_string();
+
+    // Replace formatted durations.  These are pretty specific to the background
+    // task output.
+    let s = regex::Regex::new(r"\d+s ago")
+        .unwrap()
+        .replace_all(&s, "<REDACTED DURATION>s ago")
+        .to_string();
+
+    let s = regex::Regex::new(r"\d+ms")
+        .unwrap()
+        .replace_all(&s, "<REDACTED DURATION>ms")
+        .to_string();
+
+    let mut s = regex::Regex::new(
+        r"note: database schema version matches expected \(\d+\.\d+\.\d+\)",
+    )
+    .unwrap()
+    .replace_all(
+        &s,
+        "note: database schema version matches expected \
+        (<redacted database version>)",
+    )
+    .to_string();
+
+    for r in extra_redactions {
+        s = s.replace(r, "<REDACTED>");
+    }
+
+    s
+}

From 358eec408b1df996ccf1ea115ea1770de80637a0 Mon Sep 17 00:00:00 2001
From: David Crespo <david-crespo@users.noreply.github.com>
Date: Thu, 7 Mar 2024 16:00:23 -0600
Subject: [PATCH 090/157] Unquarantine clickhouse binary on mac (#5223)

After #5127 (I assume, because this problem came up immediately after
that) macOS Gatekeeper is putting the `clickhouse` binary in quarantine
after download, which means we can't execute it unless we take it out of
quarantine. This is apparently a new thing, as [this
doc](https://github.com/ClickHouse/clickhouse-docs/blob/08d7a329d/knowledgebase/fix-developer-verification-error-in-macos.md)
about how to do that was only [added Jan
9](https://github.com/ClickHouse/clickhouse-docs/pull/1835).

Here I am doing it in the simplest way possible: if mac, then remove
quarantine attr. If anyone has a better way, I'm all ears.
---
 tools/ci_download_clickhouse | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tools/ci_download_clickhouse b/tools/ci_download_clickhouse
index 93201a772b..c5f1cae914 100755
--- a/tools/ci_download_clickhouse
+++ b/tools/ci_download_clickhouse
@@ -84,6 +84,12 @@ function main
 	# Unpack the tarball into a local directory
 	do_untar "$TARBALL_FILE" "$DEST_DIR"
 
+	# on macOS, we need to take the binary out of quarantine after download
+	# https://github.com/ClickHouse/clickhouse-docs/blob/08d7a329d/knowledgebase/fix-developer-verification-error-in-macos.md
+	if [[ $CIDL_OS == darwin* ]]; then
+		xattr -d com.apple.quarantine "$DEST_DIR/clickhouse"
+	fi
+
 	# Run the binary as a sanity-check.
 	"$DEST_DIR/clickhouse" server --version
 }

From 19ad11128b40fccfe4a19fc22171ef388f94781d Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Thu, 7 Mar 2024 17:45:53 -0500
Subject: [PATCH 091/157] `omdb nexus blueprints target set`: take an `enabled`
 setting (#5224)

I made this a required argument because I'm nervous to default it to
`Enabled` when we've talked so much about manually setting a disabled
blueprint. If folks feel strongly this should be an optional
`--enabled={true,false,inherit}` defaulting to `true` instead, I
wouldn't put up a fight.

Addresses the first part of #5210.
---
 dev-tools/omdb/src/bin/omdb/nexus.rs | 49 +++++++++++++++++++++++-----
 1 file changed, 40 insertions(+), 9 deletions(-)

diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs
index 692931db51..4705da2a32 100644
--- a/dev-tools/omdb/src/bin/omdb/nexus.rs
+++ b/dev-tools/omdb/src/bin/omdb/nexus.rs
@@ -11,6 +11,7 @@ use chrono::SecondsFormat;
 use chrono::Utc;
 use clap::Args;
 use clap::Subcommand;
+use clap::ValueEnum;
 use futures::TryStreamExt;
 use nexus_client::types::ActivationReason;
 use nexus_client::types::BackgroundTask;
@@ -77,7 +78,7 @@ enum BlueprintsCommands {
     Diff(BlueprintIdsArgs),
     /// Delete a blueprint
     Delete(BlueprintIdArgs),
-    /// Set the current target blueprint
+    /// Interact with the current target blueprint
     Target(BlueprintsTargetArgs),
     /// Generate an initial blueprint from a specific inventory collection
     GenerateFromCollection(CollectionIdArgs),
@@ -116,7 +117,25 @@ enum BlueprintTargetCommands {
     /// Show the current target blueprint
     Show,
     /// Change the current target blueprint
-    Set(BlueprintIdArgs),
+    Set(BlueprintTargetSetArgs),
+}
+
+#[derive(Debug, Args)]
+struct BlueprintTargetSetArgs {
+    /// id of blueprint to make target
+    blueprint_id: Uuid,
+    /// whether this blueprint should be enabled
+    enabled: BlueprintTargetSetEnabled,
+}
+
+#[derive(Debug, Clone, Copy, ValueEnum)]
+enum BlueprintTargetSetEnabled {
+    /// set the new current target as enabled
+    Enabled,
+    /// set the new current target as disabled
+    Disabled,
+    /// use the enabled setting from the parent blueprint
+    Inherit,
 }
 
 #[derive(Debug, Args)]
@@ -914,14 +933,26 @@ async fn cmd_nexus_blueprints_target_show(
 
 async fn cmd_nexus_blueprints_target_set(
     client: &nexus_client::Client,
-    args: &BlueprintIdArgs,
+    args: &BlueprintTargetSetArgs,
 ) -> Result<(), anyhow::Error> {
-    // Try to preserve the value of "enabled", if possible.
-    let enabled = client
-        .blueprint_target_view()
-        .await
-        .map(|current| current.into_inner().enabled)
-        .unwrap_or(true);
+    let enabled = match args.enabled {
+        BlueprintTargetSetEnabled::Enabled => true,
+        BlueprintTargetSetEnabled::Disabled => false,
+        // There's a small TOCTOU race with "inherit": What if the user wants to
+        // inherit the parent blueprint enabled bit but the current target
+        // blueprint enabled bit is flipped or the current target blueprint is
+        // changed? We expect neither of these to be problematic in practice:
+        // the only way for the `enable` bit to be set to anything at all is via
+        // `omdb`, so the user would have to be racing with another `omdb`
+        // operator. (In the case of the current target blueprint being changed
+        // entirely, that will result in a failure to set the current target
+        // below, because its parent will no longer be the current target.)
+        BlueprintTargetSetEnabled::Inherit => client
+            .blueprint_target_view()
+            .await
+            .map(|current| current.into_inner().enabled)
+            .context("failed to fetch current target blueprint")?,
+    };
     client
         .blueprint_target_set(&nexus_client::types::BlueprintTargetSet {
             target_id: args.blueprint_id,

From 87c9b13037e8e9d9ec12abc6d997336674993760 Mon Sep 17 00:00:00 2001
From: "Andrew J. Stone" <andrew@oxidecomputer.com>
Date: Thu, 7 Mar 2024 17:57:24 -0500
Subject: [PATCH 092/157] Setup the VMM reservoir in a background task (#5124)

Fixes the most immediate problem of #5121
---
 Cargo.lock                                    |   1 +
 clients/nexus-client/src/lib.rs               |   5 +
 nexus/db-model/src/schema.rs                  |   3 +-
 nexus/db-model/src/sled.rs                    |  49 +-
 nexus/db-queries/src/db/datastore/dataset.rs  |   8 +-
 nexus/db-queries/src/db/datastore/mod.rs      |  11 +-
 .../src/db/datastore/physical_disk.rs         |   3 +-
 nexus/db-queries/src/db/datastore/rack.rs     |   4 +-
 nexus/db-queries/src/db/datastore/sled.rs     | 178 ++--
 .../reconfigurator/execution/src/datasets.rs  |   8 +-
 .../src/app/background/blueprint_execution.rs |   4 +-
 .../app/background/inventory_collection.rs    |   4 +-
 nexus/src/app/sled.rs                         |  23 +-
 nexus/src/internal_api/http_entrypoints.rs    |  26 +-
 nexus/tests/integration_tests/rack.rs         |   7 +-
 nexus/types/src/internal_api/params.rs        |  14 +-
 openapi/nexus-internal.json                   |  53 +-
 schema/crdb/39.0.0/up.sql                     |   4 +
 schema/crdb/dbinit.sql                        |   5 +-
 sled-agent/Cargo.toml                         |   1 +
 sled-agent/src/fakes/nexus.rs                 |  61 +-
 sled-agent/src/hardware_monitor.rs            |  12 +-
 sled-agent/src/instance_manager.rs            | 113 +--
 sled-agent/src/lib.rs                         |   1 +
 sled-agent/src/nexus.rs                       | 894 +++++++++++++++++-
 sled-agent/src/sim/server.rs                  |   5 +-
 sled-agent/src/sled_agent.rs                  | 184 +---
 sled-agent/src/vmm_reservoir.rs               | 272 ++++++
 28 files changed, 1525 insertions(+), 428 deletions(-)
 create mode 100644 schema/crdb/39.0.0/up.sql
 create mode 100644 sled-agent/src/vmm_reservoir.rs

diff --git a/Cargo.lock b/Cargo.lock
index fd9155bb8e..416766b9cb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5297,6 +5297,7 @@ dependencies = [
  "dropshot",
  "expectorate",
  "flate2",
+ "flume",
  "futures",
  "gateway-client",
  "glob",
diff --git a/clients/nexus-client/src/lib.rs b/clients/nexus-client/src/lib.rs
index 17fb5aa367..55bdf3d0aa 100644
--- a/clients/nexus-client/src/lib.rs
+++ b/clients/nexus-client/src/lib.rs
@@ -33,6 +33,11 @@ progenitor::generate_api!(
         MacAddr = omicron_common::api::external::MacAddr,
         Name = omicron_common::api::external::Name,
         NewPasswordHash = omicron_passwords::NewPasswordHash,
+    },
+    patch = {
+        SledAgentInfo = { derives = [PartialEq, Eq] },
+        ByteCount = { derives = [PartialEq, Eq] },
+        Baseboard = { derives = [PartialEq, Eq] }
     }
 );
 
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index 55d3e9b43f..d70b0a267e 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(38, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(39, 0, 0);
 
 table! {
     disk (id) {
@@ -827,6 +827,7 @@ table! {
         last_used_address -> Inet,
         sled_policy -> crate::sled_policy::SledPolicyEnum,
         sled_state -> crate::SledStateEnum,
+        sled_agent_gen -> Int8,
     }
 }
 
diff --git a/nexus/db-model/src/sled.rs b/nexus/db-model/src/sled.rs
index 47912f89cc..c2cbb65bd6 100644
--- a/nexus/db-model/src/sled.rs
+++ b/nexus/db-model/src/sled.rs
@@ -6,10 +6,15 @@ use super::{ByteCount, Generation, SledState, SqlU16, SqlU32};
 use crate::collection::DatastoreCollectionConfig;
 use crate::ipv6;
 use crate::schema::{physical_disk, service, sled, zpool};
+use crate::sled::shared::Baseboard;
 use crate::sled_policy::DbSledPolicy;
 use chrono::{DateTime, Utc};
 use db_macros::Asset;
-use nexus_types::{external_api::shared, external_api::views, identity::Asset};
+use nexus_types::{
+    external_api::{shared, views},
+    identity::Asset,
+    internal_api::params,
+};
 use std::net::Ipv6Addr;
 use std::net::SocketAddrV6;
 use uuid::Uuid;
@@ -41,7 +46,7 @@ pub struct Sled {
     #[diesel(embed)]
     identity: SledIdentity,
     time_deleted: Option<DateTime<Utc>>,
-    rcgen: Generation,
+    pub rcgen: Generation,
 
     pub rack_id: Uuid,
 
@@ -66,6 +71,12 @@ pub struct Sled {
 
     #[diesel(column_name = sled_state)]
     state: SledState,
+
+    /// A generation number owned and incremented by sled-agent
+    ///
+    /// This is specifically distinct from `rcgen`, which is incremented by
+    /// child resources as part of `DatastoreCollectionConfig`.
+    pub sled_agent_gen: Generation,
 }
 
 impl Sled {
@@ -119,6 +130,34 @@ impl From<Sled> for views::Sled {
     }
 }
 
+impl From<Sled> for params::SledAgentInfo {
+    fn from(sled: Sled) -> Self {
+        let role = if sled.is_scrimlet {
+            params::SledRole::Scrimlet
+        } else {
+            params::SledRole::Gimlet
+        };
+        let decommissioned = match sled.state {
+            SledState::Active => false,
+            SledState::Decommissioned => true,
+        };
+        Self {
+            sa_address: sled.address(),
+            role,
+            baseboard: Baseboard {
+                serial: sled.serial_number.clone(),
+                part: sled.part_number.clone(),
+                revision: sled.revision,
+            },
+            usable_hardware_threads: sled.usable_hardware_threads.into(),
+            usable_physical_ram: sled.usable_physical_ram.into(),
+            reservoir_size: sled.reservoir_size.into(),
+            generation: sled.sled_agent_gen.into(),
+            decommissioned,
+        }
+    }
+}
+
 impl DatastoreCollectionConfig<super::PhysicalDisk> for Sled {
     type CollectionId = Uuid;
     type GenerationNumberColumn = sled::dsl::rcgen;
@@ -161,6 +200,9 @@ pub struct SledUpdate {
     // ServiceAddress (Sled Agent).
     pub ip: ipv6::Ipv6Addr,
     pub port: SqlU16,
+
+    // Generation number - owned and incremented by sled-agent.
+    pub sled_agent_gen: Generation,
 }
 
 impl SledUpdate {
@@ -170,6 +212,7 @@ impl SledUpdate {
         baseboard: SledBaseboard,
         hardware: SledSystemHardware,
         rack_id: Uuid,
+        sled_agent_gen: Generation,
     ) -> Self {
         Self {
             id,
@@ -185,6 +228,7 @@ impl SledUpdate {
             reservoir_size: hardware.reservoir_size,
             ip: addr.ip().into(),
             port: addr.port().into(),
+            sled_agent_gen,
         }
     }
 
@@ -220,6 +264,7 @@ impl SledUpdate {
             ip: self.ip,
             port: self.port,
             last_used_address,
+            sled_agent_gen: self.sled_agent_gen,
         }
     }
 
diff --git a/nexus/db-queries/src/db/datastore/dataset.rs b/nexus/db-queries/src/db/datastore/dataset.rs
index 3c1fd0afb1..792f8f81a4 100644
--- a/nexus/db-queries/src/db/datastore/dataset.rs
+++ b/nexus/db-queries/src/db/datastore/dataset.rs
@@ -186,6 +186,7 @@ impl DataStore {
 mod test {
     use super::*;
     use crate::db::datastore::test_utils::datastore_test;
+    use nexus_db_model::Generation;
     use nexus_db_model::SledBaseboard;
     use nexus_db_model::SledSystemHardware;
     use nexus_db_model::SledUpdate;
@@ -222,12 +223,9 @@ mod test {
                 reservoir_size: (16 << 30).try_into().unwrap(),
             },
             Uuid::new_v4(),
+            Generation::new(),
         );
-        datastore
-            .sled_upsert(sled)
-            .await
-            .expect("failed to upsert sled")
-            .unwrap();
+        datastore.sled_upsert(sled).await.expect("failed to upsert sled");
 
         // Create a fake zpool that backs our fake datasets.
         let zpool_id = Uuid::new_v4();
diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs
index dea6fcb997..b164186fdf 100644
--- a/nexus/db-queries/src/db/datastore/mod.rs
+++ b/nexus/db-queries/src/db/datastore/mod.rs
@@ -108,7 +108,6 @@ pub use instance::InstanceAndActiveVmm;
 pub use inventory::DataStoreInventoryTest;
 pub use rack::RackInit;
 pub use silo::Discoverability;
-pub use sled::SledUpsertOutput;
 pub use switch_port::SwitchPortSettingsCombinedResult;
 pub use virtual_provisioning_collection::StorageType;
 pub use volume::read_only_resources_associated_with_volume;
@@ -393,6 +392,7 @@ mod test {
     use futures::stream;
     use futures::StreamExt;
     use nexus_config::RegionAllocationStrategy;
+    use nexus_db_model::Generation;
     use nexus_db_model::IpAttachState;
     use nexus_test_utils::db::test_setup_database;
     use nexus_types::external_api::params;
@@ -619,8 +619,9 @@ mod test {
             sled_baseboard_for_test(),
             sled_system_hardware_for_test(),
             rack_id,
+            Generation::new(),
         );
-        datastore.sled_upsert(sled_update).await.unwrap().unwrap();
+        datastore.sled_upsert(sled_update).await.unwrap();
         sled_id
     }
 
@@ -1338,8 +1339,9 @@ mod test {
             sled_baseboard_for_test(),
             sled_system_hardware_for_test(),
             rack_id,
+            Generation::new(),
         );
-        datastore.sled_upsert(sled1).await.unwrap().unwrap();
+        datastore.sled_upsert(sled1).await.unwrap();
 
         let addr2 = "[fd00:1df::1]:12345".parse().unwrap();
         let sled2_id = "66285c18-0c79-43e0-e54f-95271f271314".parse().unwrap();
@@ -1349,8 +1351,9 @@ mod test {
             sled_baseboard_for_test(),
             sled_system_hardware_for_test(),
             rack_id,
+            Generation::new(),
         );
-        datastore.sled_upsert(sled2).await.unwrap().unwrap();
+        datastore.sled_upsert(sled2).await.unwrap();
 
         let ip = datastore.next_ipv6_address(&opctx, sled1_id).await.unwrap();
         let expected_ip = Ipv6Addr::new(0xfd00, 0x1de, 0, 0, 0, 0, 1, 0);
diff --git a/nexus/db-queries/src/db/datastore/physical_disk.rs b/nexus/db-queries/src/db/datastore/physical_disk.rs
index d4e94745aa..81fc14d1d7 100644
--- a/nexus/db-queries/src/db/datastore/physical_disk.rs
+++ b/nexus/db-queries/src/db/datastore/physical_disk.rs
@@ -143,6 +143,7 @@ mod test {
     use crate::db::datastore::test_utils::datastore_test;
     use crate::db::model::{PhysicalDiskKind, Sled, SledUpdate};
     use dropshot::PaginationOrder;
+    use nexus_db_model::Generation;
     use nexus_test_utils::db::test_setup_database;
     use nexus_types::identity::Asset;
     use omicron_test_utils::dev;
@@ -159,11 +160,11 @@ mod test {
             sled_baseboard_for_test(),
             sled_system_hardware_for_test(),
             rack_id,
+            Generation::new(),
         );
         db.sled_upsert(sled_update)
             .await
             .expect("Could not upsert sled during test prep")
-            .unwrap()
     }
 
     fn list_disk_params() -> DataPageParams<'static, Uuid> {
diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs
index 32e059bf81..3224f36b8d 100644
--- a/nexus/db-queries/src/db/datastore/rack.rs
+++ b/nexus/db-queries/src/db/datastore/rack.rs
@@ -868,7 +868,7 @@ mod test {
     use async_bb8_diesel::AsyncSimpleConnection;
     use internal_params::DnsRecord;
     use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
-    use nexus_db_model::{DnsGroup, InitialDnsGroup, SledUpdate};
+    use nexus_db_model::{DnsGroup, Generation, InitialDnsGroup, SledUpdate};
     use nexus_test_utils::db::test_setup_database;
     use nexus_types::external_api::shared::SiloIdentityMode;
     use nexus_types::identity::Asset;
@@ -1065,11 +1065,11 @@ mod test {
             sled_baseboard_for_test(),
             sled_system_hardware_for_test(),
             rack_id(),
+            Generation::new(),
         );
         db.sled_upsert(sled_update)
             .await
             .expect("Could not upsert sled during test prep")
-            .unwrap()
     }
 
     // Hacky macro helper to:
diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs
index 8d812f69cc..93d3d0e6a2 100644
--- a/nexus/db-queries/src/db/datastore/sled.rs
+++ b/nexus/db-queries/src/db/datastore/sled.rs
@@ -41,21 +41,17 @@ use uuid::Uuid;
 impl DataStore {
     /// Stores a new sled in the database.
     ///
-    /// Produces `SledUpsertOutput::Decommissioned` if the sled is
-    /// decommissioned. This is not an error, because `sled_upsert`'s only
-    /// caller (sled-agent) is not expected to receive this error.
+    /// Returns an error if `sled_agent_gen` is stale, or the sled is
+    /// decommissioned.
     pub async fn sled_upsert(
         &self,
         sled_update: SledUpdate,
-    ) -> CreateResult<SledUpsertOutput> {
+    ) -> CreateResult<Sled> {
         use db::schema::sled::dsl;
         // required for conditional upsert
         use diesel::query_dsl::methods::FilterDsl;
 
-        // TODO: figure out what to do with time_deleted. We want to replace it
-        // with a time_decommissioned, most probably.
-
-        let query = diesel::insert_into(dsl::sled)
+        diesel::insert_into(dsl::sled)
             .values(sled_update.clone().into_insertable())
             .on_conflict(dsl::id)
             .do_update()
@@ -69,14 +65,13 @@ impl DataStore {
                     .eq(sled_update.usable_hardware_threads),
                 dsl::usable_physical_ram.eq(sled_update.usable_physical_ram),
                 dsl::reservoir_size.eq(sled_update.reservoir_size),
+                dsl::sled_agent_gen.eq(sled_update.sled_agent_gen),
             ))
+            .filter(dsl::sled_agent_gen.lt(sled_update.sled_agent_gen))
             .filter(dsl::sled_state.ne(SledState::Decommissioned))
-            .returning(Sled::as_returning());
-
-        let sled: Option<Sled> = query
+            .returning(Sled::as_returning())
             .get_result_async(&*self.pool_connection_unauthorized().await?)
             .await
-            .optional()
             .map_err(|e| {
                 public_error_from_diesel(
                     e,
@@ -85,19 +80,7 @@ impl DataStore {
                         &sled_update.id().to_string(),
                     ),
                 )
-            })?;
-
-        // The only situation in which a sled is not returned is if the
-        // `.filter(dsl::sled_state.ne(SledState::Decommissioned))` is not
-        // satisfied.
-        //
-        // If we want to return a sled even if it's decommissioned here, we may
-        // have to do something more complex. See
-        // https://stackoverflow.com/q/34708509.
-        match sled {
-            Some(sled) => Ok(SledUpsertOutput::Updated(sled)),
-            None => Ok(SledUpsertOutput::Decommissioned),
-        }
+            })
     }
 
     pub async fn sled_list(
@@ -532,28 +515,6 @@ impl DataStore {
     }
 }
 
-/// The result of [`DataStore::sled_upsert`].
-#[derive(Clone, Debug)]
-#[must_use]
-pub enum SledUpsertOutput {
-    /// The sled was updated.
-    Updated(Sled),
-    /// The sled was not updated because it is decommissioned.
-    Decommissioned,
-}
-
-impl SledUpsertOutput {
-    /// Returns the sled if it was updated, or panics if it was not.
-    pub fn unwrap(self) -> Sled {
-        match self {
-            SledUpsertOutput::Updated(sled) => sled,
-            SledUpsertOutput::Decommissioned => {
-                panic!("sled was decommissioned, not updated")
-            }
-        }
-    }
-}
-
 // ---
 // State transition validators
 // ---
@@ -713,6 +674,7 @@ mod test {
     use crate::db::model::SqlU32;
     use anyhow::{Context, Result};
     use itertools::Itertools;
+    use nexus_db_model::Generation;
     use nexus_test_utils::db::test_setup_database;
     use nexus_types::identity::Asset;
     use omicron_common::api::external;
@@ -732,7 +694,7 @@ mod test {
 
         let mut sled_update = test_new_sled_update();
         let observed_sled =
-            datastore.sled_upsert(sled_update.clone()).await.unwrap().unwrap();
+            datastore.sled_upsert(sled_update.clone()).await.unwrap();
         assert_eq!(
             observed_sled.usable_hardware_threads,
             sled_update.usable_hardware_threads
@@ -760,12 +722,14 @@ mod test {
             .unwrap(),
         );
 
+        // Bump the generation number so the insert succeeds.
+        sled_update.sled_agent_gen.0 = sled_update.sled_agent_gen.0.next();
+
         // Test that upserting the sled propagates those changes to the DB.
         let observed_sled = datastore
             .sled_upsert(sled_update.clone())
             .await
-            .expect("Could not upsert sled during test prep")
-            .unwrap();
+            .expect("Could not upsert sled during test prep");
         assert_eq!(
             observed_sled.usable_hardware_threads,
             sled_update.usable_hardware_threads
@@ -780,6 +744,70 @@ mod test {
         logctx.cleanup_successful();
     }
 
+    #[tokio::test]
+    async fn upsert_sled_updates_fails_with_stale_sled_agent_gen() {
+        let logctx = dev::test_setup_log(
+            "upsert_sled_updates_fails_with_stale_sled_agent_gen",
+        );
+        let mut db = test_setup_database(&logctx.log).await;
+        let (_opctx, datastore) = datastore_test(&logctx, &db).await;
+
+        let mut sled_update = test_new_sled_update();
+        let observed_sled =
+            datastore.sled_upsert(sled_update.clone()).await.unwrap();
+
+        assert_eq!(observed_sled.reservoir_size, sled_update.reservoir_size);
+
+        // Modify the reservoir size
+        const MIB: u64 = 1024 * 1024;
+
+        sled_update.reservoir_size = ByteCount::from(
+            external::ByteCount::try_from(
+                sled_update.reservoir_size.0.to_bytes() + MIB,
+            )
+            .unwrap(),
+        );
+
+        // Fail the update, since the generation number didn't change.
+        assert!(datastore.sled_upsert(sled_update.clone()).await.is_err());
+
+        // Bump the generation number so the next insert succeeds.
+        sled_update.sled_agent_gen.0 = sled_update.sled_agent_gen.0.next();
+
+        // Test that upserting the sled propagates those changes to the DB.
+        let observed_sled = datastore
+            .sled_upsert(sled_update.clone())
+            .await
+            .expect("Could not upsert sled during test prep");
+        assert_eq!(observed_sled.reservoir_size, sled_update.reservoir_size);
+
+        // Now reset the generation to a lower value and try again.
+        // This should fail.
+        let current_gen = sled_update.sled_agent_gen;
+        sled_update.sled_agent_gen = Generation::new();
+        assert!(datastore.sled_upsert(sled_update.clone()).await.is_err());
+
+        // Now bump the generation from the saved `current_gen`
+        // Change the reservoir value again. This should succeed.
+        sled_update.reservoir_size = ByteCount::from(
+            external::ByteCount::try_from(
+                sled_update.reservoir_size.0.to_bytes() + MIB,
+            )
+            .unwrap(),
+        );
+        sled_update.sled_agent_gen.0 = current_gen.0.next();
+        // Test that upserting the sled propagates those changes to the DB.
+        let observed_sled = datastore
+            .sled_upsert(sled_update.clone())
+            .await
+            .expect("Could not upsert sled during test prep");
+        assert_eq!(observed_sled.reservoir_size, sled_update.reservoir_size);
+        assert_eq!(observed_sled.sled_agent_gen, sled_update.sled_agent_gen);
+
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
+
     #[tokio::test]
     async fn upsert_sled_doesnt_update_decommissioned() {
         let logctx =
@@ -789,7 +817,7 @@ mod test {
 
         let mut sled_update = test_new_sled_update();
         let observed_sled =
-            datastore.sled_upsert(sled_update.clone()).await.unwrap().unwrap();
+            datastore.sled_upsert(sled_update.clone()).await.unwrap();
         assert_eq!(
             observed_sled.usable_hardware_threads,
             sled_update.usable_hardware_threads
@@ -830,15 +858,8 @@ mod test {
             .unwrap(),
         );
 
-        // Upserting the sled should produce the `Decommisioned` variant.
-        let sled = datastore
-            .sled_upsert(sled_update.clone())
-            .await
-            .expect("updating a decommissioned sled should succeed");
-        assert!(
-            matches!(sled, SledUpsertOutput::Decommissioned),
-            "sled should be decommissioned"
-        );
+        // Upserting the sled should produce an error, because it is decommissioend.
+        assert!(datastore.sled_upsert(sled_update.clone()).await.is_err());
 
         // The sled should not have been updated.
         let (_, observed_sled_2) = LookupPath::new(&opctx, &datastore)
@@ -874,26 +895,14 @@ mod test {
         let (opctx, datastore) = datastore_test(&logctx, &db).await;
 
         // Define some sleds that resources cannot be provisioned on.
-        let non_provisionable_sled = datastore
-            .sled_upsert(test_new_sled_update())
-            .await
-            .unwrap()
-            .unwrap();
-        let expunged_sled = datastore
-            .sled_upsert(test_new_sled_update())
-            .await
-            .unwrap()
-            .unwrap();
-        let decommissioned_sled = datastore
-            .sled_upsert(test_new_sled_update())
-            .await
-            .unwrap()
-            .unwrap();
-        let illegal_decommissioned_sled = datastore
-            .sled_upsert(test_new_sled_update())
-            .await
-            .unwrap()
-            .unwrap();
+        let non_provisionable_sled =
+            datastore.sled_upsert(test_new_sled_update()).await.unwrap();
+        let expunged_sled =
+            datastore.sled_upsert(test_new_sled_update()).await.unwrap();
+        let decommissioned_sled =
+            datastore.sled_upsert(test_new_sled_update()).await.unwrap();
+        let illegal_decommissioned_sled =
+            datastore.sled_upsert(test_new_sled_update()).await.unwrap();
 
         let ineligible_sleds = IneligibleSleds {
             non_provisionable: non_provisionable_sled.id(),
@@ -926,7 +935,7 @@ mod test {
         // Now add a provisionable sled and try again.
         let sled_update = test_new_sled_update();
         let provisionable_sled =
-            datastore.sled_upsert(sled_update.clone()).await.unwrap().unwrap();
+            datastore.sled_upsert(sled_update.clone()).await.unwrap();
 
         // Try a few times to ensure that resources never get allocated to the
         // non-provisionable sled.
@@ -1048,11 +1057,7 @@ mod test {
             .enumerate();
 
         // Set up a sled to test against.
-        let sled = datastore
-            .sled_upsert(test_new_sled_update())
-            .await
-            .unwrap()
-            .unwrap();
+        let sled = datastore.sled_upsert(test_new_sled_update()).await.unwrap();
         let sled_id = sled.id();
 
         for (i, ((policy, state), after)) in all_transitions {
@@ -1173,6 +1178,7 @@ mod test {
             sled_baseboard_for_test(),
             sled_system_hardware_for_test(),
             rack_id(),
+            Generation::new(),
         )
     }
 
diff --git a/nexus/reconfigurator/execution/src/datasets.rs b/nexus/reconfigurator/execution/src/datasets.rs
index 97d8324fdb..5e677b98be 100644
--- a/nexus/reconfigurator/execution/src/datasets.rs
+++ b/nexus/reconfigurator/execution/src/datasets.rs
@@ -138,6 +138,7 @@ pub(crate) async fn ensure_crucible_dataset_records_exist(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use nexus_db_model::Generation;
     use nexus_db_model::SledBaseboard;
     use nexus_db_model::SledSystemHardware;
     use nexus_db_model::SledUpdate;
@@ -184,12 +185,9 @@ mod tests {
                     reservoir_size: (16 << 30).try_into().unwrap(),
                 },
                 rack_id,
+                Generation::new(),
             );
-            datastore
-                .sled_upsert(sled)
-                .await
-                .expect("failed to upsert sled")
-                .unwrap();
+            datastore.sled_upsert(sled).await.expect("failed to upsert sled");
 
             for zone in &config.zones.zones {
                 let OmicronZoneType::Crucible { dataset, .. } = &zone.zone_type
diff --git a/nexus/src/app/background/blueprint_execution.rs b/nexus/src/app/background/blueprint_execution.rs
index 3c2530a3d3..ceb3cd7a05 100644
--- a/nexus/src/app/background/blueprint_execution.rs
+++ b/nexus/src/app/background/blueprint_execution.rs
@@ -191,12 +191,12 @@ mod test {
                     reservoir_size: ByteCount(999.into()),
                 },
                 rack_id,
+                nexus_db_model::Generation::new(),
             );
             datastore
                 .sled_upsert(update)
                 .await
-                .expect("Failed to insert sled to db")
-                .unwrap();
+                .expect("Failed to insert sled to db");
         }
 
         let (blueprint_tx, blueprint_rx) = watch::channel(None);
diff --git a/nexus/src/app/background/inventory_collection.rs b/nexus/src/app/background/inventory_collection.rs
index c0d64d554a..27f08ec738 100644
--- a/nexus/src/app/background/inventory_collection.rs
+++ b/nexus/src/app/background/inventory_collection.rs
@@ -194,6 +194,7 @@ mod test {
     use crate::app::background::common::BackgroundTask;
     use crate::app::background::inventory_collection::DbSledAgentEnumerator;
     use crate::app::background::inventory_collection::InventoryCollector;
+    use nexus_db_model::Generation;
     use nexus_db_model::SledBaseboard;
     use nexus_db_model::SledSystemHardware;
     use nexus_db_model::SledUpdate;
@@ -337,8 +338,9 @@ mod test {
                     reservoir_size: ByteCount::from_gibibytes_u32(8).into(),
                 },
                 rack_id,
+                Generation::new(),
             );
-            sleds.push(datastore.sled_upsert(sled).await.unwrap().unwrap());
+            sleds.push(datastore.sled_upsert(sled).await.unwrap());
         }
 
         // The same enumerator should immediately find all the new sleds.
diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs
index a9341472e0..88f70c7d0d 100644
--- a/nexus/src/app/sled.rs
+++ b/nexus/src/app/sled.rs
@@ -5,13 +5,12 @@
 //! Sleds, and the hardware and services within them.
 
 use crate::internal_api::params::{
-    PhysicalDiskDeleteRequest, PhysicalDiskPutRequest, SledAgentStartupInfo,
-    SledRole, ZpoolPutRequest,
+    PhysicalDiskDeleteRequest, PhysicalDiskPutRequest, SledAgentInfo, SledRole,
+    ZpoolPutRequest,
 };
 use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db;
-use nexus_db_queries::db::datastore::SledUpsertOutput;
 use nexus_db_queries::db::lookup;
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_db_queries::db::model::DatasetKind;
@@ -45,7 +44,7 @@ impl super::Nexus {
         &self,
         _opctx: &OpContext,
         id: Uuid,
-        info: SledAgentStartupInfo,
+        info: SledAgentInfo,
     ) -> Result<(), Error> {
         info!(self.log, "registered sled agent"; "sled_uuid" => id.to_string());
 
@@ -69,21 +68,9 @@ impl super::Nexus {
                 reservoir_size: info.reservoir_size.into(),
             },
             self.rack_id,
+            info.generation.into(),
         );
-        match self.db_datastore.sled_upsert(sled).await? {
-            SledUpsertOutput::Updated(_) => {}
-            SledUpsertOutput::Decommissioned => {
-                // We currently don't bubble up errors for decommissioned sleds
-                // -- if it ever happens, a decommissioned sled-agent doesn't
-                // know about that.
-                warn!(
-                    self.log,
-                    "decommissioned sled-agent reached out for upserts";
-                    "sled_uuid" => id.to_string()
-                );
-            }
-        }
-
+        self.db_datastore.sled_upsert(sled).await?;
         Ok(())
     }
 
diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs
index 5702539a4b..e298935fee 100644
--- a/nexus/src/internal_api/http_entrypoints.rs
+++ b/nexus/src/internal_api/http_entrypoints.rs
@@ -8,7 +8,7 @@ use crate::ServerContext;
 
 use super::params::{
     OximeterInfo, PhysicalDiskDeleteRequest, PhysicalDiskPutRequest,
-    PhysicalDiskPutResponse, RackInitializationRequest, SledAgentStartupInfo,
+    PhysicalDiskPutResponse, RackInitializationRequest, SledAgentInfo,
     ZpoolPutRequest, ZpoolPutResponse,
 };
 use dropshot::endpoint;
@@ -58,6 +58,7 @@ type NexusApiDescription = ApiDescription<Arc<ServerContext>>;
 /// Returns a description of the internal nexus API
 pub(crate) fn internal_api() -> NexusApiDescription {
     fn register_endpoints(api: &mut NexusApiDescription) -> Result<(), String> {
+        api.register(sled_agent_get)?;
         api.register(sled_agent_put)?;
         api.register(sled_firewall_rules_request)?;
         api.register(switch_put)?;
@@ -109,6 +110,27 @@ struct SledAgentPathParam {
     sled_id: Uuid,
 }
 
+/// Return information about the given sled agent
+#[endpoint {
+     method = GET,
+     path = "/sled-agents/{sled_id}",
+ }]
+async fn sled_agent_get(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    path_params: Path<SledAgentPathParam>,
+) -> Result<HttpResponseOk<SledAgentInfo>, HttpError> {
+    let apictx = rqctx.context();
+    let nexus = &apictx.nexus;
+    let opctx = crate::context::op_context_for_internal_api(&rqctx).await;
+    let path = path_params.into_inner();
+    let sled_id = &path.sled_id;
+    let handler = async {
+        let (.., sled) = nexus.sled_lookup(&opctx, sled_id)?.fetch().await?;
+        Ok(HttpResponseOk(sled.into()))
+    };
+    apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
 /// Report that the sled agent for the specified sled has come online.
 #[endpoint {
      method = POST,
@@ -117,7 +139,7 @@ struct SledAgentPathParam {
 async fn sled_agent_put(
     rqctx: RequestContext<Arc<ServerContext>>,
     path_params: Path<SledAgentPathParam>,
-    sled_info: TypedBody<SledAgentStartupInfo>,
+    sled_info: TypedBody<SledAgentInfo>,
 ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
     let apictx = rqctx.context();
     let nexus = &apictx.nexus;
diff --git a/nexus/tests/integration_tests/rack.rs b/nexus/tests/integration_tests/rack.rs
index 2098ff660d..1148655195 100644
--- a/nexus/tests/integration_tests/rack.rs
+++ b/nexus/tests/integration_tests/rack.rs
@@ -13,9 +13,10 @@ use nexus_test_utils_macros::nexus_test;
 use nexus_types::external_api::params;
 use nexus_types::external_api::shared::UninitializedSled;
 use nexus_types::external_api::views::Rack;
-use nexus_types::internal_api::params::SledAgentStartupInfo;
+use nexus_types::internal_api::params::SledAgentInfo;
 use nexus_types::internal_api::params::SledRole;
 use omicron_common::api::external::ByteCount;
+use omicron_common::api::external::Generation;
 use omicron_nexus::TestInterfaces;
 use uuid::Uuid;
 
@@ -108,13 +109,15 @@ async fn test_sled_list_uninitialized(cptestctx: &ControlPlaneTestContext) {
     // Just pick some random fields other than `baseboard`
     let baseboard = uninitialized_sleds.pop().unwrap().baseboard;
     let sled_uuid = Uuid::new_v4();
-    let sa = SledAgentStartupInfo {
+    let sa = SledAgentInfo {
         sa_address: "[fd00:1122:3344:0100::1]:8080".parse().unwrap(),
         role: SledRole::Gimlet,
         baseboard,
         usable_hardware_threads: 32,
         usable_physical_ram: ByteCount::from_gibibytes_u32(100),
         reservoir_size: ByteCount::from_mebibytes_u32(100),
+        generation: Generation::new(),
+        decommissioned: false,
     };
     internal_client
         .make_request(
diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs
index 31ea86907a..e8c4703008 100644
--- a/nexus/types/src/internal_api/params.rs
+++ b/nexus/types/src/internal_api/params.rs
@@ -9,6 +9,7 @@ use crate::external_api::params::UserId;
 use crate::external_api::shared::Baseboard;
 use crate::external_api::shared::IpRange;
 use omicron_common::api::external::ByteCount;
+use omicron_common::api::external::Generation;
 use omicron_common::api::external::MacAddr;
 use omicron_common::api::external::Name;
 use omicron_common::api::internal::shared::ExternalPortDiscovery;
@@ -36,9 +37,9 @@ pub enum SledRole {
     Scrimlet,
 }
 
-/// Sent by a sled agent on startup to Nexus to request further instruction
+/// Sent by a sled agent to Nexus to inform about resources
 #[derive(Serialize, Deserialize, Debug, JsonSchema)]
-pub struct SledAgentStartupInfo {
+pub struct SledAgentInfo {
     /// The address of the sled agent's API endpoint
     pub sa_address: SocketAddrV6,
 
@@ -58,6 +59,15 @@ pub struct SledAgentStartupInfo {
     ///
     /// Must be smaller than "usable_physical_ram"
     pub reservoir_size: ByteCount,
+
+    /// The generation number of this request from sled-agent
+    pub generation: Generation,
+
+    /// Whether the sled-agent has been decommissioned by nexus
+    ///
+    /// This flag is only set to true by nexus. Setting it on an upsert from
+    /// sled-agent has no effect.
+    pub decommissioned: bool,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json
index 65c167f177..fb3ff976ae 100644
--- a/openapi/nexus-internal.json
+++ b/openapi/nexus-internal.json
@@ -798,6 +798,39 @@
       }
     },
     "/sled-agents/{sled_id}": {
+      "get": {
+        "summary": "Return information about the given sled agent",
+        "operationId": "sled_agent_get",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "sled_id",
+            "required": true,
+            "schema": {
+              "type": "string",
+              "format": "uuid"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/SledAgentInfo"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      },
       "post": {
         "summary": "Report that the sled agent for the specified sled has come online.",
         "operationId": "sled_agent_put",
@@ -816,7 +849,7 @@
           "content": {
             "application/json": {
               "schema": {
-                "$ref": "#/components/schemas/SledAgentStartupInfo"
+                "$ref": "#/components/schemas/SledAgentInfo"
               }
             }
           },
@@ -6394,8 +6427,8 @@
           "sled_id"
         ]
       },
-      "SledAgentStartupInfo": {
-        "description": "Sent by a sled agent on startup to Nexus to request further instruction",
+      "SledAgentInfo": {
+        "description": "Sent by a sled agent to Nexus to inform about resources",
         "type": "object",
         "properties": {
           "baseboard": {
@@ -6406,6 +6439,18 @@
               }
             ]
           },
+          "decommissioned": {
+            "description": "Whether the sled-agent has been decommissioned by nexus\n\nThis flag is only set to true by nexus. Setting it on an upsert from sled-agent has no effect.",
+            "type": "boolean"
+          },
+          "generation": {
+            "description": "The generation number of this request from sled-agent",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Generation"
+              }
+            ]
+          },
           "reservoir_size": {
             "description": "Amount of RAM dedicated to the VMM reservoir\n\nMust be smaller than \"usable_physical_ram\"",
             "allOf": [
@@ -6443,6 +6488,8 @@
         },
         "required": [
           "baseboard",
+          "decommissioned",
+          "generation",
           "reservoir_size",
           "role",
           "sa_address",
diff --git a/schema/crdb/39.0.0/up.sql b/schema/crdb/39.0.0/up.sql
new file mode 100644
index 0000000000..d11fabb297
--- /dev/null
+++ b/schema/crdb/39.0.0/up.sql
@@ -0,0 +1,4 @@
+-- Sled Agent upserts are now conditional on a generation number
+ALTER TABLE omicron.public.sled
+ADD COLUMN IF NOT EXISTS sled_agent_gen INT8 NOT NULL
+    DEFAULT 1;
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 40a6fd463f..79b6131d85 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -147,6 +147,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.sled (
     /* The actual state of the sled, updated exclusively by Nexus */
     sled_state omicron.public.sled_state NOT NULL,
 
+    /* Generation number owned and incremented by the sled-agent */
+    sled_agent_gen INT8 NOT NULL DEFAULT 1,
+
     -- This constraint should be upheld, even for deleted disks
     -- in the fleet.
     CONSTRAINT serial_part_revision_unique UNIQUE (
@@ -3552,7 +3555,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '38.0.0', NULL)
+    ( TRUE, NOW(), NOW(), '39.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;
diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml
index d61f2d09e4..7f46a277ce 100644
--- a/sled-agent/Cargo.toml
+++ b/sled-agent/Cargo.toml
@@ -28,6 +28,7 @@ dpd-client.workspace = true
 display-error-chain.workspace = true
 dropshot.workspace = true
 flate2.workspace = true
+flume.workspace = true
 futures.workspace = true
 glob.workspace = true
 hex.workspace = true
diff --git a/sled-agent/src/fakes/nexus.rs b/sled-agent/src/fakes/nexus.rs
index 5ba4b6c501..5920d40f6c 100644
--- a/sled-agent/src/fakes/nexus.rs
+++ b/sled-agent/src/fakes/nexus.rs
@@ -8,13 +8,17 @@
 //! to operate correctly.
 
 use dropshot::{
-    endpoint, ApiDescription, FreeformBody, HttpError, HttpResponseOk, Path,
-    RequestContext,
+    endpoint, ApiDescription, FreeformBody, HttpError, HttpResponseOk,
+    HttpResponseUpdatedNoContent, Path, RequestContext, TypedBody,
 };
 use hyper::Body;
 use internal_dns::ServiceName;
+use nexus_client::types::SledAgentInfo;
 use omicron_common::api::external::Error;
 use omicron_common::api::internal::nexus::UpdateArtifactId;
+use schemars::JsonSchema;
+use serde::Deserialize;
+use uuid::Uuid;
 
 /// Implements a fake Nexus.
 ///
@@ -28,6 +32,18 @@ pub trait FakeNexusServer: Send + Sync {
     ) -> Result<Vec<u8>, Error> {
         Err(Error::internal_error("Not implemented"))
     }
+
+    fn sled_agent_get(&self, _sled_id: Uuid) -> Result<SledAgentInfo, Error> {
+        Err(Error::internal_error("Not implemented"))
+    }
+
+    fn sled_agent_put(
+        &self,
+        _sled_id: Uuid,
+        _info: SledAgentInfo,
+    ) -> Result<(), Error> {
+        Err(Error::internal_error("Not implemented"))
+    }
 }
 
 /// Describes the server context type.
@@ -52,9 +68,50 @@ async fn cpapi_artifact_download(
     ))
 }
 
+/// Path parameters for Sled Agent requests (internal API)
+#[derive(Deserialize, JsonSchema)]
+struct SledAgentPathParam {
+    sled_id: Uuid,
+}
+
+/// Return information about the given sled agent
+#[endpoint {
+     method = GET,
+     path = "/sled-agents/{sled_id}",
+ }]
+async fn sled_agent_get(
+    request_context: RequestContext<ServerContext>,
+    path_params: Path<SledAgentPathParam>,
+) -> Result<HttpResponseOk<SledAgentInfo>, HttpError> {
+    let context = request_context.context();
+
+    Ok(HttpResponseOk(
+        context.sled_agent_get(path_params.into_inner().sled_id)?,
+    ))
+}
+
+#[endpoint {
+     method = POST,
+     path = "/sled-agents/{sled_id}",
+ }]
+async fn sled_agent_put(
+    request_context: RequestContext<ServerContext>,
+    path_params: Path<SledAgentPathParam>,
+    sled_info: TypedBody<SledAgentInfo>,
+) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+    let context = request_context.context();
+    context.sled_agent_put(
+        path_params.into_inner().sled_id,
+        sled_info.into_inner(),
+    )?;
+    Ok(HttpResponseUpdatedNoContent())
+}
+
 fn api() -> ApiDescription<ServerContext> {
     let mut api = ApiDescription::new();
     api.register(cpapi_artifact_download).unwrap();
+    api.register(sled_agent_get).unwrap();
+    api.register(sled_agent_put).unwrap();
     api
 }
 
diff --git a/sled-agent/src/hardware_monitor.rs b/sled-agent/src/hardware_monitor.rs
index 698d2d4608..5dcd060f98 100644
--- a/sled-agent/src/hardware_monitor.rs
+++ b/sled-agent/src/hardware_monitor.rs
@@ -172,7 +172,7 @@ impl HardwareMonitor {
                 }
                 HardwareUpdate::TofinoDeviceChange => {
                     if let Some(sled_agent) = &mut self.sled_agent {
-                        sled_agent.notify_nexus_about_self(&self.log);
+                        sled_agent.notify_nexus_about_self(&self.log).await;
                     }
                 }
                 HardwareUpdate::DiskAdded(disk) => {
@@ -234,10 +234,12 @@ impl HardwareMonitor {
     // We use this when we're monitoring hardware for the first
     // time, and if we miss notifications.
     async fn check_latest_hardware_snapshot(&mut self) {
-        let underlay_network = self.sled_agent.as_ref().map(|sled_agent| {
-            sled_agent.notify_nexus_about_self(&self.log);
-            sled_agent.switch_zone_underlay_info()
-        });
+        let underlay_network = if let Some(sled_agent) = &self.sled_agent {
+            sled_agent.notify_nexus_about_self(&self.log).await;
+            Some(sled_agent.switch_zone_underlay_info())
+        } else {
+            None
+        };
         info!(
             self.log, "Checking current full hardware snapshot";
             "underlay_network_info" => ?underlay_network,
diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs
index 666d970538..badaf2d7ba 100644
--- a/sled-agent/src/instance_manager.rs
+++ b/sled-agent/src/instance_manager.rs
@@ -14,16 +14,16 @@ use crate::params::{
     InstanceHardware, InstanceMigrationSourceParams, InstancePutStateResponse,
     InstanceStateRequested, InstanceUnregisterResponse,
 };
+use crate::vmm_reservoir::VmmReservoirManagerHandle;
 use crate::zone_bundle::BundleError;
 use crate::zone_bundle::ZoneBundler;
+use omicron_common::api::external::ByteCount;
 
 use anyhow::anyhow;
 use illumos_utils::dladm::Etherstub;
 use illumos_utils::link::VnicAllocator;
 use illumos_utils::opte::PortManager;
 use illumos_utils::running_zone::ZoneBuilderFactory;
-use illumos_utils::vmm_reservoir;
-use omicron_common::api::external::ByteCount;
 use omicron_common::api::internal::nexus::InstanceRuntimeState;
 use omicron_common::api::internal::nexus::SledInstanceState;
 use omicron_common::api::internal::nexus::VmmRuntimeState;
@@ -31,7 +31,7 @@ use sled_storage::manager::StorageHandle;
 use slog::Logger;
 use std::collections::BTreeMap;
 use std::net::SocketAddr;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
 use tokio::sync::{mpsc, oneshot};
 use uuid::Uuid;
 
@@ -49,12 +49,6 @@ pub enum Error {
     #[error("OPTE port management error: {0}")]
     Opte(#[from] illumos_utils::opte::Error),
 
-    #[error("Failed to create reservoir: {0}")]
-    Reservoir(#[from] vmm_reservoir::Error),
-
-    #[error("Invalid reservoir configuration: {0}")]
-    ReservoirConfig(String),
-
     #[error("Cannot find data link: {0}")]
     Underlay(#[from] sled_hardware::underlay::Error),
 
@@ -73,12 +67,6 @@ pub enum Error {
     RequestDropped(#[from] oneshot::error::RecvError),
 }
 
-pub enum ReservoirMode {
-    None,
-    Size(u32),
-    Percentage(u8),
-}
-
 pub(crate) struct InstanceManagerServices {
     pub nexus_client: NexusClientWithResolver,
     pub vnic_allocator: VnicAllocator<Etherstub>,
@@ -91,14 +79,8 @@ pub(crate) struct InstanceManagerServices {
 // Describes the internals of the "InstanceManager", though most of the
 // instance manager's state exists within the "InstanceManagerRunner" structure.
 struct InstanceManagerInternal {
-    log: Logger,
     tx: mpsc::Sender<InstanceManagerRequest>,
-    // NOTE: Arguably, this field could be "owned" by the InstanceManagerRunner.
-    // It was not moved there, and the reservoir functions were not converted to
-    // use the message-passing interface (see: "InstanceManagerRequest") because
-    // callers of "get/set reservoir size" are not async, and (in the case of
-    // getting the size) they also do not expect a "Result" type.
-    reservoir_size: Mutex<ByteCount>,
+    vmm_reservoir_manager: VmmReservoirManagerHandle,
 
     #[allow(dead_code)]
     runner_handle: tokio::task::JoinHandle<()>,
@@ -111,6 +93,7 @@ pub struct InstanceManager {
 
 impl InstanceManager {
     /// Initializes a new [`InstanceManager`] object.
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         log: Logger,
         nexus_client: NexusClientWithResolver,
@@ -119,6 +102,7 @@ impl InstanceManager {
         storage: StorageHandle,
         zone_bundler: ZoneBundler,
         zone_builder_factory: ZoneBuilderFactory,
+        vmm_reservoir_manager: VmmReservoirManagerHandle,
     ) -> Result<InstanceManager, Error> {
         let (tx, rx) = mpsc::channel(QUEUE_SIZE);
         let (terminate_tx, terminate_rx) = mpsc::unbounded_channel();
@@ -143,91 +127,13 @@ impl InstanceManager {
 
         Ok(Self {
             inner: Arc::new(InstanceManagerInternal {
-                log,
                 tx,
-                // no reservoir size set on startup
-                reservoir_size: Mutex::new(ByteCount::from_kibibytes_u32(0)),
+                vmm_reservoir_manager,
                 runner_handle,
             }),
         })
     }
 
-    /// Sets the VMM reservoir to the requested percentage of usable physical
-    /// RAM or to a size in MiB. Either mode will round down to the nearest
-    /// aligned size required by the control plane.
-    pub fn set_reservoir_size(
-        &self,
-        hardware: &sled_hardware::HardwareManager,
-        mode: ReservoirMode,
-    ) -> Result<(), Error> {
-        let hardware_physical_ram_bytes = hardware.usable_physical_ram_bytes();
-        let req_bytes = match mode {
-            ReservoirMode::None => return Ok(()),
-            ReservoirMode::Size(mb) => {
-                let bytes = ByteCount::from_mebibytes_u32(mb).to_bytes();
-                if bytes > hardware_physical_ram_bytes {
-                    return Err(Error::ReservoirConfig(format!(
-                        "cannot specify a reservoir of {bytes} bytes when \
-                        physical memory is {hardware_physical_ram_bytes} bytes",
-                    )));
-                }
-                bytes
-            }
-            ReservoirMode::Percentage(percent) => {
-                if !matches!(percent, 1..=99) {
-                    return Err(Error::ReservoirConfig(format!(
-                        "VMM reservoir percentage of {} must be between 0 and \
-                        100",
-                        percent
-                    )));
-                };
-                (hardware_physical_ram_bytes as f64 * (percent as f64 / 100.0))
-                    .floor() as u64
-            }
-        };
-
-        let req_bytes_aligned = vmm_reservoir::align_reservoir_size(req_bytes);
-
-        if req_bytes_aligned == 0 {
-            warn!(
-                self.inner.log,
-                "Requested reservoir size of {} bytes < minimum aligned size \
-                of {} bytes",
-                req_bytes,
-                vmm_reservoir::RESERVOIR_SZ_ALIGN
-            );
-            return Ok(());
-        }
-
-        // The max ByteCount value is i64::MAX, which is ~8 million TiB.
-        // As this value is either a percentage of DRAM or a size in MiB
-        // represented as a u32, constructing this should always work.
-        let reservoir_size = ByteCount::try_from(req_bytes_aligned).unwrap();
-        if let ReservoirMode::Percentage(percent) = mode {
-            info!(
-                self.inner.log,
-                "{}% of {} physical ram = {} bytes)",
-                percent,
-                hardware_physical_ram_bytes,
-                req_bytes,
-            );
-        }
-        info!(
-            self.inner.log,
-            "Setting reservoir size to {reservoir_size} bytes"
-        );
-        vmm_reservoir::ReservoirControl::set(reservoir_size)?;
-
-        *self.inner.reservoir_size.lock().unwrap() = reservoir_size;
-
-        Ok(())
-    }
-
-    /// Returns the last-set size of the reservoir
-    pub fn reservoir_size(&self) -> ByteCount {
-        *self.inner.reservoir_size.lock().unwrap()
-    }
-
     #[allow(clippy::too_many_arguments)]
     pub async fn ensure_registered(
         &self,
@@ -383,6 +289,11 @@ impl InstanceManager {
             .map_err(|_| Error::FailedSendInstanceManagerClosed)?;
         rx.await?
     }
+
+    /// Returns the last-set size of the reservoir
+    pub fn reservoir_size(&self) -> ByteCount {
+        self.inner.vmm_reservoir_manager.reservoir_size()
+    }
 }
 
 // Most requests that can be sent to the "InstanceManagerRunner" task.
diff --git a/sled-agent/src/lib.rs b/sled-agent/src/lib.rs
index 527b483ee8..bfc23b248d 100644
--- a/sled-agent/src/lib.rs
+++ b/sled-agent/src/lib.rs
@@ -40,6 +40,7 @@ mod smf_helper;
 mod storage_monitor;
 mod swap_device;
 mod updates;
+mod vmm_reservoir;
 mod zone_bundle;
 
 #[cfg(test)]
diff --git a/sled-agent/src/nexus.rs b/sled-agent/src/nexus.rs
index 4cd97d9ba8..7e7d60f6a4 100644
--- a/sled-agent/src/nexus.rs
+++ b/sled-agent/src/nexus.rs
@@ -3,16 +3,20 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 pub use nexus_client::Client as NexusClient;
+use omicron_common::api::external::Generation;
 
+use crate::vmm_reservoir::VmmReservoirManagerHandle;
 use internal_dns::resolver::{ResolveError, Resolver};
 use internal_dns::ServiceName;
+use nexus_client::types::SledAgentInfo;
 use omicron_common::address::NEXUS_INTERNAL_PORT;
+use sled_hardware::HardwareManager;
 use slog::Logger;
-use std::future::Future;
-use std::pin::Pin;
+use std::net::SocketAddrV6;
 use std::sync::Arc;
-use tokio::sync::mpsc;
-use tokio::task::JoinHandle;
+use tokio::sync::{broadcast, mpsc, oneshot, Notify};
+use tokio::time::{interval, Duration, MissedTickBehavior};
+use uuid::Uuid;
 
 /// A thin wrapper over a progenitor-generated NexusClient.
 ///
@@ -72,45 +76,6 @@ impl NexusClientWithResolver {
     }
 }
 
-type NexusRequestFut = dyn Future<Output = ()> + Send;
-type NexusRequest = Pin<Box<NexusRequestFut>>;
-
-/// A queue of futures which represent requests to Nexus.
-pub struct NexusRequestQueue {
-    tx: mpsc::UnboundedSender<NexusRequest>,
-    _worker: JoinHandle<()>,
-}
-
-impl NexusRequestQueue {
-    /// Creates a new request queue, along with a worker which executes
-    /// any incoming tasks.
-    pub fn new() -> Self {
-        // TODO(https://github.com/oxidecomputer/omicron/issues/1917):
-        // In the future, this should basically just be a wrapper around a
-        // generation number, and we shouldn't be serializing requests to Nexus.
-        //
-        // In the meanwhile, we're using an unbounded_channel for simplicity, so
-        // that we don't need to cope with dropped notifications /
-        // retransmissions.
-        let (tx, mut rx) = mpsc::unbounded_channel();
-
-        let _worker = tokio::spawn(async move {
-            while let Some(fut) = rx.recv().await {
-                fut.await;
-            }
-        });
-
-        Self { tx, _worker }
-    }
-
-    /// Gets access to the sending portion of the request queue.
-    ///
-    /// Callers can use this to add their own requests.
-    pub fn sender(&self) -> &mpsc::UnboundedSender<NexusRequest> {
-        &self.tx
-    }
-}
-
 pub fn d2n_params(
     params: &dns_service_client::types::DnsConfigParams,
 ) -> nexus_client::types::DnsConfigParams {
@@ -202,3 +167,846 @@ impl ConvertInto<nexus_client::types::DatasetKind>
         }
     }
 }
+
+// Somewhat arbitrary bound size, large enough that we should never hit it.
+const QUEUE_SIZE: usize = 256;
+
+enum NexusNotifierMsg {
+    // Inform nexus about a change to this sled-agent. This is just a
+    // notification to perform a send. The request is constructed inside
+    // `NexusNotifierTask`.
+    NotifyNexusAboutSelf,
+
+    // Return status of the `NexusNotifierTask`
+    Status(oneshot::Sender<NexusNotifierTaskStatus>),
+}
+
+#[derive(Debug)]
+pub struct NexusNotifierTaskStatus {
+    pub nexus_known_info: Option<NexusKnownInfo>,
+    pub has_pending_notification: bool,
+    pub has_outstanding_request: bool,
+    pub total_get_requests_started: u64,
+    pub total_get_requests_completed: u64,
+    pub total_put_requests_started: u64,
+    pub total_put_requests_completed: u64,
+    pub cancelled_pending_notifications: u64,
+}
+
+#[derive(Debug)]
+pub struct NexusNotifierHandle {
+    tx: mpsc::Sender<NexusNotifierMsg>,
+}
+
+#[derive(Debug)]
+pub struct SenderOrReceiverDropped {}
+
+impl NexusNotifierHandle {
+    pub async fn notify_nexus_about_self(&self, log: &Logger) {
+        if let Err(_) =
+            self.tx.send(NexusNotifierMsg::NotifyNexusAboutSelf).await
+        {
+            warn!(log, "Failed to send to NexusNotifierTask: did it exit?");
+        }
+    }
+
+    #[allow(unused)]
+    pub async fn get_status(
+        &self,
+    ) -> Result<NexusNotifierTaskStatus, SenderOrReceiverDropped> {
+        let (tx, rx) = oneshot::channel();
+        self.tx
+            .send(NexusNotifierMsg::Status(tx))
+            .await
+            .map_err(|_| SenderOrReceiverDropped {})?;
+        rx.await.map_err(|_| SenderOrReceiverDropped {})
+    }
+}
+
+// The type of operation issued to nexus
+enum NexusOp {
+    Get,
+    Put,
+}
+
+/// What sled-agent has confirmed that Nexus knows about this sled-agent
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum NexusKnownInfo {
+    // CRDB doesn't contain a record for this sled-agent
+    NotFound,
+    Found(SledAgentInfo),
+}
+
+impl NexusKnownInfo {
+    fn generation(&self) -> Generation {
+        match self {
+            NexusKnownInfo::NotFound => Generation::new(),
+            NexusKnownInfo::Found(known) => known.generation,
+        }
+    }
+}
+
+/// Return the latest sled-agent info. Boxed for use in testing.
+type GetSledAgentInfo = Box<dyn Fn(Generation) -> SledAgentInfo + Send>;
+
+// A mechanism owned by the `NexusNotifierTask` that allows it to access
+// enough information to send a `SledAgentInfo` to Nexus.
+pub struct NexusNotifierInput {
+    pub sled_id: Uuid,
+    pub sled_address: SocketAddrV6,
+    pub nexus_client: NexusClient,
+    pub hardware: HardwareManager,
+    pub vmm_reservoir_manager: VmmReservoirManagerHandle,
+}
+
+// Type returned from a join on the "outstanding request" task`
+type NexusRsp = (
+    NexusOp,
+    Result<SledAgentInfo, nexus_client::Error<nexus_client::types::Error>>,
+);
+
+/// A mechanism for notifying nexus about this sled agent
+///
+/// The semantics are as follows:
+///  1. At any time there is a single outstanding HTTP request to nexus
+///  2. On startup, this task gets the latest sled-agent info, if any, from
+///     nexus, and saves it.
+///  3. Whenever the state needs to be updated to a value different
+///     from what nexus has, the generation number is bumped
+///     and the new state transmitted.
+///  4. If a caller requests an update to be made to nexus and it succeeds
+///     the known value is set to what was updated.
+///  5. If the request fails, we go ahead and set the known state to `None`.
+///     This will trigger a get request for the latest state. In the case of
+///     a timeout we are not sure if the last update succeeded and so need to
+///     learn that. In the case of an explicit rejection from nexus, we also
+///     want to learn the latest state. We need to learn the latest state this
+///     because an update can fail if the sled-agent's state is out of date
+///     with respect to what's in CRDB. The only way to fix that is to get the
+///     latest known state, and it's easiest to just do that unconditionally
+///     rather than trying to reason about exactly why Nexus rejected the
+///     request.
+///  6. An exception to step 5 is that if the latest state returned from Nexus
+///     contains `decommissioned = true`, then we stop dead in our tracks and no
+///     longer send requests to nexus.
+pub struct NexusNotifierTask {
+    sled_id: Uuid,
+    nexus_client: NexusClient,
+    get_sled_agent_info: GetSledAgentInfo,
+
+    // Notifies when the VMM reservoir size changes.
+    //
+    // This is an option, since we don't use a reservoir during testing. There
+    // really isn't a better place to put this in sled-agent, and this task is
+    // the only one that cares about updates.
+    vmm_reservoir_size_updated: Option<broadcast::Receiver<()>>,
+
+    log: Logger,
+    rx: mpsc::Receiver<NexusNotifierMsg>,
+
+    // The last known value either put or gotten from nexus
+    //
+    // We only send `Get` requests if we haven't learned any info yet
+    nexus_known_info: Option<NexusKnownInfo>,
+
+    // Do we need to notify nexus about an update to our state?
+    pending_notification: bool,
+
+    // We only have one outstanding nexus request at a time.
+    //
+    // We spawn a task to manage this request so we don't block our main
+    // notifier task. We wait for a notification on `outstanding_req_ready`
+    // and then get the result from the `JoinHandle`.
+    outstanding_request: Option<tokio::task::JoinHandle<NexusRsp>>,
+
+    // A notification sent from the outstanding task when it has completed.
+    outstanding_req_ready: Arc<Notify>,
+
+    // Some stats for testing/debugging
+    total_get_requests_started: u64,
+    total_put_requests_started: u64,
+    total_get_requests_completed: u64,
+    total_put_requests_completed: u64,
+    cancelled_pending_notifications: u64,
+}
+
+impl NexusNotifierTask {
+    pub fn new(
+        input: NexusNotifierInput,
+        log: &Logger,
+    ) -> (NexusNotifierTask, NexusNotifierHandle) {
+        let NexusNotifierInput {
+            sled_id,
+            sled_address,
+            nexus_client,
+            hardware,
+            vmm_reservoir_manager,
+        } = input;
+
+        let vmm_reservoir_size_updated =
+            Some(vmm_reservoir_manager.subscribe_for_size_updates());
+
+        // Box a function that can return the latest `SledAgentInfo`
+        let get_sled_agent_info = Box::new(move |generation| {
+            let role = if hardware.is_scrimlet() {
+                nexus_client::types::SledRole::Scrimlet
+            } else {
+                nexus_client::types::SledRole::Gimlet
+            };
+            SledAgentInfo {
+                sa_address: sled_address.to_string(),
+                role,
+                baseboard: hardware.baseboard().convert(),
+                usable_hardware_threads: hardware.online_processor_count(),
+                usable_physical_ram: hardware
+                    .usable_physical_ram_bytes()
+                    .into(),
+                reservoir_size: vmm_reservoir_manager.reservoir_size().into(),
+                generation,
+                decommissioned: false,
+            }
+        });
+
+        let (tx, rx) = mpsc::channel(QUEUE_SIZE);
+        (
+            NexusNotifierTask {
+                sled_id,
+                nexus_client,
+                get_sled_agent_info,
+                vmm_reservoir_size_updated,
+                log: log.new(o!("component" => "NexusNotifierTask")),
+                rx,
+                nexus_known_info: None,
+                // We start with pending true, because we always want to attempt
+                // to retrieve the current generation number before we upsert
+                // ourselves.
+                pending_notification: true,
+                outstanding_request: None,
+                outstanding_req_ready: Arc::new(Notify::new()),
+                total_get_requests_started: 0,
+                total_put_requests_started: 0,
+                total_get_requests_completed: 0,
+                total_put_requests_completed: 0,
+                cancelled_pending_notifications: 0,
+            },
+            NexusNotifierHandle { tx },
+        )
+    }
+
+    #[cfg(test)]
+    pub fn new_for_test(
+        sled_id: Uuid,
+        nexus_client: NexusClient,
+        get_sled_agent_info: GetSledAgentInfo,
+        log: &Logger,
+    ) -> (NexusNotifierTask, NexusNotifierHandle) {
+        let (tx, rx) = mpsc::channel(QUEUE_SIZE);
+
+        // During testing we don't actually have a reservoir. Just dummy it out.
+        let vmm_reservoir_size_updated = None;
+
+        (
+            NexusNotifierTask {
+                sled_id,
+                nexus_client,
+                get_sled_agent_info,
+                vmm_reservoir_size_updated,
+                log: log.new(o!("component" => "NexusNotifierTask")),
+                rx,
+                nexus_known_info: None,
+                // We start with pending true, because we always want to attempt
+                // to retrieve the current generation number before we upsert
+                // ourselves.
+                pending_notification: true,
+                outstanding_request: None,
+                outstanding_req_ready: Arc::new(Notify::new()),
+                total_get_requests_started: 0,
+                total_put_requests_started: 0,
+                total_get_requests_completed: 0,
+                total_put_requests_completed: 0,
+                cancelled_pending_notifications: 0,
+            },
+            NexusNotifierHandle { tx },
+        )
+    }
+
+    /// Run the main receive loop of the `NexusNotifierTask`
+    ///
+    /// This should be spawned into a tokio task
+    pub async fn run(mut self) {
+        let (_tx, mut vmm_size_updated) = if let Some(vmm_size_updated) =
+            self.vmm_reservoir_size_updated.take()
+        {
+            (None, vmm_size_updated)
+        } else {
+            // Dummy channel for testing
+            let (tx, rx) = broadcast::channel(1);
+            (Some(tx), rx)
+        };
+
+        loop {
+            const RETRY_TIMEOUT: Duration = Duration::from_secs(2);
+            let mut interval = interval(RETRY_TIMEOUT);
+            interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
+            tokio::select! {
+                req = self.rx.recv() => {
+                    let Some(req) = req else {
+                        warn!(self.log, "All senders dropped. Exiting.");
+                        break;
+                    };
+                    match req {
+                        NexusNotifierMsg::NotifyNexusAboutSelf => {
+                            // We'll contact nexus on the next timeout
+                            self.pending_notification = true;
+                        }
+                        NexusNotifierMsg::Status(reply_tx) => {
+                            let _ = reply_tx.send(NexusNotifierTaskStatus {
+                                nexus_known_info: self.nexus_known_info.clone(),
+                                has_outstanding_request: self.outstanding_request.is_some(),
+                                has_pending_notification: self.pending_notification,
+                                total_get_requests_started: self.total_get_requests_started,
+                                total_put_requests_started: self.total_put_requests_started,
+                                total_get_requests_completed: self.total_get_requests_completed,
+                                total_put_requests_completed: self.total_put_requests_completed,
+                                cancelled_pending_notifications: self.cancelled_pending_notifications
+                            });
+                        }
+
+                    }
+                }
+                _ = vmm_size_updated.recv() => {
+                    self.pending_notification = true;
+                }
+                _ =  self.outstanding_req_ready.notified() => {
+                    // Our request task has completed. Let's check the result.
+                    self.handle_nexus_reply().await;
+
+                }
+                _ = interval.tick(), if self.pending_notification => {
+                    self.contact_nexus().await;
+                }
+            }
+        }
+    }
+
+    /// If we haven't yet learned the latest info that nexus has about us
+    /// then go ahead and send a get request. Otherwise, if necessaary, send
+    /// a put request to nexus with the latest `SledAgentInfo`.
+    ///
+    /// Only one outstanding request is allowed at a time, so if there is
+    /// already one outstanding then we return and will try again later.
+    async fn contact_nexus(&mut self) {
+        // Is there already an outstanding request to nexus?
+        if self.outstanding_request.is_some() {
+            return;
+        }
+
+        let client = self.nexus_client.clone();
+        let sled_id = self.sled_id;
+
+        // Have we learned about any generations stored in CRDB yet?
+        if let Some(known_info) = &self.nexus_known_info {
+            let mut info = (self.get_sled_agent_info)(known_info.generation());
+
+            // Does CRDB actually contain an existing record for this sled?
+            match known_info {
+                NexusKnownInfo::NotFound => {
+                    // Nothing to do. We must send the request as is.
+                }
+                NexusKnownInfo::Found(known) => {
+                    // Don't send any more requests if this sled-agent has been decommissioned.
+                    if known.decommissioned {
+                        return;
+                    }
+
+                    // We don't need to send a request if the info is identical to what
+                    // nexus knows
+                    if info == *known {
+                        self.pending_notification = false;
+                        self.cancelled_pending_notifications += 1;
+                        return;
+                    }
+
+                    // Bump the generation of the value that nexus knows, so
+                    // that the update takes precedence.
+                    info.generation = known.generation.next();
+                }
+            }
+
+            let outstanding_req_ready = self.outstanding_req_ready.clone();
+            self.total_put_requests_started += 1;
+            self.outstanding_request = Some(tokio::spawn(async move {
+                let res =
+                    client.sled_agent_put(&sled_id, &info).await.map(|_| info);
+                outstanding_req_ready.notify_one();
+                (NexusOp::Put, res)
+            }));
+        } else {
+            let outstanding_req_ready = self.outstanding_req_ready.clone();
+            self.total_get_requests_started += 1;
+            self.outstanding_request = Some(tokio::spawn(async move {
+                let res = client
+                    .sled_agent_get(&sled_id)
+                    .await
+                    .map(|info| info.into_inner());
+                outstanding_req_ready.notify_one();
+                (NexusOp::Get, res)
+            }));
+        }
+    }
+
+    /// Handle a reply from nexus by extracting the value from the `JoinHandle` of
+    /// the last outstanding request.
+    async fn handle_nexus_reply(&mut self) {
+        let (op, res) = match self
+            .outstanding_request
+            .take()
+            .expect("missing JoinHandle")
+            .await
+        {
+            Ok(res) => res,
+            Err(e) => {
+                error!(self.log, "Nexus request task exited prematurely: {e}");
+                return;
+            }
+        };
+        match (op, res) {
+            (NexusOp::Get, Ok(info)) => {
+                info!(
+                    self.log,
+                    "Retrieved SledAgentInfo from Nexus: {:?}", info
+                );
+                assert!(self.nexus_known_info.is_none());
+                if info.decommissioned {
+                    info!(self.log, "Sled Agent Decommissioned.");
+                }
+                self.nexus_known_info = Some(NexusKnownInfo::Found(info));
+                self.total_get_requests_completed += 1;
+            }
+            (NexusOp::Put, Ok(info)) => {
+                // Unwrap Safety: we must have a known value in order to have
+                // submitted a PUT request in the first place.
+                info!(
+                    self.log,
+                    "Successfully put SledAgentInfo to nexus";
+                    "old_generation" =>
+                        %self.nexus_known_info.as_ref().unwrap().generation(),
+                    "new_generation" => %info.generation,
+                );
+                self.nexus_known_info = Some(NexusKnownInfo::Found(info));
+                self.total_put_requests_completed += 1;
+            }
+            (NexusOp::Get, Err(e)) => {
+                self.total_get_requests_completed += 1;
+                if e.status() == Some(http::StatusCode::NOT_FOUND) {
+                    if self.nexus_known_info.is_none() {
+                        self.nexus_known_info = Some(NexusKnownInfo::NotFound);
+                        return;
+                    }
+                    warn!(
+                        self.log,
+                        "Nexus doesn't have have the latest state of this\
+                        sled-agent, but sled-agent thinks it should. Setting \
+                        known state to `None` and trying again.",
+                    );
+                    self.nexus_known_info = None;
+                    return;
+                }
+                self.nexus_known_info = None;
+                warn!(
+                    self.log,
+                    "Received Error from Nexus for Get request: {:?}", e
+                );
+            }
+            (NexusOp::Put, Err(e)) => {
+                self.total_put_requests_completed += 1;
+                self.nexus_known_info = None;
+                warn!(
+                    self.log,
+                    "Received Error from Nexus for Put request: {:?}", e
+                );
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::sync::atomic::{AtomicBool, Ordering};
+
+    use crate::fakes::nexus::FakeNexusServer;
+    use omicron_test_utils::dev::poll::{
+        wait_for_condition as wait_for, CondCheckError,
+    };
+
+    use super::*;
+    use omicron_common::api::external::{
+        ByteCount, Error, Generation, LookupType, MessagePair, ResourceType,
+    };
+    use omicron_test_utils::dev::test_setup_log;
+    use sled_hardware::Baseboard;
+
+    /// Pretend to be CRDB storing info about a sled-agent
+    #[derive(Default, Clone)]
+    struct FakeCrdb {
+        info: Arc<std::sync::Mutex<Option<SledAgentInfo>>>,
+    }
+
+    // Puts and Gets with our magical fake nexus
+    struct NexusServer {
+        fake_crdb: FakeCrdb,
+        // Injectable errors
+        get_error: Arc<AtomicBool>,
+        put_error: Arc<AtomicBool>,
+    }
+    impl FakeNexusServer for NexusServer {
+        fn sled_agent_get(
+            &self,
+            sled_id: Uuid,
+        ) -> Result<SledAgentInfo, Error> {
+            // Always disable any errors after the first time. This simplifies
+            // testing due to lack of races.
+            let injected_err = self.get_error.swap(false, Ordering::SeqCst);
+            if injected_err {
+                return Err(Error::ServiceUnavailable {
+                    internal_message: "go away".into(),
+                });
+            }
+
+            self.fake_crdb.info.lock().unwrap().clone().ok_or(
+                Error::ObjectNotFound {
+                    type_name: ResourceType::Sled,
+                    lookup_type: LookupType::ById(sled_id),
+                },
+            )
+        }
+
+        fn sled_agent_put(
+            &self,
+            _sled_id: Uuid,
+            info: SledAgentInfo,
+        ) -> Result<(), Error> {
+            // Always disable any errors after the first time. This simplifies
+            // testing due to lack of races.
+            let injected_err = self.put_error.swap(false, Ordering::SeqCst);
+            if injected_err {
+                return Err(Error::Conflict {
+                    message: MessagePair::new(
+                        "I don't like the cut of your jib".into(),
+                    ),
+                });
+            }
+
+            let mut crdb_info = self.fake_crdb.info.lock().unwrap();
+            *crdb_info = Some(info);
+            Ok(())
+        }
+    }
+
+    #[tokio::test]
+    async fn nexus_self_notification_test() {
+        let logctx = test_setup_log("nexus_notification_test");
+        let log = &logctx.log;
+        let sa_address = "::1".to_string();
+        let fake_crdb = FakeCrdb::default();
+        let sled_id = Uuid::new_v4();
+        let get_error = Arc::new(AtomicBool::new(false));
+        let put_error = Arc::new(AtomicBool::new(false));
+
+        let nexus_server = crate::fakes::nexus::start_test_server(
+            log.clone(),
+            Box::new(NexusServer {
+                fake_crdb: fake_crdb.clone(),
+                get_error: get_error.clone(),
+                put_error: put_error.clone(),
+            }),
+        );
+        let nexus_client = NexusClient::new(
+            &format!("http://{}", nexus_server.local_addr()),
+            log.clone(),
+        );
+
+        // Pretend we are retrieving the latest `SledAgentInfo` from hardware and
+        // the VMM reservoir.
+        let latest_sled_agent_info =
+            Arc::new(std::sync::Mutex::new(SledAgentInfo {
+                sa_address: sa_address.clone(),
+                role: nexus_client::types::SledRole::Gimlet,
+                baseboard: Baseboard::new_pc("test".into(), "test".into())
+                    .convert(),
+                usable_hardware_threads: 16,
+                usable_physical_ram: ByteCount::from(1024 * 1024 * 1024u32)
+                    .into(),
+                reservoir_size: ByteCount::from(0u32).into(),
+                generation: Generation::new(),
+                decommissioned: false,
+            }));
+        let latest_sled_agent_info2 = latest_sled_agent_info.clone();
+
+        // Return `SledAgentInfo` from our test object
+        let get_sled_agent_info: GetSledAgentInfo =
+            Box::new(move |generation| {
+                let mut info = latest_sled_agent_info2.lock().unwrap().clone();
+                info.generation = generation;
+                info
+            });
+
+        let (nexus_notifier_task, handle) = NexusNotifierTask::new_for_test(
+            sled_id,
+            nexus_client,
+            get_sled_agent_info,
+            log,
+        );
+
+        tokio::spawn(async move {
+            nexus_notifier_task.run().await;
+        });
+
+        // Ensure that the task will try to initially talk to Nexus and get back
+        // some `SledAgentInfo` or a `NotFound`, depending upon timing.
+        let status = wait_for::<_, (), _, _>(
+            || async {
+                let status = handle.get_status().await.unwrap();
+                if status.nexus_known_info.is_some() {
+                    Ok(status)
+                } else {
+                    Err(CondCheckError::NotYet)
+                }
+            },
+            &Duration::from_millis(2),
+            &Duration::from_secs(15),
+        )
+        .await
+        .expect("Failed to get status from Nexus");
+
+        if status.total_put_requests_completed == 0 {
+            assert_eq!(status.nexus_known_info, Some(NexusKnownInfo::NotFound));
+        }
+
+        // Wait for a steady state, when the the latest info has been put to nexus
+        let status = wait_for::<_, (), _, _>(
+            || async {
+                let status = handle.get_status().await.unwrap();
+                if !status.has_pending_notification {
+                    Ok(status)
+                } else {
+                    Err(CondCheckError::NotYet)
+                }
+            },
+            &Duration::from_millis(2),
+            &Duration::from_secs(15),
+        )
+        .await
+        .expect("Failed to get status from Nexus");
+
+        assert_eq!(status.total_get_requests_started, 1u64);
+        assert_eq!(status.total_put_requests_started, 1u64);
+        assert_eq!(status.total_get_requests_completed, 1u64);
+        assert_eq!(status.total_put_requests_completed, 1u64);
+        assert_eq!(status.has_outstanding_request, false);
+        assert_eq!(status.cancelled_pending_notifications, 1);
+        let expected = latest_sled_agent_info.lock().unwrap().clone();
+        assert_eq!(
+            status.nexus_known_info,
+            Some(NexusKnownInfo::Found(expected)),
+        );
+
+        // Trigger another update notification
+        //
+        // We haven't changed the underlying `SledAgentInfo` so this request
+        // should be cancelled without a request going to nexus.
+        handle.notify_nexus_about_self(log).await;
+        let status = wait_for::<_, (), _, _>(
+            || async {
+                let status = handle.get_status().await.unwrap();
+                if status.cancelled_pending_notifications == 2 {
+                    Ok(status)
+                } else {
+                    Err(CondCheckError::NotYet)
+                }
+            },
+            &Duration::from_millis(2),
+            &Duration::from_secs(15),
+        )
+        .await
+        .expect("Failed to get status from Nexus");
+        assert_eq!(status.total_get_requests_started, 1u64);
+        assert_eq!(status.total_put_requests_started, 1u64);
+        assert_eq!(status.total_get_requests_completed, 1u64);
+        assert_eq!(status.total_put_requests_completed, 1u64);
+        assert_eq!(status.has_outstanding_request, false);
+        let expected = latest_sled_agent_info.lock().unwrap().clone();
+        assert_eq!(
+            status.nexus_known_info,
+            Some(NexusKnownInfo::Found(expected)),
+        );
+
+        // Update the VMM reservoir size and trigger a successful put to Nexus.
+        {
+            let mut info = latest_sled_agent_info.lock().unwrap();
+            info.reservoir_size = (1024 * 1024u64).into();
+            info.generation = info.generation.next();
+        }
+
+        // Wait for a steady state, when the the latest info has been put to nexus
+        handle.notify_nexus_about_self(log).await;
+        let status = wait_for::<_, (), _, _>(
+            || async {
+                let status = handle.get_status().await.unwrap();
+                if !status.has_pending_notification {
+                    Ok(status)
+                } else {
+                    Err(CondCheckError::NotYet)
+                }
+            },
+            &Duration::from_millis(2),
+            &Duration::from_secs(15),
+        )
+        .await
+        .expect("Failed to get status from Nexus");
+
+        assert_eq!(status.total_get_requests_started, 1u64);
+        assert_eq!(status.total_get_requests_completed, 1u64);
+        assert_eq!(status.total_put_requests_started, 2u64);
+        assert_eq!(status.total_put_requests_completed, 2u64);
+        assert_eq!(status.has_outstanding_request, false);
+        assert_eq!(status.cancelled_pending_notifications, 3);
+        let expected = latest_sled_agent_info.lock().unwrap().clone();
+        assert_eq!(
+            status.nexus_known_info,
+            Some(NexusKnownInfo::Found(expected)),
+        );
+
+        // Inject a put error and trigger a put to nexus after updating the VMM
+        // reservoir size. It should eventually succeed.
+        put_error.store(true, Ordering::SeqCst);
+        {
+            let mut info = latest_sled_agent_info.lock().unwrap();
+            info.reservoir_size = (2 * 1024 * 1024u64).into();
+            info.generation = info.generation.next();
+        }
+        handle.notify_nexus_about_self(log).await;
+        // Wait for a steady state, when the the latest info has been put to nexus.
+        // Ensure the second get request has been sent.
+        let status = wait_for::<_, (), _, _>(
+            || async {
+                let status = handle.get_status().await.unwrap();
+                if !status.has_pending_notification
+                    && status.total_get_requests_started == 2
+                {
+                    Ok(status)
+                } else {
+                    Err(CondCheckError::NotYet)
+                }
+            },
+            &Duration::from_millis(2),
+            &Duration::from_secs(15),
+        )
+        .await
+        .expect("Failed to get status from Nexus");
+
+        // One extra get and put request because of the put error
+        assert_eq!(status.total_get_requests_completed, 2u64);
+
+        assert_eq!(status.total_put_requests_started, 4u64);
+        assert_eq!(status.total_put_requests_completed, 4u64);
+        assert_eq!(status.has_outstanding_request, false);
+        assert_eq!(status.cancelled_pending_notifications, 4);
+        let expected = latest_sled_agent_info.lock().unwrap().clone();
+        assert_eq!(
+            status.nexus_known_info,
+            Some(NexusKnownInfo::Found(expected)),
+        );
+
+        // Inject a get error and trigger a put to nexus after updating the VMM
+        // reservoir size. We shouldn't end up calling Get at all since we know
+        // our state, and so we will not trigger the error here. However, later
+        // on when we trigger a put error, it will perform a get and that will
+        // trigger this injected get errror.
+        get_error.store(true, Ordering::SeqCst);
+        {
+            let mut info = latest_sled_agent_info.lock().unwrap();
+            info.reservoir_size = (3 * 1024 * 1024u64).into();
+            info.generation = info.generation.next();
+        }
+        handle.notify_nexus_about_self(log).await;
+        // Wait for a steady state, when the the latest info has been put to nexus.
+        // Ensure the second get request has been sent.
+        let status = wait_for::<_, (), _, _>(
+            || async {
+                let status = handle.get_status().await.unwrap();
+                if !status.has_pending_notification
+                    && status.total_put_requests_started == 5
+                {
+                    Ok(status)
+                } else {
+                    Err(CondCheckError::NotYet)
+                }
+            },
+            &Duration::from_millis(2),
+            &Duration::from_secs(15),
+        )
+        .await
+        .expect("Failed to get status from Nexus");
+
+        // Get wasn't called
+        assert_eq!(status.total_get_requests_started, 2u64);
+        assert_eq!(status.total_get_requests_completed, 2u64);
+
+        assert_eq!(status.total_put_requests_started, 5u64);
+        assert_eq!(status.total_put_requests_completed, 5u64);
+        assert_eq!(status.has_outstanding_request, false);
+        assert_eq!(status.cancelled_pending_notifications, 5);
+        let expected = latest_sled_agent_info.lock().unwrap().clone();
+        assert_eq!(
+            status.nexus_known_info,
+            Some(NexusKnownInfo::Found(expected)),
+        );
+
+        // Now inject a put error. This will trigger the put error and
+        // previously injected get error.
+        put_error.store(true, Ordering::SeqCst);
+        {
+            let mut info = latest_sled_agent_info.lock().unwrap();
+            info.reservoir_size = (4 * 1024 * 1024u64).into();
+            info.generation = info.generation.next();
+        }
+        handle.notify_nexus_about_self(log).await;
+        // Wait for a steady state, when the the latest info has been put to nexus.
+        // Ensure the second get request has been sent.
+        let status = wait_for::<_, (), _, _>(
+            || async {
+                let status = handle.get_status().await.unwrap();
+                if !status.has_pending_notification
+                    && status.total_put_requests_started > 5
+                {
+                    Ok(status)
+                } else {
+                    Err(CondCheckError::NotYet)
+                }
+            },
+            &Duration::from_millis(2),
+            &Duration::from_secs(15),
+        )
+        .await
+        .expect("Failed to get status from Nexus");
+
+        // Get was called twice, once for error, once for success
+        assert_eq!(status.total_get_requests_started, 4u64);
+        assert_eq!(status.total_get_requests_completed, 4u64);
+
+        // Put was called twice, once for error, once for success
+        assert_eq!(status.total_put_requests_started, 7u64);
+        assert_eq!(status.total_put_requests_completed, 7u64);
+
+        assert_eq!(status.has_outstanding_request, false);
+        assert_eq!(status.cancelled_pending_notifications, 6);
+        let expected = latest_sled_agent_info.lock().unwrap().clone();
+        assert_eq!(
+            status.nexus_known_info,
+            Some(NexusKnownInfo::Found(expected)),
+        );
+
+        logctx.cleanup_successful();
+    }
+}
diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs
index 784e3f4938..dd815775ff 100644
--- a/sled-agent/src/sim/server.rs
+++ b/sled-agent/src/sim/server.rs
@@ -18,6 +18,7 @@ use nexus_client::types::{IpRange, Ipv4Range, Ipv6Range};
 use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use omicron_common::address::DNS_OPTE_IPV4_SUBNET;
 use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET;
+use omicron_common::api::external::Generation;
 use omicron_common::api::external::MacAddr;
 use omicron_common::backoff::{
     retry_notify, retry_policy_internal_service_aggressive, BackoffError,
@@ -101,7 +102,7 @@ impl Server {
                 nexus_client
                     .sled_agent_put(
                         &config.id,
-                        &NexusTypes::SledAgentStartupInfo {
+                        &NexusTypes::SledAgentInfo {
                             sa_address: sa_address.to_string(),
                             role: NexusTypes::SledRole::Scrimlet,
                             baseboard: NexusTypes::Baseboard {
@@ -124,6 +125,8 @@ impl Server {
                                 config.hardware.reservoir_ram,
                             )
                             .unwrap(),
+                            generation: Generation::new(),
+                            decommissioned: false,
                         },
                     )
                     .await
diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs
index 5381337b8c..8f737f879b 100644
--- a/sled-agent/src/sled_agent.rs
+++ b/sled-agent/src/sled_agent.rs
@@ -11,10 +11,13 @@ use crate::bootstrap::early_networking::{
 };
 use crate::bootstrap::params::{BaseboardId, StartSledAgentRequest};
 use crate::config::Config;
-use crate::instance_manager::{InstanceManager, ReservoirMode};
+use crate::instance_manager::InstanceManager;
 use crate::long_running_tasks::LongRunningTaskHandles;
 use crate::metrics::MetricsManager;
-use crate::nexus::{ConvertInto, NexusClientWithResolver, NexusRequestQueue};
+use crate::nexus::{
+    NexusClientWithResolver, NexusNotifierHandle, NexusNotifierInput,
+    NexusNotifierTask,
+};
 use crate::params::{
     DiskStateRequested, InstanceExternalIpBody, InstanceHardware,
     InstanceMetadata, InstanceMigrationSourceParams, InstancePutStateResponse,
@@ -25,6 +28,7 @@ use crate::params::{
 use crate::services::{self, ServiceManager};
 use crate::storage_monitor::UnderlayAccess;
 use crate::updates::{ConfigUpdates, UpdateManager};
+use crate::vmm_reservoir::{ReservoirMode, VmmReservoirManager};
 use crate::zone_bundle;
 use crate::zone_bundle::BundleError;
 use bootstore::schemes::v0 as bootstore;
@@ -57,7 +61,7 @@ use omicron_common::api::{
     internal::nexus::UpdateArtifactId,
 };
 use omicron_common::backoff::{
-    retry_notify, retry_notify_ext, retry_policy_internal_service,
+    retry_notify, retry_policy_internal_service,
     retry_policy_internal_service_aggressive, BackoffError,
 };
 use oximeter::types::ProducerRegistry;
@@ -288,8 +292,8 @@ struct SledAgentInner {
     // Connection to Nexus.
     nexus_client: NexusClientWithResolver,
 
-    // A serialized request queue for operations interacting with Nexus.
-    nexus_request_queue: NexusRequestQueue,
+    // A mechanism for notifiying nexus about sled-agent updates
+    nexus_notifier: NexusNotifierHandle,
 
     // The rack network config provided at RSS time.
     rack_network_config: Option<RackNetworkConfig>,
@@ -466,6 +470,19 @@ impl SledAgent {
             .map_err(|_| ())
             .expect("Failed to send to StorageMonitor");
 
+        // Configure the VMM reservoir as either a percentage of DRAM or as an
+        // exact size in MiB.
+        let reservoir_mode = ReservoirMode::from_config(
+            config.vmm_reservoir_percentage,
+            config.vmm_reservoir_size_mb,
+        );
+
+        let vmm_reservoir_manager = VmmReservoirManager::spawn(
+            &log,
+            long_running_task_handles.hardware_manager.clone(),
+            reservoir_mode,
+        );
+
         let instances = InstanceManager::new(
             parent_log.clone(),
             nexus_client.clone(),
@@ -474,41 +491,9 @@ impl SledAgent {
             storage_manager.clone(),
             long_running_task_handles.zone_bundler.clone(),
             ZoneBuilderFactory::default(),
+            vmm_reservoir_manager.clone(),
         )?;
 
-        // Configure the VMM reservoir as either a percentage of DRAM or as an
-        // exact size in MiB.
-        let reservoir_mode = match (
-            config.vmm_reservoir_percentage,
-            config.vmm_reservoir_size_mb,
-        ) {
-            (None, None) => ReservoirMode::None,
-            (Some(p), None) => ReservoirMode::Percentage(p),
-            (None, Some(mb)) => ReservoirMode::Size(mb),
-            (Some(_), Some(_)) => panic!(
-                "only one of vmm_reservoir_percentage and \
-                vmm_reservoir_size_mb is allowed"
-            ),
-        };
-
-        match reservoir_mode {
-            ReservoirMode::None => warn!(log, "Not using VMM reservoir"),
-            ReservoirMode::Size(0) | ReservoirMode::Percentage(0) => {
-                warn!(log, "Not using VMM reservoir (size 0 bytes requested)")
-            }
-            _ => {
-                instances
-                    .set_reservoir_size(
-                        &long_running_task_handles.hardware_manager,
-                        reservoir_mode,
-                    )
-                    .map_err(|e| {
-                        error!(log, "Failed to setup VMM reservoir: {e}");
-                        e
-                    })?;
-            }
-        }
-
         let update_config = ConfigUpdates {
             zone_artifact_path: Utf8PathBuf::from("/opt/oxide"),
         };
@@ -570,6 +555,22 @@ impl SledAgent {
             rack_network_config.clone(),
         )?;
 
+        // Spawn a background task for managing notifications to nexus
+        // about this sled-agent.
+        let nexus_notifier_input = NexusNotifierInput {
+            sled_id: request.body.id,
+            sled_address: get_sled_address(request.body.subnet),
+            nexus_client: nexus_client.client().clone(),
+            hardware: long_running_task_handles.hardware_manager.clone(),
+            vmm_reservoir_manager: vmm_reservoir_manager.clone(),
+        };
+        let (nexus_notifier_task, nexus_notifier_handle) =
+            NexusNotifierTask::new(nexus_notifier_input, &log);
+
+        tokio::spawn(async move {
+            nexus_notifier_task.run().await;
+        });
+
         let sled_agent = SledAgent {
             inner: Arc::new(SledAgentInner {
                 id: request.body.id,
@@ -582,14 +583,7 @@ impl SledAgent {
                 port_manager,
                 services,
                 nexus_client,
-
-                // TODO(https://github.com/oxidecomputer/omicron/issues/1917):
-                // Propagate usage of this request queue throughout the Sled
-                // Agent.
-                //
-                // Also, we could maybe de-dup some of the backoff code in the
-                // request queue?
-                nexus_request_queue: NexusRequestQueue::new(),
+                nexus_notifier: nexus_notifier_handle,
                 rack_network_config,
                 zone_bundler: long_running_task_handles.zone_bundler.clone(),
                 bootstore: long_running_task_handles.bootstore.clone(),
@@ -603,7 +597,7 @@ impl SledAgent {
         // existence. If inspection of the hardware later informs us that we're
         // actually running on a scrimlet, that's fine, the updated value will
         // be received by Nexus eventually.
-        sled_agent.notify_nexus_about_self(&log);
+        sled_agent.notify_nexus_about_self(&log).await;
 
         Ok(sled_agent)
     }
@@ -681,98 +675,10 @@ impl SledAgent {
         Ok(())
     }
 
-    /// Sends a request to Nexus informing it that the current sled exists,
-    /// with information abou the existing set of hardware.
-    ///
-    /// Does not block until Nexus is available -- the future created by this
-    /// function is retried in a queue that is polled in the background.
-    pub(crate) fn notify_nexus_about_self(&self, log: &Logger) {
-        let sled_id = self.inner.id;
-        let nexus_client = self.inner.nexus_client.clone();
-        let sled_address = self.inner.sled_address();
-        let is_scrimlet = self.inner.hardware.is_scrimlet();
-        let baseboard = self.inner.hardware.baseboard().convert();
-        let usable_hardware_threads =
-            self.inner.hardware.online_processor_count();
-        let usable_physical_ram =
-            self.inner.hardware.usable_physical_ram_bytes();
-        let reservoir_size = self.inner.instances.reservoir_size();
-
-        let log = log.clone();
-        let fut = async move {
-            // Notify the control plane that we're up, and continue trying this
-            // until it succeeds. We retry with a randomized, capped
-            // exponential backoff.
-            //
-            // TODO-robustness if this returns a 400 error, we probably want to
-            // return a permanent error from the `notify_nexus` closure.
-            let notify_nexus = || async {
-                info!(
-                    log,
-                    "contacting server nexus, registering sled";
-                    "id" => ?sled_id,
-                    "baseboard" => ?baseboard,
-                );
-                let role = if is_scrimlet {
-                    nexus_client::types::SledRole::Scrimlet
-                } else {
-                    nexus_client::types::SledRole::Gimlet
-                };
-
-                nexus_client
-                    .client()
-                    .sled_agent_put(
-                        &sled_id,
-                        &nexus_client::types::SledAgentStartupInfo {
-                            sa_address: sled_address.to_string(),
-                            role,
-                            baseboard: baseboard.clone(),
-                            usable_hardware_threads,
-                            usable_physical_ram: nexus_client::types::ByteCount(
-                                usable_physical_ram,
-                            ),
-                            reservoir_size: nexus_client::types::ByteCount(
-                                reservoir_size.to_bytes(),
-                            ),
-                        },
-                    )
-                    .await
-                    .map_err(|err| BackoffError::transient(err.to_string()))
-            };
-            // This notification is often invoked before Nexus has started
-            // running, so avoid flagging any errors as concerning until some
-            // time has passed.
-            let log_notification_failure = |err, call_count, total_duration| {
-                if call_count == 0 {
-                    info!(
-                        log,
-                        "failed to notify nexus about sled agent";
-                        "error" => %err,
-                    );
-                } else if total_duration > std::time::Duration::from_secs(30) {
-                    warn!(
-                        log,
-                        "failed to notify nexus about sled agent";
-                        "error" => %err,
-                        "total duration" => ?total_duration,
-                    );
-                }
-            };
-            retry_notify_ext(
-                retry_policy_internal_service_aggressive(),
-                notify_nexus,
-                log_notification_failure,
-            )
-            .await
-            .expect("Expected an infinite retry loop contacting Nexus");
-        };
-        self.inner
-            .nexus_request_queue
-            .sender()
-            .send(Box::pin(fut))
-            .unwrap_or_else(|err| {
-                panic!("Failed to send future to request queue: {err}");
-            });
+    /// Trigger a request to Nexus informing it that the current sled exists,
+    /// with information about the existing set of hardware.
+    pub(crate) async fn notify_nexus_about_self(&self, log: &Logger) {
+        self.inner.nexus_notifier.notify_nexus_about_self(log).await;
     }
 
     /// List all zone bundles on the system, for any zones live or dead.
diff --git a/sled-agent/src/vmm_reservoir.rs b/sled-agent/src/vmm_reservoir.rs
new file mode 100644
index 0000000000..d7b6b64ecf
--- /dev/null
+++ b/sled-agent/src/vmm_reservoir.rs
@@ -0,0 +1,272 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! A background thread for managing the VMM reservoir
+
+use illumos_utils::vmm_reservoir;
+use omicron_common::api::external::ByteCount;
+use slog::Logger;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+use std::thread;
+use tokio::sync::{broadcast, oneshot};
+
+use sled_hardware::HardwareManager;
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+    #[error("Failed to create reservoir: {0}")]
+    Reservoir(#[from] vmm_reservoir::Error),
+
+    #[error("Invalid reservoir configuration: {0}")]
+    ReservoirConfig(String),
+
+    #[error("VmmReservoirManager is currently busy")]
+    Busy,
+
+    #[error("VmmReservoirManager has shutdown")]
+    Shutdown,
+
+    #[error(
+        "Communication error with VmmReservoirManager: ReplySenderDropped"
+    )]
+    ReplySenderDropped,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum ReservoirMode {
+    Size(u32),
+    Percentage(u8),
+}
+
+impl ReservoirMode {
+    /// Return a configuration of the VMM reservoir as either a percentage of
+    /// DRAM or as an exact size in MiB.
+    ///
+    /// Panic upon invalid configuration
+    pub fn from_config(
+        percentage: Option<u8>,
+        size_mb: Option<u32>,
+    ) -> Option<ReservoirMode> {
+        match (percentage, size_mb) {
+            (None, None) => None,
+            (Some(p), None) => Some(ReservoirMode::Percentage(p)),
+            (None, Some(mb)) => Some(ReservoirMode::Size(mb)),
+            (Some(_), Some(_)) => panic!(
+                "only one of vmm_reservoir_percentage and \
+                vmm_reservoir_size_mb is allowed"
+            ),
+        }
+    }
+}
+
+/// A message sent from [`VmmReservoirManagerHandle`] to [`VmmReservoirManager`]
+enum ReservoirManagerMsg {
+    SetReservoirSize {
+        mode: ReservoirMode,
+        reply_tx: oneshot::Sender<Result<(), Error>>,
+    },
+}
+
+#[derive(Clone)]
+/// A mechanism to interact with the [`VmmReservoirManager`]
+pub struct VmmReservoirManagerHandle {
+    reservoir_size: Arc<AtomicU64>,
+    tx: flume::Sender<ReservoirManagerMsg>,
+    // A notification channel indicating that the size of the VMM reservoir has
+    // changed. We use a broadcast channel instead of a `Notify` to prevent lost
+    // updates with multiple receivers. Importantly, a `RecvError::Lagged` is
+    // just as valuable as an `Ok(())`, and so this acts as a pure notification
+    // channel.
+    size_updated_tx: broadcast::Sender<()>,
+    _manager_handle: Arc<thread::JoinHandle<()>>,
+}
+
+impl VmmReservoirManagerHandle {
+    /// Returns the last-set size of the reservoir
+    pub fn reservoir_size(&self) -> ByteCount {
+        self.reservoir_size.load(Ordering::SeqCst).try_into().unwrap()
+    }
+
+    // Return the receiver that notifies us about a size update. The value
+    // itself is held in an atomic. While we could have replaced both of these
+    // with a `watch`, that is not the semantics all callers want. Sometimes
+    // they just want to "read" the latest size of the reservoir.
+    pub fn subscribe_for_size_updates(&self) -> broadcast::Receiver<()> {
+        self.size_updated_tx.subscribe()
+    }
+
+    /// Tell the [`VmmReservoirManager`] to set the reservoir size and wait for
+    /// a response.
+    ///
+    /// Returns an error if the allocation fails or if the manager is currently
+    /// busy handling another request.
+    //
+    // It's anticipated this will be used to change the reservation
+    #[allow(unused)]
+    pub async fn set_reservoir_size(
+        &self,
+        mode: ReservoirMode,
+    ) -> Result<(), Error> {
+        let (tx, rx) = oneshot::channel();
+        let msg = ReservoirManagerMsg::SetReservoirSize { mode, reply_tx: tx };
+        if let Err(e) = self.tx.try_send(msg) {
+            return Err(match e {
+                flume::TrySendError::Full(e) => Error::Busy,
+                flume::TrySendError::Disconnected(e) => Error::Shutdown,
+            });
+        }
+        rx.await.map_err(|_| Error::ReplySenderDropped)?
+    }
+}
+
+/// Manage the VMM reservoir in a background thread
+pub struct VmmReservoirManager {
+    reservoir_size: Arc<AtomicU64>,
+    rx: flume::Receiver<ReservoirManagerMsg>,
+    size_updated_tx: broadcast::Sender<()>,
+    // We maintain a copy of the receiver so sends never fail.
+    _size_updated_rx: broadcast::Receiver<()>,
+    log: Logger,
+}
+
+impl VmmReservoirManager {
+    pub fn spawn(
+        log: &Logger,
+        hardware_manager: HardwareManager,
+        reservoir_mode: Option<ReservoirMode>,
+    ) -> VmmReservoirManagerHandle {
+        let log = log.new(o!("component" => "VmmReservoirManager"));
+        let (size_updated_tx, _size_updated_rx) = broadcast::channel(1);
+        // We use a rendevous channel to only allow one request at a time.
+        // Resizing a reservoir may block the thread for up to two minutes, so
+        // we want to ensure it is complete before allowing another call.
+        let (tx, rx) = flume::bounded(0);
+        let reservoir_size = Arc::new(AtomicU64::new(0));
+        let manager = VmmReservoirManager {
+            reservoir_size: reservoir_size.clone(),
+            size_updated_tx: size_updated_tx.clone(),
+            _size_updated_rx,
+            rx,
+            log,
+        };
+        let _manager_handle = Arc::new(thread::spawn(move || {
+            manager.run(hardware_manager, reservoir_mode)
+        }));
+        VmmReservoirManagerHandle {
+            reservoir_size,
+            tx,
+            size_updated_tx,
+            _manager_handle,
+        }
+    }
+
+    fn run(
+        self,
+        hardware_manager: HardwareManager,
+        reservoir_mode: Option<ReservoirMode>,
+    ) {
+        match reservoir_mode {
+            None => warn!(self.log, "Not using VMM reservoir"),
+            Some(ReservoirMode::Size(0))
+            | Some(ReservoirMode::Percentage(0)) => {
+                warn!(
+                    self.log,
+                    "Not using VMM reservoir (size 0 bytes requested)"
+                )
+            }
+            Some(mode) => {
+                if let Err(e) = self.set_reservoir_size(&hardware_manager, mode)
+                {
+                    error!(self.log, "Failed to setup VMM reservoir: {e}");
+                }
+            }
+        }
+
+        while let Ok(msg) = self.rx.recv() {
+            let ReservoirManagerMsg::SetReservoirSize { mode, reply_tx } = msg;
+            match self.set_reservoir_size(&hardware_manager, mode) {
+                Ok(()) => {
+                    let _ = reply_tx.send(Ok(()));
+                }
+                Err(e) => {
+                    error!(self.log, "Failed to setup VMM reservoir: {e}");
+                    let _ = reply_tx.send(Err(e));
+                }
+            }
+        }
+    }
+
+    /// Sets the VMM reservoir to the requested percentage of usable physical
+    /// RAM or to a size in MiB. Either mode will round down to the nearest
+    /// aligned size required by the control plane.
+    fn set_reservoir_size(
+        &self,
+        hardware: &sled_hardware::HardwareManager,
+        mode: ReservoirMode,
+    ) -> Result<(), Error> {
+        let hardware_physical_ram_bytes = hardware.usable_physical_ram_bytes();
+        let req_bytes = match mode {
+            ReservoirMode::Size(mb) => {
+                let bytes = ByteCount::from_mebibytes_u32(mb).to_bytes();
+                if bytes > hardware_physical_ram_bytes {
+                    return Err(Error::ReservoirConfig(format!(
+                        "cannot specify a reservoir of {bytes} bytes when \
+                        physical memory is {hardware_physical_ram_bytes} bytes",
+                    )));
+                }
+                bytes
+            }
+            ReservoirMode::Percentage(percent) => {
+                if !matches!(percent, 1..=99) {
+                    return Err(Error::ReservoirConfig(format!(
+                        "VMM reservoir percentage of {} must be between 0 and \
+                        100",
+                        percent
+                    )));
+                };
+                (hardware_physical_ram_bytes as f64 * (percent as f64 / 100.0))
+                    .floor() as u64
+            }
+        };
+
+        let req_bytes_aligned = vmm_reservoir::align_reservoir_size(req_bytes);
+
+        if req_bytes_aligned == 0 {
+            warn!(
+                self.log,
+                "Requested reservoir size of {} bytes < minimum aligned size \
+                of {} bytes",
+                req_bytes,
+                vmm_reservoir::RESERVOIR_SZ_ALIGN
+            );
+            return Ok(());
+        }
+
+        // The max ByteCount value is i64::MAX, which is ~8 million TiB.
+        // As this value is either a percentage of DRAM or a size in MiB
+        // represented as a u32, constructing this should always work.
+        let reservoir_size = ByteCount::try_from(req_bytes_aligned).unwrap();
+        if let ReservoirMode::Percentage(percent) = mode {
+            info!(
+                self.log,
+                "{}% of {} physical ram = {} bytes)",
+                percent,
+                hardware_physical_ram_bytes,
+                req_bytes,
+            );
+        }
+        info!(self.log, "Setting reservoir size to {reservoir_size} bytes");
+        vmm_reservoir::ReservoirControl::set(reservoir_size)?;
+
+        self.reservoir_size.store(reservoir_size.to_bytes(), Ordering::SeqCst);
+        info!(
+            self.log,
+            "Finished setting reservoir size to {reservoir_size} bytes"
+        );
+        self.size_updated_tx.send(()).unwrap();
+
+        Ok(())
+    }
+}

From 8697f39aac3323ffda3f33f4b5aa44fd7ad857e3 Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Fri, 8 Mar 2024 14:38:35 -0500
Subject: [PATCH 093/157] Teach `omdb db network list-eips` and
 `DataStore::vpc_resolve_to_sleds` about blueprints (#5202)

While working on #4886, I ran into a couple places that expect all
services to have rows in the `service` table, but Reconfigurator does
not add new rows to `service` when it spins up new services (see #4947
for rationale). This PR teaches these two users of `service` to _also_
check the current target blueprint when checking for services.

I also removed `LookupPath::service_id` and `lookup_resource! { ...
Service ... }`; the only user of these was `omdb`, and I think their
existence is misleading in the post-`service`-table world we're moving
toward.
---
 dev-tools/omdb/src/bin/omdb/db.rs             |  99 +++-
 nexus/db-model/src/inventory.rs               |  21 +-
 nexus/db-model/src/schema.rs                  |   2 +
 .../db-queries/src/db/datastore/deployment.rs |   6 +-
 nexus/db-queries/src/db/datastore/vpc.rs      | 450 +++++++++++++++++-
 nexus/db-queries/src/db/lookup.rs             |  14 -
 6 files changed, 550 insertions(+), 42 deletions(-)

diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs
index 02b68144df..339c257d8e 100644
--- a/dev-tools/omdb/src/bin/omdb/db.rs
+++ b/dev-tools/omdb/src/bin/omdb/db.rs
@@ -80,6 +80,7 @@ use nexus_db_queries::db::DataStore;
 use nexus_reconfigurator_preparation::policy_from_db;
 use nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL;
 use nexus_types::deployment::Blueprint;
+use nexus_types::deployment::OmicronZoneType;
 use nexus_types::deployment::UnstableReconfiguratorState;
 use nexus_types::identity::Resource;
 use nexus_types::internal_api::params::DnsRecord;
@@ -628,7 +629,7 @@ fn first_page<'a, T>(limit: NonZeroU32) -> DataPageParams<'a, T> {
     }
 }
 
-/// Helper function to looks up an instance with the given ID.
+/// Helper function to look up an instance with the given ID.
 async fn lookup_instance(
     datastore: &DataStore,
     instance_id: Uuid,
@@ -646,7 +647,71 @@ async fn lookup_instance(
         .with_context(|| format!("loading instance {instance_id}"))
 }
 
-/// Helper function to looks up a project with the given ID.
+/// Helper function to look up the kind of the service with the given ID.
+///
+/// Requires the caller to first have fetched the current target blueprint, so
+/// we can find services that have been added by Reconfigurator.
+async fn lookup_service_kind(
+    datastore: &DataStore,
+    service_id: Uuid,
+    current_target_blueprint: Option<&Blueprint>,
+) -> anyhow::Result<Option<ServiceKind>> {
+    let conn = datastore.pool_connection_for_tests().await?;
+
+    // We need to check the `service` table (populated during rack setup)...
+    {
+        use db::schema::service::dsl;
+        if let Some(kind) = dsl::service
+            .filter(dsl::id.eq(service_id))
+            .limit(1)
+            .select(dsl::kind)
+            .get_result_async(&*conn)
+            .await
+            .optional()
+            .with_context(|| format!("loading service {service_id}"))?
+        {
+            return Ok(Some(kind));
+        }
+    }
+
+    // ...and if we don't find the service, check the latest blueprint, because
+    // the service might have been added by Reconfigurator after RSS ran.
+    let Some(blueprint) = current_target_blueprint else {
+        return Ok(None);
+    };
+
+    let Some(zone_config) =
+        blueprint.all_omicron_zones().find_map(|(_sled_id, zone_config)| {
+            if zone_config.id == service_id {
+                Some(zone_config)
+            } else {
+                None
+            }
+        })
+    else {
+        return Ok(None);
+    };
+
+    let service_kind = match &zone_config.zone_type {
+        OmicronZoneType::BoundaryNtp { .. }
+        | OmicronZoneType::InternalNtp { .. } => ServiceKind::Ntp,
+        OmicronZoneType::Clickhouse { .. } => ServiceKind::Clickhouse,
+        OmicronZoneType::ClickhouseKeeper { .. } => {
+            ServiceKind::ClickhouseKeeper
+        }
+        OmicronZoneType::CockroachDb { .. } => ServiceKind::Cockroach,
+        OmicronZoneType::Crucible { .. } => ServiceKind::Crucible,
+        OmicronZoneType::CruciblePantry { .. } => ServiceKind::CruciblePantry,
+        OmicronZoneType::ExternalDns { .. } => ServiceKind::ExternalDns,
+        OmicronZoneType::InternalDns { .. } => ServiceKind::InternalDns,
+        OmicronZoneType::Nexus { .. } => ServiceKind::Nexus,
+        OmicronZoneType::Oximeter { .. } => ServiceKind::Oximeter,
+    };
+
+    Ok(Some(service_kind))
+}
+
+/// Helper function to look up a project with the given ID.
 async fn lookup_project(
     datastore: &DataStore,
     project_id: Uuid,
@@ -1886,26 +1951,26 @@ async fn cmd_db_eips(
 
     let mut rows = Vec::new();
 
+    let current_target_blueprint = datastore
+        .blueprint_target_get_current_full(opctx)
+        .await
+        .context("loading current target blueprint")?
+        .map(|(_, blueprint)| blueprint);
+
     for ip in &ips {
         let owner = if let Some(owner_id) = ip.parent_id {
             if ip.is_service {
-                let service = match LookupPath::new(opctx, datastore)
-                    .service_id(owner_id)
-                    .fetch()
-                    .await
+                let kind = match lookup_service_kind(
+                    datastore,
+                    owner_id,
+                    current_target_blueprint.as_ref(),
+                )
+                .await?
                 {
-                    Ok(instance) => instance,
-                    Err(e) => {
-                        eprintln!(
-                            "error looking up service with id {owner_id}: {e}"
-                        );
-                        continue;
-                    }
+                    Some(kind) => format!("{kind:?}"),
+                    None => "UNKNOWN (service ID not found)".to_string(),
                 };
-                Owner::Service {
-                    id: owner_id,
-                    kind: format!("{:?}", service.1.kind),
-                }
+                Owner::Service { id: owner_id, kind }
             } else {
                 let instance =
                     match lookup_instance(datastore, owner_id).await? {
diff --git a/nexus/db-model/src/inventory.rs b/nexus/db-model/src/inventory.rs
index d8314f97b8..0992eb60b5 100644
--- a/nexus/db-model/src/inventory.rs
+++ b/nexus/db-model/src/inventory.rs
@@ -12,8 +12,8 @@ use crate::schema::{
     inv_sled_omicron_zones, sw_caboose, sw_root_of_trust_page,
 };
 use crate::{
-    impl_enum_type, ipv6, ByteCount, Generation, MacAddr, Name, SqlU16, SqlU32,
-    SqlU8,
+    impl_enum_type, ipv6, ByteCount, Generation, MacAddr, Name, ServiceKind,
+    SqlU16, SqlU32, SqlU8,
 };
 use anyhow::anyhow;
 use chrono::DateTime;
@@ -715,6 +715,23 @@ impl_enum_type!(
     Oximeter => b"oximeter"
 );
 
+impl From<ZoneType> for ServiceKind {
+    fn from(zone_type: ZoneType) -> Self {
+        match zone_type {
+            ZoneType::BoundaryNtp | ZoneType::InternalNtp => Self::Ntp,
+            ZoneType::Clickhouse => Self::Clickhouse,
+            ZoneType::ClickhouseKeeper => Self::ClickhouseKeeper,
+            ZoneType::CockroachDb => Self::Cockroach,
+            ZoneType::Crucible => Self::Crucible,
+            ZoneType::CruciblePantry => Self::CruciblePantry,
+            ZoneType::ExternalDns => Self::ExternalDns,
+            ZoneType::InternalDns => Self::InternalDns,
+            ZoneType::Nexus => Self::Nexus,
+            ZoneType::Oximeter => Self::Oximeter,
+        }
+    }
+}
+
 /// See [`nexus_types::inventory::OmicronZoneConfig`].
 #[derive(Queryable, Clone, Debug, Selectable, Insertable)]
 #[diesel(table_name = inv_omicron_zone)]
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index d70b0a267e..09bc963936 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -1550,6 +1550,8 @@ allow_tables_to_appear_in_same_query!(
 allow_tables_to_appear_in_same_query!(hw_baseboard_id, inv_sled_agent,);
 
 allow_tables_to_appear_in_same_query!(
+    bp_omicron_zone,
+    bp_target,
     dataset,
     disk,
     image,
diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs
index 1dae1030f0..5fc17c88d0 100644
--- a/nexus/db-queries/src/db/datastore/deployment.rs
+++ b/nexus/db-queries/src/db/datastore/deployment.rs
@@ -1201,7 +1201,7 @@ mod tests {
     #[tokio::test]
     async fn test_empty_blueprint() {
         // Setup
-        let logctx = dev::test_setup_log("inventory_insert");
+        let logctx = dev::test_setup_log("test_empty_blueprint");
         let mut db = test_setup_database(&logctx.log).await;
         let (opctx, datastore) = datastore_test(&logctx, &db).await;
 
@@ -1264,7 +1264,7 @@ mod tests {
     #[tokio::test]
     async fn test_representative_blueprint() {
         // Setup
-        let logctx = dev::test_setup_log("inventory_insert");
+        let logctx = dev::test_setup_log("test_representative_blueprint");
         let mut db = test_setup_database(&logctx.log).await;
         let (opctx, datastore) = datastore_test(&logctx, &db).await;
 
@@ -1448,7 +1448,7 @@ mod tests {
     #[tokio::test]
     async fn test_set_target() {
         // Setup
-        let logctx = dev::test_setup_log("inventory_insert");
+        let logctx = dev::test_setup_log("test_set_target");
         let mut db = test_setup_database(&logctx.log).await;
         let (opctx, datastore) = datastore_test(&logctx, &db).await;
 
diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs
index 4626301f76..cd972c0941 100644
--- a/nexus/db-queries/src/db/datastore/vpc.rs
+++ b/nexus/db-queries/src/db/datastore/vpc.rs
@@ -640,8 +640,8 @@ impl DataStore {
             })
     }
 
-    /// Return the list of `Sled`s hosting instances with network interfaces
-    /// on the provided VPC.
+    /// Return the list of `Sled`s hosting instances or control plane services
+    /// with network interfaces on the provided VPC.
     pub async fn vpc_resolve_to_sleds(
         &self,
         vpc_id: Uuid,
@@ -650,9 +650,15 @@ impl DataStore {
         // Resolve each VNIC in the VPC to the Sled it's on, so we know which
         // Sleds to notify when firewall rules change.
         use db::schema::{
-            instance, instance_network_interface, service,
-            service_network_interface, sled, vmm,
+            bp_omicron_zone, bp_target, instance, instance_network_interface,
+            service, service_network_interface, sled, vmm,
         };
+        // Diesel requires us to use aliases in order to refer to the
+        // `bp_target` table twice in the same query.
+        let (bp_target1, bp_target2) = diesel::alias!(
+            db::schema::bp_target as bp_target1,
+            db::schema::bp_target as bp_target2
+        );
 
         let instance_query = instance_network_interface::table
             .inner_join(
@@ -671,7 +677,12 @@ impl DataStore {
             .filter(vmm::time_deleted.is_null())
             .select(Sled::as_select());
 
-        let service_query = service_network_interface::table
+        // When Nexus accepts the rack initialization handoff from RSS, it
+        // populates the `service` table. We eventually want to retire it
+        // (https://github.com/oxidecomputer/omicron/issues/4947), and the
+        // Reconfigurator does not add new entries to it. We still need to query
+        // it for systems that are not yet under Reconfigurator control...
+        let rss_service_query = service_network_interface::table
             .inner_join(
                 service::table
                     .on(service::id.eq(service_network_interface::service_id)),
@@ -681,6 +692,37 @@ impl DataStore {
             .filter(service_network_interface::time_deleted.is_null())
             .select(Sled::as_select());
 
+        // ... and we also need to query for the current target blueprint to
+        // support systems that _are_ under Reconfigurator control.
+        let reconfig_service_query = service_network_interface::table
+            .inner_join(bp_omicron_zone::table.on(
+                bp_omicron_zone::id.eq(service_network_interface::service_id),
+            ))
+            .inner_join(
+                bp_target1.on(bp_omicron_zone::blueprint_id
+                    .eq(bp_target1.field(bp_target::blueprint_id))),
+            )
+            .inner_join(sled::table.on(sled::id.eq(bp_omicron_zone::sled_id)))
+            .filter(
+                // This filters us down to the one current target blueprint (if
+                // it exists); i.e., the target with the maximal version. We
+                // could also check that the current target is `enabled`, but
+                // that could very easily be incorrect: if the current target
+                // or any of its blueprint ancestors were _ever_ enabled, it's
+                // possible the current target blueprint describes running
+                // services that were added after RSS and therefore wouldn't be
+                // seen in `rss_service_query`.
+                bp_target1.field(bp_target::version).eq_any(
+                    bp_target2
+                        .select(bp_target2.field(bp_target::version))
+                        .order_by(bp_target2.field(bp_target::version).desc())
+                        .limit(1),
+                ),
+            )
+            .filter(service_network_interface::vpc_id.eq(vpc_id))
+            .filter(service_network_interface::time_deleted.is_null())
+            .select(Sled::as_select());
+
         let mut sleds = sled::table
             .select(Sled::as_select())
             .filter(sled::time_deleted.is_null())
@@ -691,7 +733,11 @@ impl DataStore {
 
         let conn = self.pool_connection_unauthorized().await?;
         sleds
-            .intersect(instance_query.union(service_query))
+            .intersect(
+                instance_query
+                    .union(rss_service_query)
+                    .union(reconfig_service_query),
+            )
             .get_results_async(&*conn)
             .await
             .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
@@ -1171,14 +1217,35 @@ impl DataStore {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::db::datastore::test::sled_baseboard_for_test;
+    use crate::db::datastore::test::sled_system_hardware_for_test;
     use crate::db::datastore::test_utils::datastore_test;
+    use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET;
     use crate::db::model::Project;
     use crate::db::queries::vpc::MAX_VNI_SEARCH_RANGE_SIZE;
+    use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
+    use nexus_db_model::SledUpdate;
     use nexus_test_utils::db::test_setup_database;
+    use nexus_types::deployment::Blueprint;
+    use nexus_types::deployment::BlueprintTarget;
+    use nexus_types::deployment::NetworkInterface;
+    use nexus_types::deployment::NetworkInterfaceKind;
+    use nexus_types::deployment::OmicronZoneConfig;
+    use nexus_types::deployment::OmicronZoneType;
+    use nexus_types::deployment::OmicronZonesConfig;
     use nexus_types::external_api::params;
+    use nexus_types::identity::Asset;
+    use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET;
     use omicron_common::api::external;
+    use omicron_common::api::external::Generation;
+    use omicron_common::api::external::IpNet;
+    use omicron_common::api::external::MacAddr;
+    use omicron_common::api::external::Vni;
     use omicron_test_utils::dev;
     use slog::info;
+    use std::collections::BTreeMap;
+    use std::collections::BTreeSet;
+    use std::net::IpAddr;
 
     // Test that we detect the right error condition and return None when we
     // fail to insert a VPC due to VNI exhaustion.
@@ -1397,4 +1464,375 @@ mod tests {
         db.cleanup().await.unwrap();
         logctx.cleanup_successful();
     }
+
+    #[derive(Debug)]
+    struct Harness {
+        rack_id: Uuid,
+        sled_ids: Vec<Uuid>,
+        nexuses: Vec<HarnessNexus>,
+    }
+
+    #[derive(Debug)]
+    struct HarnessNexus {
+        id: Uuid,
+        ip: IpAddr,
+        mac: MacAddr,
+        nic_id: Uuid,
+    }
+
+    impl Harness {
+        fn new(num_sleds: usize) -> Self {
+            let mut sled_ids =
+                (0..num_sleds).map(|_| Uuid::new_v4()).collect::<Vec<_>>();
+            sled_ids.sort();
+
+            let mut nexus_ips = NEXUS_OPTE_IPV4_SUBNET
+                .iter()
+                .skip(NUM_INITIAL_RESERVED_IP_ADDRESSES)
+                .map(IpAddr::from);
+            let mut nexus_macs = MacAddr::iter_system();
+            let nexuses = (0..num_sleds)
+                .map(|_| HarnessNexus {
+                    id: Uuid::new_v4(),
+                    ip: nexus_ips.next().unwrap(),
+                    mac: nexus_macs.next().unwrap(),
+                    nic_id: Uuid::new_v4(),
+                })
+                .collect::<Vec<_>>();
+            Self { rack_id: Uuid::new_v4(), sled_ids, nexuses }
+        }
+
+        fn db_sleds(&self) -> impl Iterator<Item = SledUpdate> + '_ {
+            self.sled_ids.iter().copied().map(|sled_id| {
+                SledUpdate::new(
+                    sled_id,
+                    "[::1]:0".parse().unwrap(),
+                    sled_baseboard_for_test(),
+                    sled_system_hardware_for_test(),
+                    self.rack_id,
+                    Generation::new().into(),
+                )
+            })
+        }
+
+        fn db_services(
+            &self,
+        ) -> impl Iterator<
+            Item = (db::model::Service, db::model::IncompleteNetworkInterface),
+        > + '_ {
+            self.sled_ids.iter().zip(&self.nexuses).map(|(sled_id, nexus)| {
+                let service = db::model::Service::new(
+                    nexus.id,
+                    *sled_id,
+                    Some(nexus.id),
+                    "[::1]:0".parse().unwrap(),
+                    db::model::ServiceKind::Nexus,
+                );
+                let name = format!("test-nexus-{}", nexus.id);
+                let nic = db::model::IncompleteNetworkInterface::new_service(
+                    nexus.nic_id,
+                    nexus.id,
+                    NEXUS_VPC_SUBNET.clone(),
+                    IdentityMetadataCreateParams {
+                        name: name.parse().unwrap(),
+                        description: name,
+                    },
+                    nexus.ip,
+                    nexus.mac,
+                    0,
+                )
+                .expect("failed to create incomplete Nexus NIC");
+                (service, nic)
+            })
+        }
+
+        fn omicron_zone_configs(
+            &self,
+        ) -> impl Iterator<Item = (Uuid, OmicronZoneConfig)> + '_ {
+            self.db_services().map(|(service, nic)| {
+                let zone_config = OmicronZoneConfig {
+                    id: service.id(),
+                    underlay_address: "::1".parse().unwrap(),
+                    zone_type: OmicronZoneType::Nexus {
+                        internal_address: "[::1]:0".to_string(),
+                        external_ip: "::1".parse().unwrap(),
+                        nic: NetworkInterface {
+                            id: nic.identity.id,
+                            kind: NetworkInterfaceKind::Service(service.id()),
+                            name: format!("test-nic-{}", nic.identity.id)
+                                .parse()
+                                .unwrap(),
+                            ip: nic.ip.unwrap(),
+                            mac: nic.mac.unwrap(),
+                            subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET).into(),
+                            vni: Vni::SERVICES_VNI,
+                            primary: true,
+                            slot: nic.slot.unwrap(),
+                        },
+                        external_tls: false,
+                        external_dns_servers: Vec::new(),
+                    },
+                };
+                (service.sled_id, zone_config)
+            })
+        }
+    }
+
+    #[tokio::test]
+    async fn test_vpc_resolve_to_sleds_uses_current_target_blueprint() {
+        // Test setup.
+        usdt::register_probes().unwrap();
+        let logctx = dev::test_setup_log(
+            "test_vpc_resolve_to_sleds_uses_current_target_blueprint",
+        );
+        let mut db = test_setup_database(&logctx.log).await;
+        let (opctx, datastore) = datastore_test(&logctx, &db).await;
+
+        // Helper function to fetch and sort the IDs of sleds we've resolved the
+        // SERVICES_VPC_ID to.
+        let fetch_service_sled_ids = || async {
+            let mut service_sled_ids = datastore
+                .vpc_resolve_to_sleds(*SERVICES_VPC_ID, &[])
+                .await
+                .expect("failed to resolve to sleds")
+                .into_iter()
+                .map(|sled| sled.id())
+                .collect::<Vec<_>>();
+            service_sled_ids.sort();
+            service_sled_ids
+        };
+
+        // Create four sleds.
+        let harness = Harness::new(4);
+        for sled in harness.db_sleds() {
+            datastore.sled_upsert(sled).await.expect("failed to upsert sled");
+        }
+
+        // Insert two Nexus records into `service`, emulating RSS.
+        for (service, nic) in harness.db_services().take(2) {
+            datastore
+                .service_upsert(&opctx, service)
+                .await
+                .expect("failed to insert RSS-like service");
+            datastore
+                .service_create_network_interface_raw(&opctx, nic)
+                .await
+                .expect("failed to insert Nexus NIC");
+        }
+
+        // Ensure we find the two sleds we expect after adding Nexus records.
+        assert_eq!(&harness.sled_ids[..2], fetch_service_sled_ids().await);
+
+        // Create a blueprint that has a Nexus on our third sled. (This
+        // blueprint is completely invalid in many ways, but all we care about
+        // here is inserting relevant records in `bp_omicron_zone`.)
+        let bp1_omicron_zones = {
+            let (sled_id, zone_config) = harness
+                .omicron_zone_configs()
+                .nth(2)
+                .expect("fewer than 3 services in test harness");
+            let mut zones = BTreeMap::new();
+            zones.insert(
+                sled_id,
+                OmicronZonesConfig {
+                    generation: Generation::new(),
+                    zones: vec![zone_config],
+                },
+            );
+            zones
+        };
+        let bp1_id = Uuid::new_v4();
+        let bp1 = Blueprint {
+            id: bp1_id,
+            omicron_zones: bp1_omicron_zones,
+            zones_in_service: BTreeSet::new(),
+            parent_blueprint_id: None,
+            internal_dns_version: Generation::new(),
+            time_created: Utc::now(),
+            creator: "test".to_string(),
+            comment: "test".to_string(),
+        };
+        datastore
+            .blueprint_insert(&opctx, &bp1)
+            .await
+            .expect("failed to insert blueprint");
+
+        // We haven't set a blueprint target yet, so we should still only see
+        // the two RSS-inserted service-running sleds.
+        assert_eq!(&harness.sled_ids[..2], fetch_service_sled_ids().await);
+
+        // Make bp1 the current target.
+        datastore
+            .blueprint_target_set_current(
+                &opctx,
+                BlueprintTarget {
+                    target_id: bp1_id,
+                    enabled: true,
+                    time_made_target: Utc::now(),
+                },
+            )
+            .await
+            .expect("failed to set blueprint target");
+
+        // bp1 is the target, but we haven't yet inserted a vNIC record, so
+        // we'll still only see the original 2 sleds.
+        assert_eq!(&harness.sled_ids[..2], fetch_service_sled_ids().await);
+
+        // Insert the relevant service NIC record (normally performed by the
+        // reconfigurator's executor).
+        datastore
+            .service_create_network_interface_raw(
+                &opctx,
+                harness.db_services().nth(2).unwrap().1,
+            )
+            .await
+            .expect("failed to insert service VNIC");
+
+        // We should now see _three_ sleds running services.
+        assert_eq!(&harness.sled_ids[..3], fetch_service_sled_ids().await);
+
+        // Create another blueprint with no services and make it the target.
+        let bp2_id = Uuid::new_v4();
+        let bp2 = Blueprint {
+            id: bp2_id,
+            omicron_zones: BTreeMap::new(),
+            zones_in_service: BTreeSet::new(),
+            parent_blueprint_id: Some(bp1_id),
+            internal_dns_version: Generation::new(),
+            time_created: Utc::now(),
+            creator: "test".to_string(),
+            comment: "test".to_string(),
+        };
+        datastore
+            .blueprint_insert(&opctx, &bp2)
+            .await
+            .expect("failed to insert blueprint");
+        datastore
+            .blueprint_target_set_current(
+                &opctx,
+                BlueprintTarget {
+                    target_id: bp2_id,
+                    enabled: true,
+                    time_made_target: Utc::now(),
+                },
+            )
+            .await
+            .expect("failed to set blueprint target");
+
+        // We haven't removed the service NIC record, but we should no longer
+        // see the third sled here, because we should be back to just the
+        // original two services in the `service` table.
+        assert_eq!(&harness.sled_ids[..2], fetch_service_sled_ids().await);
+
+        // Insert a service NIC record for our fourth sled's Nexus. This
+        // shouldn't change our VPC resolution.
+        datastore
+            .service_create_network_interface_raw(
+                &opctx,
+                harness.db_services().nth(3).unwrap().1,
+            )
+            .await
+            .expect("failed to insert service VNIC");
+        assert_eq!(&harness.sled_ids[..2], fetch_service_sled_ids().await);
+
+        // Create a blueprint that has a Nexus on our fourth sled. This
+        // shouldn't change our VPC resolution.
+        let bp3_omicron_zones = {
+            let (sled_id, zone_config) = harness
+                .omicron_zone_configs()
+                .nth(3)
+                .expect("fewer than 3 services in test harness");
+            let mut zones = BTreeMap::new();
+            zones.insert(
+                sled_id,
+                OmicronZonesConfig {
+                    generation: Generation::new(),
+                    zones: vec![zone_config],
+                },
+            );
+            zones
+        };
+        let bp3_id = Uuid::new_v4();
+        let bp3 = Blueprint {
+            id: bp3_id,
+            omicron_zones: bp3_omicron_zones,
+            zones_in_service: BTreeSet::new(),
+            parent_blueprint_id: Some(bp2_id),
+            internal_dns_version: Generation::new(),
+            time_created: Utc::now(),
+            creator: "test".to_string(),
+            comment: "test".to_string(),
+        };
+        datastore
+            .blueprint_insert(&opctx, &bp3)
+            .await
+            .expect("failed to insert blueprint");
+        assert_eq!(&harness.sled_ids[..2], fetch_service_sled_ids().await);
+
+        // Make this blueprint the target. We've already created the service
+        // VNIC, so we should immediately see our fourth sled in VPC resolution.
+        datastore
+            .blueprint_target_set_current(
+                &opctx,
+                BlueprintTarget {
+                    target_id: bp3_id,
+                    enabled: true,
+                    time_made_target: Utc::now(),
+                },
+            )
+            .await
+            .expect("failed to set blueprint target");
+        assert_eq!(
+            &[harness.sled_ids[0], harness.sled_ids[1], harness.sled_ids[3]]
+                as &[Uuid],
+            fetch_service_sled_ids().await
+        );
+
+        // Finally, create a blueprint that includes our third and fourth sleds,
+        // make it the target, and ensure we resolve to all four sleds.
+        let bp4_omicron_zones = {
+            let mut zones = BTreeMap::new();
+            for (sled_id, zone_config) in harness.omicron_zone_configs().skip(2)
+            {
+                zones.insert(
+                    sled_id,
+                    OmicronZonesConfig {
+                        generation: Generation::new(),
+                        zones: vec![zone_config],
+                    },
+                );
+            }
+            zones
+        };
+        let bp4_id = Uuid::new_v4();
+        let bp4 = Blueprint {
+            id: bp4_id,
+            omicron_zones: bp4_omicron_zones,
+            zones_in_service: BTreeSet::new(),
+            parent_blueprint_id: Some(bp3_id),
+            internal_dns_version: Generation::new(),
+            time_created: Utc::now(),
+            creator: "test".to_string(),
+            comment: "test".to_string(),
+        };
+        datastore
+            .blueprint_insert(&opctx, &bp4)
+            .await
+            .expect("failed to insert blueprint");
+        datastore
+            .blueprint_target_set_current(
+                &opctx,
+                BlueprintTarget {
+                    target_id: bp4_id,
+                    enabled: true,
+                    time_made_target: Utc::now(),
+                },
+            )
+            .await
+            .expect("failed to set blueprint target");
+        assert_eq!(harness.sled_ids, fetch_service_sled_ids().await);
+
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
 }
diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs
index 050c1dcfe9..380c9db140 100644
--- a/nexus/db-queries/src/db/lookup.rs
+++ b/nexus/db-queries/src/db/lookup.rs
@@ -358,11 +358,6 @@ impl<'a> LookupPath<'a> {
         Zpool::PrimaryKey(Root { lookup_root: self }, id)
     }
 
-    /// Select a resource of type Service, identified by its id
-    pub fn service_id(self, id: Uuid) -> Service<'a> {
-        Service::PrimaryKey(Root { lookup_root: self }, id)
-    }
-
     /// Select a resource of type Switch, identified by its id
     pub fn switch_id(self, id: Uuid) -> Switch<'a> {
         Switch::PrimaryKey(Root { lookup_root: self }, id)
@@ -826,15 +821,6 @@ lookup_resource! {
     primary_key_columns = [ { column_name = "id", rust_type = Uuid } ],
 }
 
-lookup_resource! {
-    name = "Service",
-    ancestors = [],
-    children = [],
-    lookup_by_name = false,
-    soft_deletes = false,
-    primary_key_columns = [ { column_name = "id", rust_type = Uuid } ]
-}
-
 lookup_resource! {
     name = "Switch",
     ancestors = [],

From 65cbb829d60fbb63ada38cc54d6518cbe7e57eca Mon Sep 17 00:00:00 2001
From: Ryan Goodfellow <ryan.goodfellow@oxide.computer>
Date: Sat, 9 Mar 2024 11:00:24 -0800
Subject: [PATCH 094/157] Add on-demand communication probes. (#4585)

---
 .github/buildomat/jobs/a4x2-deploy.sh         | 205 ++++++++
 .github/buildomat/jobs/a4x2-prepare.sh        |  94 ++++
 .github/buildomat/jobs/ci-tools.sh            |   5 +
 .github/buildomat/jobs/package.sh             |   1 +
 .github/buildomat/jobs/tuf-repo.sh            |   0
 .gitignore                                    |   3 +-
 Cargo.lock                                    | 414 +++++++++------
 Cargo.toml                                    |  13 +-
 clients/nexus-client/src/lib.rs               |   2 +
 clients/sled-agent-client/src/lib.rs          |  25 +-
 common/src/api/external/mod.rs                |  11 +
 common/src/api/internal/shared.rs             |   2 +
 dev-tools/omdb/src/bin/omdb/db.rs             |  43 +-
 end-to-end-tests/Cargo.toml                   |  11 +-
 end-to-end-tests/src/bin/commtest.rs          | 443 ++++++++++++++++
 end-to-end-tests/src/bin/dhcp-server.rs       | 125 +++++
 end-to-end-tests/src/helpers/cli.rs           |  22 +
 end-to-end-tests/src/helpers/icmp.rs          | 306 +++++++++++
 end-to-end-tests/src/helpers/mod.rs           |   2 +
 end-to-end-tests/src/instance_launch.rs       |   1 +
 illumos-utils/src/running_zone.rs             |   6 +
 installinator/src/dispatch.rs                 |   4 +-
 nexus/Cargo.toml                              |   2 +-
 nexus/db-model/src/deployment.rs              |   3 +-
 nexus/db-model/src/external_ip.rs             |  47 +-
 nexus/db-model/src/inventory.rs               |   3 +-
 nexus/db-model/src/lib.rs                     |   2 +
 nexus/db-model/src/network_interface.rs       |  65 ++-
 nexus/db-model/src/omicron_zone_config.rs     |  17 +-
 nexus/db-model/src/probe.rs                   |  50 ++
 nexus/db-model/src/schema.rs                  |  16 +-
 nexus/db-model/src/unsigned.rs                |   2 +
 nexus/db-queries/Cargo.toml                   |   1 +
 .../src/db/datastore/external_ip.rs           |  82 ++-
 nexus/db-queries/src/db/datastore/mod.rs      |   5 +
 .../src/db/datastore/network_interface.rs     | 104 +++-
 nexus/db-queries/src/db/datastore/probe.rs    | 390 ++++++++++++++
 nexus/db-queries/src/db/datastore/vpc.rs      |  34 +-
 .../db-queries/src/db/queries/external_ip.rs  |   7 +
 .../src/db/queries/network_interface.rs       |   9 +-
 .../execution/src/resource_allocation.rs      |  26 +-
 .../planning/src/blueprint_builder.rs         |  10 +-
 nexus/src/app/instance_network.rs             | 179 ++++++-
 nexus/src/app/mod.rs                          |   1 +
 nexus/src/app/probe.rs                        | 109 ++++
 nexus/src/app/rack.rs                         |   5 +-
 nexus/src/app/sagas/project_create.rs         |   1 -
 nexus/src/app/switch_port.rs                  |   6 +-
 nexus/src/app/vpc.rs                          |   2 +-
 nexus/src/external_api/http_entrypoints.rs    | 153 +++++-
 nexus/src/external_api/tag-config.json        |   6 +
 nexus/src/internal_api/http_entrypoints.rs    |  35 +-
 nexus/tests/integration_tests/mod.rs          |   1 +
 nexus/tests/integration_tests/oximeter.rs     |   2 +-
 nexus/tests/integration_tests/probe.rs        | 127 +++++
 nexus/tests/output/nexus_tags.txt             |   4 +
 .../output/uncovered-authz-endpoints.txt      |   4 +
 nexus/types/src/deployment.rs                 |   2 -
 nexus/types/src/external_api/params.rs        |  19 +
 nexus/types/src/inventory.rs                  |   4 +-
 openapi/nexus-internal.json                   | 192 ++++++-
 openapi/nexus.json                            | 489 ++++++++++++++++++
 openapi/sled-agent.json                       |  20 +
 package-manifest.toml                         |  17 +
 schema/all-zone-requests.json                 |  20 +
 schema/all-zones-requests.json                |  20 +
 schema/crdb/40.0.0/up1.sql                    |  10 +
 schema/crdb/40.0.0/up2.sql                    |   4 +
 schema/crdb/40.0.0/up3.sql                    |   1 +
 schema/crdb/40.0.0/up4.sql                    |   1 +
 schema/crdb/dbinit.sql                        |  22 +-
 schema/rss-service-plan-v2.json               |  20 +
 sled-agent/src/lib.rs                         |   1 +
 sled-agent/src/params.rs                      |   6 +-
 sled-agent/src/probe_manager.rs               | 383 ++++++++++++++
 sled-agent/src/sled_agent.rs                  |  16 +
 tools/ci_download_maghemite_mgd               |   2 +
 tools/ci_download_maghemite_openapi           |   4 +-
 tools/ci_download_thundermuffin               | 153 ++++++
 tools/install_builder_prerequisites.sh        |   3 +
 tools/thundermuffin_checksums                 |   1 +
 tools/thundermuffin_version                   |   1 +
 workspace-hack/Cargo.toml                     | 108 ++--
 83 files changed, 4440 insertions(+), 327 deletions(-)
 create mode 100755 .github/buildomat/jobs/a4x2-deploy.sh
 create mode 100755 .github/buildomat/jobs/a4x2-prepare.sh
 mode change 100644 => 100755 .github/buildomat/jobs/tuf-repo.sh
 create mode 100644 end-to-end-tests/src/bin/commtest.rs
 create mode 100644 end-to-end-tests/src/bin/dhcp-server.rs
 create mode 100644 end-to-end-tests/src/helpers/cli.rs
 create mode 100644 end-to-end-tests/src/helpers/icmp.rs
 create mode 100644 nexus/db-model/src/probe.rs
 create mode 100644 nexus/db-queries/src/db/datastore/probe.rs
 create mode 100644 nexus/src/app/probe.rs
 create mode 100644 nexus/tests/integration_tests/probe.rs
 create mode 100644 schema/crdb/40.0.0/up1.sql
 create mode 100644 schema/crdb/40.0.0/up2.sql
 create mode 100644 schema/crdb/40.0.0/up3.sql
 create mode 100644 schema/crdb/40.0.0/up4.sql
 create mode 100644 sled-agent/src/probe_manager.rs
 create mode 100755 tools/ci_download_thundermuffin
 create mode 100644 tools/thundermuffin_checksums
 create mode 100644 tools/thundermuffin_version

diff --git a/.github/buildomat/jobs/a4x2-deploy.sh b/.github/buildomat/jobs/a4x2-deploy.sh
new file mode 100755
index 0000000000..a07f953281
--- /dev/null
+++ b/.github/buildomat/jobs/a4x2-deploy.sh
@@ -0,0 +1,205 @@
+#!/bin/bash
+#:
+#: name = "a4x2-deploy"
+#: variety = "basic"
+#: target = "lab-2.0-opte-0.27"
+#: rust_toolchain = "stable"
+#: output_rules = [
+#:	"/out/falcon/*.log",
+#:	"/out/falcon/*.err",
+#:  "/out/connectivity-report.json",
+#:  "/ci/out/*-sled-agent.log",
+#:  "/ci/out/*cockroach*.log",
+#:  "%/out/dhcp-server.log",
+#: ]
+#: skip_clone = true
+#: enable = false
+#:
+#: [dependencies.a4x2]
+#: job = "a4x2-prepare"
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+pfexec mkdir -p /out
+pfexec chown "$UID" /out
+
+#
+# If we fail, try to collect some debugging information
+#
+_exit_trap() {
+	local status=$?
+	[[ $status -eq 0 ]] && exit 0
+
+    set +o errexit
+
+    df -h
+
+    # show what services have issues
+    for gimlet in g0 g1 g2 g3; do
+        ./a4x2 exec $gimlet "svcs -xvZ"
+    done
+
+    mkdir -p /out/falcon
+    cp .falcon/* /out/falcon/
+    for x in ce cr1 cr2 g0 g1 g2 g3; do
+        mv /out/falcon/$x.out /out/falcon/$x.log
+    done
+    cp connectivity-report.json /out/
+
+    mkdir -p /ci/out
+
+    for gimlet in g0 g1 g2 g3; do
+        ./a4x2 exec                                           \
+            $gimlet                                           \
+            "cat /var/svc/log/oxide-sled-agent:default.log" > \
+            /ci/out/$gimlet-sled-agent.log
+    done
+
+    # collect cockroachdb logs
+    mkdir -p /ci/log
+    for gimlet in g0 g1 g2 g3; do
+        ./a4x2 exec $gimlet 'cat /pool/ext/*/crypt/zone/oxz_cockroachdb*/root/data/logs/cockroach.log' > \
+            /ci/out/$gimlet-cockroach.log
+
+        ./a4x2 exec $gimlet 'cat /pool/ext/*/crypt/zone/oxz_cockroachdb*/root/data/logs/cockroach-stderr.log' > \
+            /ci/out/$gimlet-cockroach-stderr.log
+
+        ./a4x2 exec $gimlet 'cat /pool/ext/*/crypt/zone/oxz_cockroachdb*/root/data/logs/cockroach-health.log' > \
+            /ci/out/$gimlet-cockroach-health.log
+
+        ./a4x2 exec $gimlet 'cat /pool/ext/*/crypt/zone/oxz_cockroachdb*/root/var/svc/log/oxide-cockroachdb:default.log*' > \
+            /ci/out/$gimlet-oxide-cockroachdb.log
+    done
+}
+trap _exit_trap EXIT
+
+#
+# Install propolis
+#
+curl -fOL https://buildomat.eng.oxide.computer/wg/0/artefact/01HJ4BJJY2Q9EKXHYV6HQZ8XPN/qQS2fnkS9LebcL4cDLeHRWdleSiXaGKEXGLDucRoab8pwBSi/01HJ4BJY5F995ET252YSD4NJWV/01HJ4CGFH946THBF0ZRH6SRM8X/propolis-server
+chmod +x propolis-server
+pfexec mv propolis-server /usr/bin/
+
+#
+# Make space for CI work
+#
+export DISK=${DISK:-c1t1d0}
+pfexec diskinfo
+pfexec zpool create -f cpool $DISK
+pfexec zfs create -o mountpoint=/ci cpool/ci
+
+if [[ $(curl -s http://catacomb.eng.oxide.computer:12346/trim-me) =~ "true" ]]; then
+    pfexec zpool trim cpool
+    while [[ ! $(zpool status -t cpool) =~ "100%" ]]; do sleep 10; done
+fi
+
+pfexec chown "$UID" /ci
+cd /ci
+
+#
+# Fetch and decompress the cargo bay from the a4x2-prepeare job
+#
+for x in ce cr1 cr2 omicron-common g0 g1 g2 g3 tools; do
+    tar -xvzf /input/a4x2/out/cargo-bay-$x.tgz
+done
+
+for sled in g0 g1 g2 g3; do
+    cp -r cargo-bay/omicron-common/omicron/out/* cargo-bay/$sled/omicron/out/
+done
+ls -R
+
+#
+# Fetch the a4x2 topology manager program
+#
+buildomat_url=https://buildomat.eng.oxide.computer
+testbed_artifact_path=public/file/oxidecomputer/testbed/topo/
+testbed_rev=677559e30b4dfc65c374b24336ac23d40102de81
+curl -fOL $buildomat_url/$testbed_artifact_path/$testbed_rev/a4x2
+chmod +x a4x2
+
+#
+# Create a zpool for falcon images and disks
+#
+
+#
+# Install falcon base images
+#
+export FALCON_DATASET=cpool/falcon
+images="debian-11.0_0 helios-2.0_0"
+for img in $images; do
+    file=$img.raw.xz
+    curl -OL http://catacomb.eng.oxide.computer:12346/falcon/$file
+    unxz --keep -T 0 $file
+
+    file=$img.raw
+    name=${img%_*}
+    fsize=`ls -l $img.raw | awk '{print $5}'`
+    let vsize=(fsize + 4096 - size%4096)
+
+    pfexec zfs create -p -V $vsize -o volblocksize=4k "$FALCON_DATASET/img/$name"
+    pfexec dd if=$img.raw of="/dev/zvol/rdsk/$FALCON_DATASET/img/$name" bs=1024k status=progress
+    pfexec zfs snapshot "$FALCON_DATASET/img/$name@base"
+done
+
+#
+# Install OVMF
+#
+curl -fOL http://catacomb.eng.oxide.computer:12346/falcon/OVMF_CODE.fd
+pfexec mkdir -p /var/ovmf
+pfexec cp OVMF_CODE.fd /var/ovmf/OVMF_CODE.fd
+
+#
+# Fetch the arista image
+#
+curl -fOL http://catacomb.eng.oxide.computer:12346/falcon/arista.gz.xz
+unxz arista.gz.xz
+pfexec zfs receive cpool/falcon/img/arista@base < arista.gz
+
+#
+# Run the VM dhcp server
+#
+export EXT_INTERFACE=${EXT_INTERFACE:-igb0}
+
+cp /input/a4x2/out/dhcp-server .
+chmod +x dhcp-server
+first=`bmat address ls -f extra -Ho first`
+last=`bmat address ls -f extra -Ho last`
+gw=`bmat address ls -f extra -Ho gateway`
+server=`ipadm show-addr $EXT_INTERFACE/dhcp -po ADDR | sed 's#/.*##g'`
+pfexec ./dhcp-server $first $last $gw $server &> /out/dhcp-server.log &
+
+#
+# Run the topology
+#
+pfexec ./a4x2 launch
+
+#
+# Add a route to the rack ip pool
+#
+
+# Get the DHCP address for the external interface of the customer edge VM. This
+# VM interface is attached to the host machine's external interface via viona.
+customer_edge_addr=$(./a4x2 exec ce \
+    "ip -4 -j addr show enp0s10 | jq -r '.[0].addr_info[] | select(.dynamic == true) | .local'")
+
+# Add the route to the rack via the customer edge VM
+pfexec dladm
+pfexec ipadm
+pfexec netstat -nr
+pfexec route add 198.51.100.0/24 $customer_edge_addr
+
+#
+# Run the communications test program
+#
+cp /input/a4x2/out/commtest .
+chmod +x commtest
+pfexec ./commtest http://198.51.100.23 run \
+    --ip-pool-begin 198.51.100.40 \
+    --ip-pool-end 198.51.100.70 \
+    --icmp-loss-tolerance 10 \
+    --test-duration 300s \
+    --packet-rate 30
+
+cp connectivity-report.json /out/
diff --git a/.github/buildomat/jobs/a4x2-prepare.sh b/.github/buildomat/jobs/a4x2-prepare.sh
new file mode 100755
index 0000000000..bc88ddd4c0
--- /dev/null
+++ b/.github/buildomat/jobs/a4x2-prepare.sh
@@ -0,0 +1,94 @@
+#!/bin/bash
+#:
+#: name = "a4x2-prepare"
+#: variety = "basic"
+#: target = "helios-2.0"
+#: rust_toolchain = "stable"
+#: output_rules = [
+#:	"=/out/cargo-bay-ce.tgz",
+#:	"=/out/cargo-bay-cr1.tgz",
+#:	"=/out/cargo-bay-cr2.tgz",
+#:	"=/out/cargo-bay-g0.tgz",
+#:	"=/out/cargo-bay-g1.tgz",
+#:	"=/out/cargo-bay-g2.tgz",
+#:	"=/out/cargo-bay-g3.tgz",
+#:	"=/out/cargo-bay-tools.tgz",
+#:	"=/out/cargo-bay-omicron-common.tgz",
+#:	"=/out/commtest",
+#:	"=/out/dhcp-server",
+#: ]
+#: access_repos = [
+#:	"oxidecomputer/testbed",
+#: ]
+#: enable = false
+
+source ./env.sh
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+pfexec mkdir -p /out
+pfexec chown "$UID" /out
+
+#
+# Prep to build omicron
+#
+banner "prerequisites"
+set -o xtrace
+./tools/install_builder_prerequisites.sh -y
+
+#
+# Build the commtest program and place in the output
+#
+banner "commtest"
+cargo build -p end-to-end-tests --bin commtest --bin dhcp-server --release
+cp target/release/commtest /out/
+cp target/release/dhcp-server /out/
+
+#
+# Clone the testbed repo
+#
+banner "testbed"
+cd /work/oxidecomputer
+rm -rf testbed
+git clone https://github.com/oxidecomputer/testbed
+cd testbed/a4x2
+
+#
+# Build the a4x2 cargo bay using the omicron sources in this branch, fetch the
+# softnpu artifacts into the cargo bay, zip up the cargo bay and place it in the
+# output.
+#
+OMICRON=/work/oxidecomputer/omicron ./config/build-packages.sh
+
+# Create an omicron archive that captures common assets
+
+pushd cargo-bay
+mkdir -p omicron-common/omicron/
+cp -r g0/omicron/out omicron-common/omicron/
+# sled agent, gateway and switch archives are sled-specific
+rm omicron-common/omicron/out/omicron-sled-agent.tar 
+rm omicron-common/omicron/out/omicron-gateway*
+rm omicron-common/omicron/out/switch-softnpu.tar.gz
+popd
+
+# Remove everything in $sled/omicron/out except sled-agent, mgs (gateway), and
+# switch tar archives, these common elements are in the omicron-common archive
+for sled in g0 g1 g2 g3; do
+    find cargo-bay/$sled/omicron/out/ -maxdepth 1 -mindepth 1 \
+        | grep -v sled-agent \
+        | grep -v omicron-gateway \
+        | grep -v switch-softnpu \
+        | xargs -l rm -rf
+done
+
+# Put the softnpu artifacts in place.
+./config/fetch-softnpu-artifacts.sh
+
+# Archive everything up and place it in the output
+for x in ce cr1 cr2 g0 g1 g2 g3 tools omicron-common; do
+    tar -czf cargo-bay-$x.tgz cargo-bay/$x
+    mv cargo-bay-$x.tgz /out/
+done
+
diff --git a/.github/buildomat/jobs/ci-tools.sh b/.github/buildomat/jobs/ci-tools.sh
index 07a63af30c..ce17d4fb30 100755
--- a/.github/buildomat/jobs/ci-tools.sh
+++ b/.github/buildomat/jobs/ci-tools.sh
@@ -8,6 +8,7 @@
 #:	"=/work/end-to-end-tests/*.gz",
 #:	"=/work/caboose-util.gz",
 #:	"=/work/tufaceous.gz",
+#:	"=/work/commtest",
 #: ]
 
 set -o errexit
@@ -33,6 +34,10 @@ export CARGO_INCREMENTAL=0
 ptime -m cargo build --locked -p end-to-end-tests --tests --bin bootstrap \
 	--message-format json-render-diagnostics >/tmp/output.end-to-end.json
 
+mkdir -p /work
+ptime -m cargo build --locked -p end-to-end-tests --tests --bin commtest
+cp target/debug/commtest /work/commtest
+
 mkdir -p /work/end-to-end-tests
 for p in target/debug/bootstrap $(/opt/ooce/bin/jq -r 'select(.profile.test) | .executable' /tmp/output.end-to-end.json); do
 	# shellcheck disable=SC2094
diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
index dc89bc787b..d290976d9f 100755
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -124,6 +124,7 @@ zones=(
   out/omicron-gateway-softnpu.tar.gz
   out/omicron-gateway-asic.tar.gz
   out/overlay.tar.gz
+  out/probe.tar.gz
 )
 cp "${zones[@]}" /work/zones/
 
diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh
old mode 100644
new mode 100755
diff --git a/.gitignore b/.gitignore
index fc5fd5f297..a13536aab7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,4 +13,5 @@ debug.out
 rusty-tags.vi
 *.sw*
 tags
-.direnv
\ No newline at end of file
+.direnv
+connectivity-report.json
diff --git a/Cargo.lock b/Cargo.lock
index 416766b9cb..0cfc7b4500 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -54,12 +54,12 @@ dependencies = [
 
 [[package]]
 name = "ahash"
-version = "0.8.7"
+version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01"
+checksum = "42cd52102d3df161c77a887b608d7a4897d7cc112886a9537b738a887a03aaff"
 dependencies = [
  "cfg-if",
- "getrandom 0.2.10",
+ "getrandom 0.2.12",
  "once_cell",
  "version_check",
  "zerocopy 0.7.32",
@@ -67,9 +67,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "1.0.4"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a"
+checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0"
 dependencies = [
  "memchr",
 ]
@@ -160,9 +160,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.75"
+version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
+checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca"
 dependencies = [
  "backtrace",
 ]
@@ -241,7 +241,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed72493ac66d5804837f480ab3766c72bdfab91a65e565fc54fa9e42db0073a8"
 dependencies = [
  "anstyle",
- "bstr 1.6.0",
+ "bstr 1.9.0",
  "doc-comment",
  "predicates",
  "predicates-core",
@@ -377,7 +377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1"
 dependencies = [
  "futures-core",
- "getrandom 0.2.10",
+ "getrandom 0.2.12",
  "instant",
  "pin-project-lite",
  "rand 0.8.5",
@@ -485,7 +485,7 @@ version = "0.69.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a4c69fae65a523209d34240b60abe0c42d33d1045d445c0839d8a4894a736e2d"
 dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.4.2",
  "cexpr",
  "clang-sys",
  "lazy_static",
@@ -531,9 +531,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.4.0"
+version = "2.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
+checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf"
 dependencies = [
  "serde",
 ]
@@ -678,12 +678,12 @@ dependencies = [
 
 [[package]]
 name = "bstr"
-version = "1.6.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05"
+checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc"
 dependencies = [
  "memchr",
- "regex-automata 0.3.8",
+ "regex-automata 0.4.5",
  "serde",
 ]
 
@@ -909,9 +909,9 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.31"
+version = "0.4.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
+checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
@@ -919,7 +919,7 @@ dependencies = [
  "num-traits",
  "serde",
  "wasm-bindgen",
- "windows-targets 0.48.5",
+ "windows-targets 0.52.0",
 ]
 
 [[package]]
@@ -988,9 +988,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.0"
+version = "4.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80c21025abd42669a92efc996ef13cfb2c5c627858421ea58d5c3b331a6c134f"
+checksum = "c918d541ef2913577a0f9566e9ce27cb35b6df072075769e0b26cb5a554520da"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -998,9 +998,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.0"
+version = "4.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "458bf1f341769dfcf849846f65dffdf9146daa56bcd2a47cb4e1de9915567c99"
+checksum = "9f3e7391dad68afb0c2ede1bf619f579a3dc9c2ec67f089baa397123a2f3d1eb"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1050,11 +1050,10 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
 
 [[package]]
 name = "colored"
-version = "2.0.4"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6"
+checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8"
 dependencies = [
- "is-terminal",
  "lazy_static",
  "windows-sys 0.48.0",
 ]
@@ -1087,9 +1086,9 @@ dependencies = [
 
 [[package]]
 name = "const-oid"
-version = "0.9.5"
+version = "0.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f"
+checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
 
 [[package]]
 name = "constant_time_eq"
@@ -1109,6 +1108,17 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
 
+[[package]]
+name = "cookie"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7efb37c3e1ccb1ff97164ad95ac1606e8ccd35b3fa0a7d99a304c7f4a428cc24"
+dependencies = [
+ "percent-encoding",
+ "time",
+ "version_check",
+]
+
 [[package]]
 name = "cookie"
 version = "0.18.0"
@@ -1119,6 +1129,23 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "cookie_store"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "387461abbc748185c3a6e1673d826918b450b87ff22639429c694619a83b6cf6"
+dependencies = [
+ "cookie 0.17.0",
+ "idna 0.3.0",
+ "log",
+ "publicsuffix",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "time",
+ "url",
+]
+
 [[package]]
 name = "core-foundation"
 version = "0.9.3"
@@ -1201,7 +1228,7 @@ dependencies = [
  "anes",
  "cast",
  "ciborium",
- "clap 4.5.0",
+ "clap 4.5.1",
  "criterion-plot",
  "futures",
  "is-terminal",
@@ -1259,25 +1286,18 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.15"
+version = "0.9.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
 dependencies = [
- "autocfg",
- "cfg-if",
  "crossbeam-utils",
- "memoffset 0.9.0",
- "scopeguard",
 ]
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.16"
+version = "0.8.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
-dependencies = [
- "cfg-if",
-]
+checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
 
 [[package]]
 name = "crossterm"
@@ -1285,7 +1305,7 @@ version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df"
 dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.4.2",
  "crossterm_winapi",
  "futures-core",
  "libc",
@@ -1646,10 +1666,11 @@ dependencies = [
 
 [[package]]
 name = "deranged"
-version = "0.3.8"
+version = "0.3.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946"
+checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
 dependencies = [
+ "powerfmt",
  "serde",
 ]
 
@@ -1719,13 +1740,34 @@ dependencies = [
  "syn 2.0.51",
 ]
 
+[[package]]
+name = "dhcproto"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "000717b4f6913807b6195419e0bacb008d449ba6023ca26abf349c4ff2f1866b"
+dependencies = [
+ "dhcproto-macros",
+ "hex",
+ "ipnet",
+ "rand 0.8.5",
+ "thiserror",
+ "trust-dns-proto",
+ "url",
+]
+
+[[package]]
+name = "dhcproto-macros"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7993efb860416547839c115490d4951c6d0f8ec04a3594d9dd99d50ed7ec170"
+
 [[package]]
 name = "diesel"
 version = "2.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "62c6fcf842f17f8c78ecf7c81d75c5ce84436b41ee07e03f490fbb5f5a8731d8"
 dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.4.2",
  "byteorder",
  "chrono",
  "diesel_derives",
@@ -1863,7 +1905,7 @@ dependencies = [
  "anyhow",
  "camino",
  "chrono",
- "clap 4.5.0",
+ "clap 4.5.1",
  "dns-service-client",
  "dropshot",
  "expectorate",
@@ -1988,7 +2030,7 @@ dependencies = [
  "futures",
  "hostname",
  "http 0.2.12",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "indexmap 2.2.5",
  "multer",
  "openapiv3",
@@ -2165,12 +2207,20 @@ dependencies = [
 name = "end-to-end-tests"
 version = "0.1.0"
 dependencies = [
+ "anstyle",
  "anyhow",
  "async-trait",
  "base64",
  "chrono",
+ "clap 4.5.1",
+ "colored",
+ "dhcproto",
  "http 0.2.12",
- "hyper 0.14.27",
+ "humantime",
+ "hyper 0.14.28",
+ "internet-checksum",
+ "ispf",
+ "macaddr",
  "omicron-sled-agent",
  "omicron-test-utils",
  "omicron-workspace-hack",
@@ -2181,6 +2231,7 @@ dependencies = [
  "russh-keys",
  "serde",
  "serde_json",
+ "socket2 0.5.5",
  "tokio",
  "toml 0.8.10",
  "trust-dns-resolver",
@@ -2569,7 +2620,7 @@ name = "gateway-cli"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "clap 4.5.0",
+ "clap 4.5.1",
  "futures",
  "gateway-client",
  "gateway-messages",
@@ -2711,9 +2762,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.10"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
+checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -2751,7 +2802,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d"
 dependencies = [
  "aho-corasick",
- "bstr 1.6.0",
+ "bstr 1.9.0",
  "fnv",
  "log",
  "regex",
@@ -3093,7 +3144,7 @@ dependencies = [
  "form_urlencoded",
  "futures",
  "http 0.2.12",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "log",
  "once_cell",
  "regex",
@@ -3170,9 +3221,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "hyper"
-version = "0.14.27"
+version = "0.14.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
+checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80"
 dependencies = [
  "bytes",
  "futures-channel",
@@ -3185,7 +3236,7 @@ dependencies = [
  "httpdate",
  "itoa",
  "pin-project-lite",
- "socket2 0.4.9",
+ "socket2 0.5.5",
  "tokio",
  "tower-service",
  "tracing",
@@ -3218,7 +3269,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
 dependencies = [
  "futures-util",
  "http 0.2.12",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "rustls 0.21.9",
  "tokio",
  "tokio-rustls 0.24.1",
@@ -3253,7 +3304,7 @@ dependencies = [
  "http 0.2.12",
  "http-range",
  "httpdate",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "mime_guess",
  "percent-encoding",
  "rand 0.8.5",
@@ -3269,7 +3320,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
 dependencies = [
  "bytes",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "native-tls",
  "tokio",
  "tokio-native-tls",
@@ -3335,6 +3386,16 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "idna"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+]
+
 [[package]]
 name = "idna"
 version = "0.5.0"
@@ -3474,7 +3535,7 @@ dependencies = [
  "bytes",
  "camino",
  "cancel-safe-futures",
- "clap 4.5.0",
+ "clap 4.5.1",
  "ddm-admin-client",
  "display-error-chain",
  "futures",
@@ -3534,10 +3595,10 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
- "clap 4.5.0",
+ "clap 4.5.1",
  "dropshot",
  "expectorate",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "installinator-common",
  "omicron-common",
  "omicron-test-utils",
@@ -3593,7 +3654,7 @@ dependencies = [
  "dropshot",
  "expectorate",
  "futures",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "omicron-common",
  "omicron-test-utils",
  "omicron-workspace-hack",
@@ -3615,7 +3676,7 @@ name = "internal-dns-cli"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "clap 4.5.0",
+ "clap 4.5.1",
  "dropshot",
  "internal-dns",
  "omicron-common",
@@ -3625,6 +3686,12 @@ dependencies = [
  "trust-dns-resolver",
 ]
 
+[[package]]
+name = "internet-checksum"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc6d6206008e25125b1f97fbe5d309eb7b85141cf9199d52dbd3729a1584dd16"
+
 [[package]]
 name = "ipcc"
 version = "0.1.0"
@@ -3686,6 +3753,14 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45"
 
+[[package]]
+name = "ispf"
+version = "0.1.0"
+source = "git+https://github.com/oxidecomputer/ispf#f78443a98397f7818b1e7a487dbb7d5cad625496"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -3884,7 +3959,7 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6d8de370f98a6cb8a4606618e53e802f93b094ddec0f96988eaec2c27e6e9ce7"
 dependencies = [
- "clap 4.5.0",
+ "clap 4.5.1",
  "termcolor",
  "threadpool",
 ]
@@ -3940,7 +4015,7 @@ version = "0.2.4"
 source = "git+https://github.com/oxidecomputer/lpc55_support#96f064eaae5e95930efaab6c29fd1b2e22225dac"
 dependencies = [
  "bitfield",
- "clap 4.5.0",
+ "clap 4.5.1",
  "packed_struct",
  "serde",
 ]
@@ -4044,9 +4119,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
 
 [[package]]
 name = "memchr"
-version = "2.6.3"
+version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
+checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
 
 [[package]]
 name = "memmap"
@@ -4076,15 +4151,6 @@ dependencies = [
  "autocfg",
 ]
 
-[[package]]
-name = "memoffset"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "mg-admin-client"
 version = "0.1.0"
@@ -4211,7 +4277,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
 dependencies = [
- "getrandom 0.2.10",
+ "getrandom 0.2.12",
 ]
 
 [[package]]
@@ -4366,7 +4432,7 @@ dependencies = [
  "camino",
  "camino-tempfile",
  "chrono",
- "cookie",
+ "cookie 0.18.0",
  "db-macros",
  "diesel",
  "diesel-dtrace",
@@ -4376,7 +4442,7 @@ dependencies = [
  "gateway-client",
  "headers",
  "http 0.2.12",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "hyper-rustls 0.26.0",
  "illumos-utils",
  "internal-dns",
@@ -4414,6 +4480,7 @@ dependencies = [
  "regex",
  "rustls 0.22.2",
  "samael",
+ "schemars",
  "serde",
  "serde_json",
  "serde_urlencoded",
@@ -4580,7 +4647,7 @@ dependencies = [
  "gateway-test-utils",
  "headers",
  "http 0.2.12",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "internal-dns",
  "nexus-config",
  "nexus-db-queries",
@@ -4659,7 +4726,7 @@ dependencies = [
  "bitflags 1.3.2",
  "cfg-if",
  "libc",
- "memoffset 0.7.1",
+ "memoffset",
  "pin-utils",
  "static_assertions",
 ]
@@ -4670,7 +4737,7 @@ version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"
 dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.4.2",
  "cfg-if",
  "libc",
 ]
@@ -4758,6 +4825,12 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "num-conv"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
+
 [[package]]
 name = "num-derive"
 version = "0.4.0"
@@ -4780,9 +4853,9 @@ dependencies = [
 
 [[package]]
 name = "num-iter"
-version = "0.1.43"
+version = "0.1.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
+checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9"
 dependencies = [
  "autocfg",
  "num-integer",
@@ -4802,9 +4875,9 @@ dependencies = [
 
 [[package]]
 name = "num-traits"
-version = "0.2.16"
+version = "0.2.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2"
+checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
 dependencies = [
  "autocfg",
  "libm",
@@ -4968,7 +5041,7 @@ dependencies = [
  "anyhow",
  "camino",
  "camino-tempfile",
- "clap 4.5.0",
+ "clap 4.5.1",
  "dropshot",
  "expectorate",
  "futures",
@@ -5002,7 +5075,7 @@ dependencies = [
  "anyhow",
  "base64",
  "camino",
- "clap 4.5.0",
+ "clap 4.5.1",
  "dropshot",
  "expectorate",
  "futures",
@@ -5011,7 +5084,7 @@ dependencies = [
  "gateway-test-utils",
  "hex",
  "http 0.2.12",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "illumos-utils",
  "ipcc",
  "omicron-common",
@@ -5053,7 +5126,7 @@ dependencies = [
  "camino-tempfile",
  "cancel-safe-futures",
  "chrono",
- "clap 4.5.0",
+ "clap 4.5.1",
  "criterion",
  "crucible-agent-client",
  "crucible-pantry-client",
@@ -5073,7 +5146,7 @@ dependencies = [
  "http 0.2.12",
  "httptest",
  "hubtools",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "hyper-rustls 0.26.0",
  "illumos-utils",
  "internal-dns",
@@ -5167,7 +5240,7 @@ dependencies = [
  "camino",
  "camino-tempfile",
  "chrono",
- "clap 4.5.0",
+ "clap 4.5.1",
  "crossterm",
  "crucible-agent-client",
  "csv",
@@ -5219,7 +5292,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "camino",
- "clap 4.5.0",
+ "clap 4.5.1",
  "expectorate",
  "futures",
  "hex",
@@ -5286,7 +5359,7 @@ dependencies = [
  "cancel-safe-futures",
  "cfg-if",
  "chrono",
- "clap 4.5.0",
+ "clap 4.5.1",
  "crucible-agent-client",
  "ddm-admin-client",
  "derive_more",
@@ -5304,7 +5377,7 @@ dependencies = [
  "guppy",
  "hex",
  "http 0.2.12",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "hyper-staticfile",
  "illumos-utils",
  "installinator-common",
@@ -5418,14 +5491,14 @@ dependencies = [
  "bit-set",
  "bit-vec",
  "bitflags 1.3.2",
- "bitflags 2.4.0",
+ "bitflags 2.4.2",
  "bstr 0.2.17",
- "bstr 1.6.0",
+ "bstr 1.9.0",
  "byteorder",
  "bytes",
  "chrono",
  "cipher",
- "clap 4.5.0",
+ "clap 4.5.1",
  "clap_builder",
  "console",
  "const-oid",
@@ -5450,12 +5523,12 @@ dependencies = [
  "futures-util",
  "gateway-messages",
  "generic-array",
- "getrandom 0.2.10",
+ "getrandom 0.2.12",
  "group",
  "hashbrown 0.14.3",
  "hex",
  "hmac",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "indexmap 2.2.5",
  "inout",
  "ipnetwork",
@@ -5483,7 +5556,7 @@ dependencies = [
  "rand 0.8.5",
  "rand_chacha 0.3.1",
  "regex",
- "regex-automata 0.4.4",
+ "regex-automata 0.4.5",
  "regex-syntax 0.8.2",
  "reqwest",
  "ring 0.17.8",
@@ -5495,7 +5568,6 @@ dependencies = [
  "sha2",
  "similar",
  "slog",
- "socket2 0.5.5",
  "spin 0.9.8",
  "string_cache",
  "subtle",
@@ -5603,7 +5675,7 @@ version = "0.10.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800"
 dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.4.2",
  "cfg-if",
  "foreign-types 0.3.2",
  "libc",
@@ -5731,7 +5803,7 @@ dependencies = [
  "chrono",
  "futures",
  "http 0.2.12",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "omicron-workspace-hack",
  "progenitor",
  "rand 0.8.5",
@@ -5804,11 +5876,11 @@ dependencies = [
  "anyhow",
  "camino",
  "chrono",
- "clap 4.5.0",
+ "clap 4.5.1",
  "dropshot",
  "expectorate",
  "futures",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "internal-dns",
  "nexus-client",
  "nexus-types",
@@ -5847,7 +5919,7 @@ dependencies = [
  "bytes",
  "camino",
  "chrono",
- "clap 4.5.0",
+ "clap 4.5.1",
  "dropshot",
  "expectorate",
  "futures",
@@ -5917,7 +5989,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "chrono",
- "clap 4.5.0",
+ "clap 4.5.1",
  "dropshot",
  "nexus-client",
  "omicron-common",
@@ -5939,7 +6011,7 @@ dependencies = [
  "anyhow",
  "camino",
  "chrono",
- "clap 4.5.0",
+ "clap 4.5.1",
  "omicron-workspace-hack",
  "uuid",
 ]
@@ -6466,6 +6538,12 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -6693,10 +6771,10 @@ dependencies = [
  "anyhow",
  "atty",
  "base64",
- "clap 4.5.0",
+ "clap 4.5.1",
  "dropshot",
  "futures",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "progenitor",
  "propolis_types",
  "rand 0.8.5",
@@ -6732,7 +6810,7 @@ checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf"
 dependencies = [
  "bit-set",
  "bit-vec",
- "bitflags 2.4.0",
+ "bitflags 2.4.2",
  "lazy_static",
  "num-traits",
  "rand 0.8.5",
@@ -6744,6 +6822,22 @@ dependencies = [
  "unarray",
 ]
 
+[[package]]
+name = "psl-types"
+version = "2.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
+
+[[package]]
+name = "publicsuffix"
+version = "2.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96a8c1bda5ae1af7f99a2962e49df150414a43d62404644d98dd5c3a93d07457"
+dependencies = [
+ "idna 0.3.0",
+ "psl-types",
+]
+
 [[package]]
 name = "quick-error"
 version = "1.2.3"
@@ -6883,7 +6977,7 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom 0.2.10",
+ "getrandom 0.2.12",
 ]
 
 [[package]]
@@ -6910,7 +7004,7 @@ version = "0.26.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bcb12f8fbf6c62614b0d56eb352af54f6a22410c3b079eb53ee93c7b97dd31d8"
 dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.4.2",
  "cassowary",
  "compact_str",
  "crossterm",
@@ -6972,7 +7066,7 @@ dependencies = [
  "anyhow",
  "camino",
  "camino-tempfile",
- "clap 4.5.0",
+ "clap 4.5.1",
  "dropshot",
  "expectorate",
  "humantime",
@@ -7027,7 +7121,7 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
 dependencies = [
- "getrandom 0.2.10",
+ "getrandom 0.2.12",
  "redox_syscall 0.2.16",
  "thiserror",
 ]
@@ -7080,7 +7174,7 @@ checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-automata 0.4.4",
+ "regex-automata 0.4.5",
  "regex-syntax 0.8.2",
 ]
 
@@ -7092,15 +7186,9 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
 
 [[package]]
 name = "regex-automata"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
-
-[[package]]
-name = "regex-automata"
-version = "0.4.4"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a"
+checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -7152,13 +7240,15 @@ checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251"
 dependencies = [
  "base64",
  "bytes",
+ "cookie 0.17.0",
+ "cookie_store",
  "encoding_rs",
  "futures-core",
  "futures-util",
  "h2",
  "http 0.2.12",
  "http-body 0.4.5",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "hyper-rustls 0.24.2",
  "hyper-tls",
  "ipnet",
@@ -7233,7 +7323,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
 dependencies = [
  "cc",
  "cfg-if",
- "getrandom 0.2.10",
+ "getrandom 0.2.12",
  "libc",
  "spin 0.9.8",
  "untrusted 0.9.0",
@@ -7247,7 +7337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94"
 dependencies = [
  "base64",
- "bitflags 2.4.0",
+ "bitflags 2.4.2",
  "serde",
  "serde_derive",
 ]
@@ -7335,7 +7425,7 @@ dependencies = [
  "aes",
  "aes-gcm",
  "async-trait",
- "bitflags 2.4.0",
+ "bitflags 2.4.2",
  "byteorder",
  "chacha20",
  "ctr",
@@ -7471,7 +7561,7 @@ version = "0.38.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949"
 dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.4.2",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -7605,7 +7695,7 @@ version = "13.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "02a2d683a4ac90aeef5b1013933f6d977bd37d51ff3f4dad829d4931a7e6be86"
 dependencies = [
- "bitflags 2.4.0",
+ "bitflags 2.4.2",
  "cfg-if",
  "clipboard-win",
  "fd-lock 4.0.2",
@@ -8134,9 +8224,9 @@ dependencies = [
 
 [[package]]
 name = "similar"
-version = "2.3.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597"
+checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21"
 dependencies = [
  "bstr 0.2.17",
  "unicode-segmentation",
@@ -8488,7 +8578,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
- "clap 4.5.0",
+ "clap 4.5.1",
  "dropshot",
  "futures",
  "gateway-messages",
@@ -9145,14 +9235,16 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.27"
+version = "0.3.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bb39ee79a6d8de55f48f2293a830e040392f1c5f16e336bdd1788cd0aadce07"
+checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749"
 dependencies = [
  "deranged",
  "itoa",
  "libc",
+ "num-conv",
  "num_threads",
+ "powerfmt",
  "serde",
  "time-core",
  "time-macros",
@@ -9160,16 +9252,17 @@ dependencies = [
 
 [[package]]
 name = "time-core"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb"
+checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
 
 [[package]]
 name = "time-macros"
-version = "0.2.13"
+version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "733d258752e9303d392b94b75230d07b0b9c489350c69b851fc6c065fde3e8f9"
+checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774"
 dependencies = [
+ "num-conv",
  "time-core",
 ]
 
@@ -9525,11 +9618,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
 
 [[package]]
 name = "tracing"
-version = "0.1.37"
+version = "0.1.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
+checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
 dependencies = [
- "cfg-if",
  "log",
  "pin-project-lite",
  "tracing-attributes",
@@ -9538,9 +9630,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-attributes"
-version = "0.1.26"
+version = "0.1.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
+checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -9549,9 +9641,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-core"
-version = "0.1.31"
+version = "0.1.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a"
+checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
 dependencies = [
  "once_cell",
 ]
@@ -9672,7 +9764,7 @@ dependencies = [
  "assert_cmd",
  "camino",
  "chrono",
- "clap 4.5.0",
+ "clap 4.5.1",
  "console",
  "datatest-stable",
  "fs-err",
@@ -9832,9 +9924,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-bidi"
-version = "0.3.13"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
+checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
 
 [[package]]
 name = "unicode-ident"
@@ -9918,7 +10010,7 @@ dependencies = [
  "camino",
  "camino-tempfile",
  "chrono",
- "clap 4.5.0",
+ "clap 4.5.1",
  "debug-ignore",
  "display-error-chain",
  "dropshot",
@@ -9949,7 +10041,7 @@ dependencies = [
  "camino",
  "camino-tempfile",
  "cancel-safe-futures",
- "clap 4.5.0",
+ "clap 4.5.1",
  "debug-ignore",
  "derive-where",
  "either",
@@ -10129,7 +10221,7 @@ version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a"
 dependencies = [
- "getrandom 0.2.10",
+ "getrandom 0.2.12",
  "serde",
 ]
 
@@ -10370,7 +10462,7 @@ dependencies = [
  "buf-list",
  "camino",
  "ciborium",
- "clap 4.5.0",
+ "clap 4.5.1",
  "crossterm",
  "futures",
  "humantime",
@@ -10431,7 +10523,7 @@ dependencies = [
  "bytes",
  "camino",
  "ciborium",
- "clap 4.5.0",
+ "clap 4.5.1",
  "crossterm",
  "omicron-workspace-hack",
  "reedline",
@@ -10456,7 +10548,7 @@ dependencies = [
  "bytes",
  "camino",
  "camino-tempfile",
- "clap 4.5.0",
+ "clap 4.5.1",
  "ddm-admin-client",
  "debug-ignore",
  "display-error-chain",
@@ -10474,7 +10566,7 @@ dependencies = [
  "hex",
  "http 0.2.12",
  "hubtools",
- "hyper 0.14.27",
+ "hyper 0.14.28",
  "illumos-utils",
  "installinator",
  "installinator-artifact-client",
@@ -10782,7 +10874,7 @@ dependencies = [
  "camino",
  "cargo_metadata",
  "cargo_toml",
- "clap 4.5.0",
+ "clap 4.5.1",
 ]
 
 [[package]]
@@ -10901,8 +10993,7 @@ dependencies = [
 [[package]]
 name = "zone"
 version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a62a428a79ea2224ce8ab05d6d8a21bdd7b4b68a8dbc1230511677a56e72ef22"
+source = "git+https://github.com/oxidecomputer/zone?branch=state-derive-eq-hash#f1920d5636c69ea8179f8ec659702dcdef43268c"
 dependencies = [
  "itertools 0.10.5",
  "thiserror",
@@ -10915,7 +11006,7 @@ name = "zone-network-setup"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "clap 4.5.0",
+ "clap 4.5.1",
  "dropshot",
  "illumos-utils",
  "omicron-common",
@@ -10928,8 +11019,7 @@ dependencies = [
 [[package]]
 name = "zone_cfg_derive"
 version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5c4f01d3785e222d5aca11c9813e9c46b69abfe258756c99c9b628683626cc8"
+source = "git+https://github.com/oxidecomputer/zone?branch=state-derive-eq-hash#f1920d5636c69ea8179f8ec659702dcdef43268c"
 dependencies = [
  "heck 0.4.1",
  "proc-macro-error",
diff --git a/Cargo.toml b/Cargo.toml
index 299d715d67..96f47be86f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -158,6 +158,7 @@ resolver = "2"
 
 [workspace.dependencies]
 anyhow = "1.0"
+anstyle = "1.0"
 api_identity = { path = "api_identity" }
 approx = "0.5.1"
 assert_matches = "1.5.0"
@@ -184,6 +185,7 @@ ciborium = "0.2.2"
 cfg-if = "1.0"
 chrono = { version = "0.4", features = [ "serde" ] }
 clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] }
+colored = "2.1"
 cookie = "0.18"
 criterion = { version = "0.5.1", features = [ "async_tokio" ] }
 crossbeam = "0.8"
@@ -246,8 +248,11 @@ installinator-common = { path = "installinator-common" }
 internal-dns = { path = "internal-dns" }
 ipcc = { path = "ipcc" }
 ipnet = "2.9"
-ipnetwork = { version = "0.20", features = ["schemars"] }
 itertools = "0.12.1"
+internet-checksum = "0.2"
+ipcc-key-value = { path = "ipcc-key-value" }
+ipnetwork = { version = "0.20", features = ["schemars"] }
+ispf = { git = "https://github.com/oxidecomputer/ispf" }
 key-manager = { path = "key-manager" }
 kstat-rs = "0.2.3"
 libc = "0.2.153"
@@ -373,6 +378,8 @@ slog-envlogger = "2.2"
 slog-error-chain = { git = "https://github.com/oxidecomputer/slog-error-chain", branch = "main", features = ["derive"] }
 slog-term = "2.9"
 smf = "0.2"
+snafu = "0.7"
+socket2 = { version = "0.5", features = ["all"] }
 sp-sim = { path = "sp-sim" }
 sprockets-common = { git = "http://github.com/oxidecomputer/sprockets", rev = "77df31efa5619d0767ffc837ef7468101608aee9" }
 sprockets-host = { git = "http://github.com/oxidecomputer/sprockets", rev = "77df31efa5619d0767ffc837ef7468101608aee9" }
@@ -654,3 +661,7 @@ branch = "oxide/omicron"
 # See also: uuid-kinds/README.adoc.
 [patch."https://github.com/oxidecomputer/omicron"]
 omicron-uuid-kinds = { path = "uuid-kinds" }
+
+[patch.crates-io.zone]
+git = 'https://github.com/oxidecomputer/zone'
+branch = 'state-derive-eq-hash'
diff --git a/clients/nexus-client/src/lib.rs b/clients/nexus-client/src/lib.rs
index 55bdf3d0aa..85c67ddbfd 100644
--- a/clients/nexus-client/src/lib.rs
+++ b/clients/nexus-client/src/lib.rs
@@ -33,6 +33,8 @@ progenitor::generate_api!(
         MacAddr = omicron_common::api::external::MacAddr,
         Name = omicron_common::api::external::Name,
         NewPasswordHash = omicron_passwords::NewPasswordHash,
+        NetworkInterface = omicron_common::api::internal::shared::NetworkInterface,
+        NetworkInterfaceKind = omicron_common::api::internal::shared::NetworkInterfaceKind,
     },
     patch = {
         SledAgentInfo = { derives = [PartialEq, Eq] },
diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs
index eb1e57b11f..b2bc232ef5 100644
--- a/clients/sled-agent-client/src/lib.rs
+++ b/clients/sled-agent-client/src/lib.rs
@@ -6,6 +6,7 @@
 
 use anyhow::Context;
 use async_trait::async_trait;
+use omicron_common::api::internal::shared::NetworkInterface;
 use std::convert::TryFrom;
 use std::net::IpAddr;
 use std::net::SocketAddr;
@@ -39,6 +40,7 @@ progenitor::generate_api!(
         PortSpeed = omicron_common::api::internal::shared::PortSpeed,
         SourceNatConfig = omicron_common::api::internal::shared::SourceNatConfig,
         Vni = omicron_common::api::external::Vni,
+        NetworkInterface = omicron_common::api::internal::shared::NetworkInterface,
     }
 );
 
@@ -141,7 +143,7 @@ impl types::OmicronZoneType {
     }
 
     /// The service vNIC providing external connectivity to this zone
-    pub fn service_vnic(&self) -> Option<&types::NetworkInterface> {
+    pub fn service_vnic(&self) -> Option<&NetworkInterface> {
         match self {
             types::OmicronZoneType::Nexus { nic, .. }
             | types::OmicronZoneType::ExternalDns { nic, .. }
@@ -556,26 +558,7 @@ impl From<omicron_common::api::internal::shared::NetworkInterfaceKind>
         match s {
             Instance { id } => Self::Instance(id),
             Service { id } => Self::Service(id),
-        }
-    }
-}
-
-impl From<omicron_common::api::internal::shared::NetworkInterface>
-    for types::NetworkInterface
-{
-    fn from(
-        s: omicron_common::api::internal::shared::NetworkInterface,
-    ) -> Self {
-        Self {
-            id: s.id,
-            kind: s.kind.into(),
-            name: s.name,
-            ip: s.ip,
-            mac: s.mac,
-            subnet: s.subnet.into(),
-            vni: s.vni,
-            primary: s.primary,
-            slot: s.slot,
+            Probe { id } => Self::Probe(id),
         }
     }
 }
diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs
index 17b4826f8c..58bc82c825 100644
--- a/common/src/api/external/mod.rs
+++ b/common/src/api/external/mod.rs
@@ -879,6 +879,8 @@ pub enum ResourceType {
     Vmm,
     Ipv4NatEntry,
     FloatingIp,
+    Probe,
+    ProbeNetworkInterface,
 }
 
 // IDENTITY METADATA
@@ -2827,6 +2829,15 @@ pub struct TufRepoGetResponse {
     pub description: TufRepoDescription,
 }
 
+#[derive(
+    Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, ObjectIdentity,
+)]
+pub struct Probe {
+    #[serde(flatten)]
+    pub identity: IdentityMetadata,
+    pub sled: Uuid,
+}
+
 #[cfg(test)]
 mod test {
     use serde::Deserialize;
diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs
index c8d8b1c786..bf825fd2e7 100644
--- a/common/src/api/internal/shared.rs
+++ b/common/src/api/internal/shared.rs
@@ -36,6 +36,8 @@ pub enum NetworkInterfaceKind {
     Instance { id: Uuid },
     /// A vNIC associated with an internal service
     Service { id: Uuid },
+    /// A vNIC associated with a probe
+    Probe { id: Uuid },
 }
 
 /// Information required to construct a virtual network interface
diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs
index 339c257d8e..98e7748054 100644
--- a/dev-tools/omdb/src/bin/omdb/db.rs
+++ b/dev-tools/omdb/src/bin/omdb/db.rs
@@ -54,6 +54,7 @@ use nexus_db_model::IpAttachState;
 use nexus_db_model::IpKind;
 use nexus_db_model::NetworkInterface;
 use nexus_db_model::NetworkInterfaceKind;
+use nexus_db_model::Probe;
 use nexus_db_model::Project;
 use nexus_db_model::Region;
 use nexus_db_model::RegionSnapshot;
@@ -711,7 +712,25 @@ async fn lookup_service_kind(
     Ok(Some(service_kind))
 }
 
-/// Helper function to look up a project with the given ID.
+/// Helper function to looks up a probe with the given ID.
+async fn lookup_probe(
+    datastore: &DataStore,
+    probe_id: Uuid,
+) -> anyhow::Result<Option<Probe>> {
+    use db::schema::probe::dsl;
+
+    let conn = datastore.pool_connection_for_tests().await?;
+    dsl::probe
+        .filter(dsl::id.eq(probe_id))
+        .limit(1)
+        .select(Probe::as_select())
+        .get_result_async(&*conn)
+        .await
+        .optional()
+        .with_context(|| format!("loading probe {probe_id}"))
+}
+
+/// Helper function to looks up a project with the given ID.
 async fn lookup_project(
     datastore: &DataStore,
     project_id: Uuid,
@@ -2122,6 +2141,28 @@ async fn cmd_db_network_list_vnics(
                     }
                 }
             }
+            NetworkInterfaceKind::Probe => {
+                match lookup_probe(datastore, nic.parent_id).await? {
+                    Some(probe) => {
+                        match lookup_project(datastore, probe.project_id)
+                            .await?
+                        {
+                            Some(project) => (
+                                "probe",
+                                format!("{}/{}", project.name(), probe.name()),
+                            ),
+                            None => {
+                                eprintln!(
+                                    "project with id {} not found",
+                                    probe.project_id
+                                );
+                                continue;
+                            }
+                        }
+                    }
+                    None => ("probe?", "parent probe not found".to_string()),
+                }
+            }
             NetworkInterfaceKind::Service => {
                 // We create service NICs named after the service, so we can use
                 // the nic name instead of looking up the service.
diff --git a/end-to-end-tests/Cargo.toml b/end-to-end-tests/Cargo.toml
index fee32a344e..0fb9efd5cc 100644
--- a/end-to-end-tests/Cargo.toml
+++ b/end-to-end-tests/Cargo.toml
@@ -15,7 +15,7 @@ omicron-sled-agent.workspace = true
 omicron-test-utils.workspace = true
 oxide-client.workspace = true
 rand.workspace = true
-reqwest.workspace = true
+reqwest = { workspace = true, features = ["cookies"] }
 russh = "0.42.0"
 russh-keys = "0.42.0"
 serde.workspace = true
@@ -25,3 +25,12 @@ toml.workspace = true
 trust-dns-resolver.workspace = true
 uuid.workspace = true
 omicron-workspace-hack.workspace = true
+ispf.workspace = true
+internet-checksum.workspace = true
+humantime.workspace = true
+socket2.workspace = true
+colored.workspace = true
+anstyle.workspace = true
+clap.workspace = true
+dhcproto = "0.11"
+macaddr.workspace = true
diff --git a/end-to-end-tests/src/bin/commtest.rs b/end-to-end-tests/src/bin/commtest.rs
new file mode 100644
index 0000000000..27ca4633ce
--- /dev/null
+++ b/end-to-end-tests/src/bin/commtest.rs
@@ -0,0 +1,443 @@
+use anyhow::{anyhow, Result};
+use clap::{Parser, Subcommand};
+use end_to_end_tests::helpers::cli::oxide_cli_style;
+use end_to_end_tests::helpers::icmp::ping4_test_run;
+use oxide_client::{
+    types::{
+        IpPoolCreate, IpPoolLinkSilo, IpRange, Ipv4Range, Name, NameOrId,
+        PingStatus, ProbeCreate, ProbeInfo, ProjectCreate,
+        UsernamePasswordCredentials,
+    },
+    ClientHiddenExt, ClientLoginExt, ClientProjectsExt,
+    ClientSystemHardwareExt, ClientSystemNetworkingExt, ClientSystemStatusExt,
+    ClientVpcsExt,
+};
+use std::{
+    net::{IpAddr, Ipv4Addr},
+    time::{Duration, Instant},
+};
+use tokio::time::sleep;
+use uuid::Uuid;
+
+#[derive(Parser, Debug)]
+#[clap(version, about, long_about = None, styles = oxide_cli_style())]
+struct Cli {
+    /// Oxide API address i.e., http://198.51.100.20
+    oxide_api: String,
+
+    /// How long to wait for the API to become available
+    #[arg(long, default_value = "60m")]
+    api_timeout: humantime::Duration,
+
+    #[clap(subcommand)]
+    command: Commands,
+}
+
+#[derive(Debug, Subcommand)]
+enum Commands {
+    Run(RunArgs),
+    Cleanup,
+}
+
+#[derive(Parser, Debug)]
+struct RunArgs {
+    /// Test Duration
+    #[arg(long, default_value = "100s")]
+    test_duration: humantime::Duration,
+
+    /// Test packet rate in packets per second
+    #[arg(long, default_value_t = 10)]
+    packet_rate: usize,
+
+    /// How many lost ICMP packets may be tolerated
+    #[arg(long, default_value_t = 0)]
+    icmp_loss_tolerance: usize,
+
+    /// First address in the IP pool to use for testing
+    #[arg(long)]
+    ip_pool_begin: Ipv4Addr,
+
+    /// Last address in the IP pool to use for testing
+    #[arg(long)]
+    ip_pool_end: Ipv4Addr,
+}
+
+const API_RETRY_ATTEMPTS: usize = 15;
+
+#[tokio::main]
+pub async fn main() -> Result<()> {
+    let cli = Cli::parse();
+    match cli.command {
+        Commands::Run(ref args) => run(&cli, args).await,
+        Commands::Cleanup => cleanup(&cli).await,
+    }
+}
+
+async fn run(cli: &Cli, args: &RunArgs) -> Result<()> {
+    wait_until_oxide_api_is_available(cli).await?;
+    let (sleds, oxide) = rack_prepare(cli, args).await?;
+    let addrs = launch_probes(sleds, &oxide).await?;
+    test_connectivity(args, addrs)?;
+    Ok(())
+}
+
+async fn cleanup(cli: &Cli) -> Result<()> {
+    wait_until_oxide_api_is_available(cli).await?;
+    let oxide = cleanup_probes(cli).await?;
+    rack_cleanup(&oxide).await?;
+    Ok(())
+}
+
+async fn wait_until_oxide_api_is_available(cli: &Cli) -> Result<()> {
+    let oxide = oxide_client::Client::new(&cli.oxide_api);
+    let start = Instant::now();
+    loop {
+        if let Ok(result) = oxide.ping().send().await.map(|x| x.into_inner()) {
+            if result.status == PingStatus::Ok {
+                println!("the api is up");
+                break;
+            }
+        }
+        if Instant::now().duration_since(start)
+            > Into::<Duration>::into(cli.api_timeout)
+        {
+            return Err(anyhow!(
+                "One hour deadline for system startup exceeded"
+            ));
+        }
+        println!("no api response yet, wating 3s ...");
+        sleep(Duration::from_secs(3)).await;
+    }
+    Ok(())
+}
+
+macro_rules! api_retry {
+    ($call:expr) => {{
+        let mut limit = API_RETRY_ATTEMPTS;
+        loop {
+            match $call {
+                res @ Ok(_) => break res,
+                Err(e) => {
+                    limit -= 1;
+                    if limit == 0 {
+                        break Err(e);
+                    }
+                    println!("API call error: {e}, retrying in 3 s");
+                    sleep(Duration::from_secs(3)).await;
+                }
+            }
+        }
+    }};
+}
+
+async fn cleanup_probes(cli: &Cli) -> Result<oxide_client::Client> {
+    let rqb = reqwest::ClientBuilder::new()
+        .cookie_store(true)
+        .timeout(Duration::from_secs(15))
+        .connect_timeout(Duration::from_secs(15))
+        .build()
+        .unwrap();
+    let oxide = oxide_client::Client::new_with_client(&cli.oxide_api, rqb);
+
+    print!("logging in ... ");
+    api_retry!(
+        oxide
+            .login_local()
+            .silo_name(Name::try_from("recovery").unwrap())
+            .body(UsernamePasswordCredentials {
+                password: "oxide".parse().unwrap(),
+                username: "recovery".parse().unwrap(),
+            })
+            .send()
+            .await
+    )?;
+    println!("done");
+
+    let probes: Vec<ProbeInfo> = api_retry!(
+        oxide
+            .probe_list()
+            .project(Name::try_from("classone").unwrap())
+            .limit(u32::MAX)
+            .send()
+            .await
+    )?
+    .into_inner()
+    .items;
+
+    for probe in &probes {
+        print!("deleting probe {} ... ", *probe.name);
+        api_retry!(
+            oxide
+                .probe_delete()
+                .project(Name::try_from("classone").unwrap())
+                .probe(probe.id)
+                .send()
+                .await
+        )?;
+        println!("done");
+    }
+
+    Ok(oxide)
+}
+
+async fn rack_cleanup(oxide: &oxide_client::Client) -> Result<()> {
+    if let Err(e) = oxide
+        .project_view()
+        .project(Name::try_from("classone").unwrap())
+        .send()
+        .await
+    {
+        if let Some(reqwest::StatusCode::NOT_FOUND) = e.status() {
+            print!("project does not exist");
+        } else {
+            Err(e)?;
+        }
+    } else {
+        print!("deleting classone subnet ... ");
+        api_retry!(
+            oxide
+                .vpc_subnet_delete()
+                .project(Name::try_from("classone").unwrap())
+                .vpc(Name::try_from("default").unwrap())
+                .subnet(Name::try_from("default").unwrap())
+                .send()
+                .await
+        )?;
+        println!("done");
+
+        print!("deleting classone vpc ... ");
+        api_retry!(
+            oxide
+                .vpc_delete()
+                .project(Name::try_from("classone").unwrap())
+                .vpc(Name::try_from("default").unwrap())
+                .send()
+                .await
+        )?;
+        println!("done");
+
+        print!("deleting classone project ... ");
+        api_retry!(
+            oxide
+                .project_delete()
+                .project(Name::try_from("classone").unwrap())
+                .send()
+                .await
+        )?;
+        println!("done");
+    }
+    Ok(())
+}
+
+async fn rack_prepare(
+    cli: &Cli,
+    args: &RunArgs,
+) -> Result<(Vec<Uuid>, oxide_client::Client)> {
+    let rqb = reqwest::ClientBuilder::new().cookie_store(true).build().unwrap();
+
+    let oxide = oxide_client::Client::new_with_client(&cli.oxide_api, rqb);
+
+    print!("logging in ... ");
+    api_retry!(
+        oxide
+            .login_local()
+            .silo_name(Name::try_from("recovery").unwrap())
+            .body(UsernamePasswordCredentials {
+                password: "oxide".parse().unwrap(),
+                username: "recovery".parse().unwrap(),
+            })
+            .send()
+            .await
+    )?;
+    println!("done");
+
+    api_retry!(if let Err(e) = oxide
+        .project_view()
+        .project(Name::try_from("classone").unwrap())
+        .send()
+        .await
+    {
+        if let Some(reqwest::StatusCode::NOT_FOUND) = e.status() {
+            print!("project does not exist, creating ... ");
+            oxide
+                .project_create()
+                .body(ProjectCreate {
+                    description: "A project for probes".into(),
+                    name: "classone".parse().unwrap(),
+                })
+                .send()
+                .await?;
+            println!("done");
+            Ok(())
+        } else {
+            Err(e)
+        }
+    } else {
+        println!("classone project already exists");
+        Ok(())
+    })?;
+
+    let pool_name = "default";
+    api_retry!(
+        if let Err(e) = oxide.ip_pool_view().pool("default").send().await {
+            if let Some(reqwest::StatusCode::NOT_FOUND) = e.status() {
+                print!("default ip pool does not exist, creating ...");
+                oxide
+                    .ip_pool_create()
+                    .body(IpPoolCreate {
+                        name: pool_name.parse().unwrap(),
+                        description: "Default IP pool".to_string(),
+                    })
+                    .send()
+                    .await?;
+                oxide
+                    .ip_pool_silo_link()
+                    .pool(pool_name)
+                    .body(IpPoolLinkSilo {
+                        silo: NameOrId::Name("recovery".parse().unwrap()),
+                        is_default: true,
+                    })
+                    .send()
+                    .await?;
+                println!("done");
+                Ok(())
+            } else {
+                Err(e)
+            }
+        } else {
+            println!("default ip pool already exists");
+            Ok(())
+        }
+    )?;
+
+    let pool = api_retry!(
+        oxide
+            .ip_pool_range_list()
+            .limit(u32::MAX)
+            .pool(Name::try_from("default").unwrap())
+            .send()
+            .await
+    )?
+    .into_inner()
+    .items;
+
+    let range = Ipv4Range { first: args.ip_pool_begin, last: args.ip_pool_end };
+
+    let range_exists = pool
+        .iter()
+        .filter_map(|x| match &x.range {
+            IpRange::V4(r) => Some(r),
+            IpRange::V6(_) => None,
+        })
+        .any(|x| x.first == range.first && x.last == range.last);
+
+    if !range_exists {
+        print!("ip range does not exist, creating ... ");
+        api_retry!(
+            oxide
+                .ip_pool_range_add()
+                .pool(Name::try_from("default").unwrap())
+                .body(IpRange::V4(range.clone()))
+                .send()
+                .await
+        )?;
+        println!("done");
+    } else {
+        println!("ip range already exists");
+    }
+
+    print!("getting sled ids ... ");
+    let sleds = api_retry!(oxide.sled_list().limit(u32::MAX).send().await)?
+        .into_inner()
+        .items
+        .iter()
+        .map(|x| x.id)
+        .collect();
+    println!("done");
+
+    Ok((sleds, oxide))
+}
+
+async fn launch_probes(
+    sleds: Vec<Uuid>,
+    oxide: &oxide_client::Client,
+) -> Result<Vec<Ipv4Addr>> {
+    for (i, sled) in sleds.into_iter().enumerate() {
+        println!("checking if probe{i} exists");
+        api_retry!(if let Err(e) = oxide
+            .probe_view()
+            .project(Name::try_from("classone").unwrap())
+            .probe(Name::try_from(format!("probe{i}")).unwrap())
+            .send()
+            .await
+        {
+            if let Some(reqwest::StatusCode::NOT_FOUND) = e.status() {
+                print!("probe{i} does not exist, creating ... ");
+                oxide
+                    .probe_create()
+                    .project(Name::try_from("classone").unwrap())
+                    .body(ProbeCreate {
+                        description: format!("probe {i}"),
+                        ip_pool: Some("default".parse().unwrap()),
+                        name: format!("probe{i}").parse().unwrap(),
+                        sled,
+                    })
+                    .send()
+                    .await?;
+                println!("done");
+                Ok(())
+            } else {
+                Err(e)
+            }
+        } else {
+            println!("probe{i} already exists");
+            Ok(())
+        })?;
+    }
+
+    Ok(api_retry!(
+        oxide
+            .probe_list()
+            .project(Name::try_from("classone").unwrap())
+            .limit(u32::MAX)
+            .send()
+            .await
+    )?
+    .into_inner()
+    .items
+    .iter()
+    .map(|x| x.external_ips.get(0).unwrap().ip)
+    .filter_map(|x| match x {
+        IpAddr::V4(ip) => Some(ip),
+        IpAddr::V6(_) => None,
+    })
+    .collect())
+}
+
+fn test_connectivity(args: &RunArgs, addrs: Vec<Ipv4Addr>) -> Result<()> {
+    let ttl = 255;
+    println!("testing connectivity to probes");
+    let report = ping4_test_run(
+        &addrs,
+        ttl,
+        args.packet_rate,
+        args.test_duration.into(),
+    );
+
+    let out = serde_json::to_string_pretty(&report).unwrap();
+    std::fs::write("connectivity-report.json", out.as_str()).unwrap();
+
+    for state in report.v4.iter() {
+        if state.lost > args.icmp_loss_tolerance {
+            panic!(
+                "{} has loss = {} packets which is greater than tolerance {}",
+                state.dest, state.lost, args.icmp_loss_tolerance,
+            );
+        }
+        if state.rx_count == 0 {
+            panic!("received no responses from {}", state.dest);
+        }
+    }
+    println!("all connectivity tests within loss tolerance");
+    Ok(())
+}
diff --git a/end-to-end-tests/src/bin/dhcp-server.rs b/end-to-end-tests/src/bin/dhcp-server.rs
new file mode 100644
index 0000000000..69681fa413
--- /dev/null
+++ b/end-to-end-tests/src/bin/dhcp-server.rs
@@ -0,0 +1,125 @@
+//! This is a dirt simple DHCP server for handing out addresses in a given
+//! range. Leases do not expire. If the server runs out of addresses, it
+//! panics. This is a stopgap program to hand out addresses to VMs in CI. It's
+//! in no way meant to be a generic DHCP server solution.
+
+use anyhow::Result;
+use clap::Parser;
+use dhcproto::{
+    v4::{
+        self, Decodable, Decoder, DhcpOptions, Encodable, Message, Opcode,
+        OptionCode,
+    },
+    Encoder,
+};
+use end_to_end_tests::helpers::cli::oxide_cli_style;
+use macaddr::MacAddr6;
+use std::{
+    collections::HashMap,
+    net::{Ipv4Addr, SocketAddrV4, UdpSocket},
+};
+
+#[derive(Parser, Debug)]
+#[clap(version, about, long_about = None, styles = oxide_cli_style())]
+struct Cli {
+    /// First address in DHCP range.
+    begin: Ipv4Addr,
+    /// Last address in DHCP range.
+    end: Ipv4Addr,
+    /// Default router to advertise.
+    router: Ipv4Addr,
+    /// Server address to advertise.
+    server: Ipv4Addr,
+}
+
+pub fn main() -> Result<()> {
+    let cli = Cli::parse();
+    let mut current = cli.begin;
+    let mut assignments = HashMap::<MacAddr6, Ipv4Addr>::new();
+
+    let sock = UdpSocket::bind("0.0.0.0:67")?;
+    loop {
+        let mut buf = [0; 8192];
+        let (n, src) = sock.recv_from(&mut buf)?;
+
+        let mut msg = match Message::decode(&mut Decoder::new(&buf[..n])) {
+            Ok(msg) => msg,
+            Err(e) => {
+                eprintln!("message decode error {e}");
+                continue;
+            }
+        };
+
+        println!("request: {msg:#?}");
+
+        if msg.opcode() != Opcode::BootRequest {
+            continue;
+        }
+
+        let mac: [u8; 6] = msg.chaddr()[0..6].try_into().unwrap();
+        let mac = MacAddr6::from(mac);
+
+        let ip = match assignments.get(&mac) {
+            Some(ip) => *ip,
+            None => {
+                assignments.insert(mac, current);
+                let ip = current;
+                current = Ipv4Addr::from(u32::from(current) + 1);
+                if u32::from(current) > u32::from(cli.end) {
+                    panic!("address exhaustion");
+                }
+                ip
+            }
+        };
+
+        let mut opts = DhcpOptions::new();
+        match msg.opts().get(OptionCode::MessageType) {
+            Some(v4::DhcpOption::MessageType(v4::MessageType::Discover)) => {
+                opts.insert(v4::DhcpOption::MessageType(
+                    v4::MessageType::Offer,
+                ));
+            }
+            Some(v4::DhcpOption::MessageType(v4::MessageType::Request)) => {
+                opts.insert(v4::DhcpOption::MessageType(v4::MessageType::Ack));
+            }
+            Some(mtype) => eprintln!("unexpected message type {mtype:?}"),
+            None => {
+                eprintln!("no message type");
+            }
+        };
+        // hardcode to /24
+        opts.insert(v4::DhcpOption::SubnetMask(Ipv4Addr::new(
+            255, 255, 255, 0,
+        )));
+        // hardcode to something stable
+        opts.insert(v4::DhcpOption::DomainNameServer(vec![Ipv4Addr::new(
+            1, 1, 1, 1,
+        )]));
+        opts.insert(v4::DhcpOption::ServerIdentifier(cli.server));
+        // just something big enough to last CI runs
+        opts.insert(v4::DhcpOption::AddressLeaseTime(60 * 60 * 24 * 30));
+        opts.insert(v4::DhcpOption::Router(vec![cli.router]));
+        if let Some(opt) = msg.opts().get(OptionCode::ClientIdentifier) {
+            opts.insert(opt.clone());
+        }
+
+        msg.set_opcode(Opcode::BootReply);
+        msg.set_siaddr(cli.server);
+        msg.set_yiaddr(ip);
+        msg.set_opts(opts);
+
+        let mut buf = Vec::new();
+        let mut e = Encoder::new(&mut buf);
+        if let Err(e) = msg.encode(&mut e) {
+            eprintln!("encode reply error: {e}");
+            continue;
+        }
+
+        // always blast replys bcast
+        let dst =
+            SocketAddrV4::new(Ipv4Addr::new(255, 255, 255, 255), src.port());
+        if let Err(e) = sock.send_to(&buf, dst) {
+            eprintln!("send reply error: {e}");
+        }
+    }
+}
diff --git a/end-to-end-tests/src/helpers/cli.rs b/end-to-end-tests/src/helpers/cli.rs
new file mode 100644
index 0000000000..049ab5a8b7
--- /dev/null
+++ b/end-to-end-tests/src/helpers/cli.rs
@@ -0,0 +1,22 @@
+/// An Oxide color theme for your clap-based CLIs
+pub fn oxide_cli_style() -> clap::builder::Styles {
+    clap::builder::Styles::styled()
+        .header(anstyle::Style::new().bold().underline().fg_color(Some(
+            anstyle::Color::Rgb(anstyle::RgbColor(245, 207, 101)),
+        )))
+        .literal(anstyle::Style::new().bold().fg_color(Some(
+            anstyle::Color::Rgb(anstyle::RgbColor(72, 213, 151)),
+        )))
+        .invalid(anstyle::Style::new().bold().fg_color(Some(
+            anstyle::Color::Rgb(anstyle::RgbColor(72, 213, 151)),
+        )))
+        .valid(anstyle::Style::new().bold().fg_color(Some(
+            anstyle::Color::Rgb(anstyle::RgbColor(72, 213, 151)),
+        )))
+        .usage(anstyle::Style::new().bold().fg_color(Some(
+            anstyle::Color::Rgb(anstyle::RgbColor(245, 207, 101)),
+        )))
+        .error(anstyle::Style::new().bold().fg_color(Some(
+            anstyle::Color::Rgb(anstyle::RgbColor(232, 104, 134)),
+        )))
+}
diff --git a/end-to-end-tests/src/helpers/icmp.rs b/end-to-end-tests/src/helpers/icmp.rs
new file mode 100644
index 0000000000..42617801d8
--- /dev/null
+++ b/end-to-end-tests/src/helpers/icmp.rs
@@ -0,0 +1,306 @@
+use colored::*;
+use internet_checksum::Checksum;
+use serde::{Deserialize, Serialize};
+use socket2::{Domain, Protocol, SockAddr, Socket, Type};
+use std::collections::BTreeMap;
+use std::mem::MaybeUninit;
+use std::net::{Ipv4Addr, SocketAddrV4};
+use std::sync::{Arc, Mutex};
+use std::thread::{sleep, spawn};
+use std::time::{Duration, Instant};
+
+const HIDE_CURSOR: &str = "\x1b[?25l";
+const SHOW_CURSOR: &str = "\x1b[?25h";
+const MOVE_CURSOR_UP: &str = "\x1b[A";
+
+const ICMP_ECHO_TYPE: u8 = 8;
+const ICMP_ECHO_CODE: u8 = 0;
+
+#[derive(Debug, Serialize, Deserialize)]
+struct EchoRequest {
+    typ: u8,
+    code: u8,
+    checksum: u16,
+    identifier: u16,
+    sequence_number: u16,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Report {
+    pub v4: Vec<Ping4State>,
+}
+
+/// Run a ping test against the provided destination addreses, with the
+/// specified time-to-live (ttl) at a given rate in packets per second
+/// (pps) for the specified duration.
+pub fn ping4_test_run(
+    dst: &[Ipv4Addr],
+    ttl: u32,
+    pps: usize,
+    duration: Duration,
+) -> Report {
+    let p = Pinger4::new(ttl);
+    for dst in dst {
+        // use a random number for the ICMP id
+        p.add_target(rand::random(), *dst, pps, duration);
+    }
+    // Use an ASCII code to hide the blinking cursor as it makes the output hard
+    // to read.
+    print!("{HIDE_CURSOR}");
+    p.clone().show();
+    // wait for the test to conclude plus a bit of buffer time for packets in
+    // flight.
+    sleep(duration + Duration::from_millis(250));
+    for _ in 0..p.targets.lock().unwrap().len() {
+        println!();
+    }
+    // turn the blinky cursor back on
+    print!("{SHOW_CURSOR}");
+
+    // return a report to the caller
+    let v4 = p.targets.lock().unwrap().values().copied().collect();
+    Report { v4 }
+}
+
+struct Pinger4 {
+    sock: Socket,
+    targets: Mutex<BTreeMap<u16, Ping4State>>,
+}
+
+/// Running results for an IPv4 ping test.
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct Ping4State {
+    /// Destination address of the ping test.
+    pub dest: Ipv4Addr,
+    /// Low water mark for ping round trip times.
+    pub low: Duration,
+    /// High water mark for ping round trip times.
+    pub high: Duration,
+    /// Summation of ping round trip times.
+    pub sum: Duration,
+    /// The last recorded ping round trip time.
+    pub current: Option<Duration>,
+    /// The number of ICMP packets considered lost. Does not start ticking
+    /// until at least one reply has been received.
+    pub lost: usize,
+    /// The number of packets sent.
+    pub tx_count: u16,
+    /// The number of packets received.
+    pub rx_count: u16,
+    /// The last time a packet was sent.
+    #[serde(skip)]
+    pub sent: Option<Instant>,
+    /// The transmit counter value when we received the first reply.
+    #[serde(skip)]
+    pub first: u16,
+}
+
+impl Ping4State {
+    fn new(addr: Ipv4Addr) -> Self {
+        Self {
+            dest: addr,
+            low: Duration::default(),
+            high: Duration::default(),
+            sum: Duration::default(),
+            current: None,
+            lost: 0,
+            tx_count: 0,
+            rx_count: 0,
+            sent: None,
+            first: 0,
+        }
+    }
+}
+
+impl Pinger4 {
+    fn new(ttl: u32) -> Arc<Self> {
+        let sock = Socket::new(Domain::IPV4, Type::RAW, Some(Protocol::ICMPV4))
+            .unwrap();
+        sock.set_ttl(ttl).unwrap();
+        let s = Arc::new(Self { sock, targets: Mutex::new(BTreeMap::new()) });
+        s.clone().rx();
+        s.clone().count_lost();
+        s
+    }
+
+    fn show(self: Arc<Self>) {
+        println!(
+            "{:15} {:7} {:7} {:7} {:7} {:7} {:7} {}",
+            "addr".dimmed(),
+            "low".dimmed(),
+            "avg".dimmed(),
+            "high".dimmed(),
+            "last".dimmed(),
+            "sent".dimmed(),
+            "received".dimmed(),
+            "lost".dimmed()
+        );
+        // run the reporting on a background thread
+        spawn(move || loop {
+            // print a status line for each target
+            for (_id, t) in self.targets.lock().unwrap().iter() {
+                println!(
+                    "{:15} {:7} {:7} {:7} {:7} {:7} {:7} {:<7}",
+                    t.dest.to_string().cyan(),
+                    format!("{:.3}", (t.low.as_micros() as f32 / 1000.0)),
+                    if t.rx_count == 0 {
+                        format!("{:.3}", 0.0)
+                    } else {
+                        format!(
+                            "{:.3}",
+                            (t.sum.as_micros() as f32
+                                / 1000.0
+                                / t.rx_count as f32)
+                        )
+                    },
+                    format!("{:.3}", (t.high.as_micros() as f32 / 1000.0)),
+                    match t.current {
+                        Some(dt) =>
+                            format!("{:.3}", (dt.as_micros() as f32 / 1000.0)),
+                        None => format!("{:.3}", 0.0),
+                    },
+                    t.tx_count.to_string(),
+                    t.rx_count.to_string(),
+                    if t.lost == 0 {
+                        t.lost.to_string().green()
+                    } else {
+                        t.lost.to_string().red()
+                    },
+                );
+            }
+            // move the cursor back to the top for another round of reporting
+            for _ in 0..self.targets.lock().unwrap().len() {
+                print!("{MOVE_CURSOR_UP}");
+            }
+            print!("\r");
+
+            sleep(Duration::from_millis(100));
+        });
+    }
+
+    fn add_target(
+        self: &Arc<Self>,
+        id: u16,
+        addr: Ipv4Addr,
+        pps: usize,
+        duration: Duration,
+    ) {
+        self.targets.lock().unwrap().insert(id, Ping4State::new(addr));
+        let interval = Duration::from_secs_f64(1.0 / pps as f64);
+        self.clone().tx(id, addr, interval, duration);
+    }
+
+    fn tx(
+        self: Arc<Self>,
+        id: u16,
+        dst: Ipv4Addr,
+        interval: Duration,
+        duration: Duration,
+    ) {
+        let mut seq = 0u16;
+        let stop = Instant::now() + duration;
+        // send ICMP test packets on a background thread
+        spawn(move || loop {
+            if Instant::now() >= stop {
+                break;
+            }
+            let mut c = Checksum::new();
+            c.add_bytes(&[ICMP_ECHO_TYPE, ICMP_ECHO_CODE]);
+            c.add_bytes(&id.to_be_bytes());
+            c.add_bytes(&seq.to_be_bytes());
+            let pkt = EchoRequest {
+                typ: ICMP_ECHO_TYPE,
+                code: ICMP_ECHO_CODE,
+                checksum: u16::from_be_bytes(c.checksum()),
+                identifier: id,
+                sequence_number: seq,
+            };
+            let msg = ispf::to_bytes_be(&pkt).unwrap();
+
+            match self.targets.lock().unwrap().get_mut(&id) {
+                Some(ref mut tgt) => {
+                    tgt.sent = Some(Instant::now());
+                    tgt.tx_count = seq;
+                    let sa: SockAddr = SocketAddrV4::new(dst, 0).into();
+                    self.sock.send_to(&msg, &sa).unwrap();
+                }
+                None => continue,
+            }
+
+            seq += 1;
+            sleep(interval);
+        });
+    }
+
+    // At the end of the day this is not strictly necessary for the final
+    // report. But it's really nice for interactive use to have a live
+    // ticker for lost packet count.
+    fn count_lost(self: Arc<Self>) {
+        spawn(move || loop {
+            for (_, tgt) in self.targets.lock().unwrap().iter_mut() {
+                // Only start considering packets lost after the first packet
+                // is received. This allows the remote endpoint time to come
+                // online without considering initial packets lost while it's
+                // coming up.
+                if tgt.first != 0 {
+                    tgt.lost = tgt
+                        .tx_count
+                        .saturating_sub(tgt.first)
+                        .saturating_sub(tgt.rx_count)
+                        as usize;
+                }
+            }
+            sleep(Duration::from_millis(10));
+        });
+    }
+
+    fn rx(self: Arc<Self>) {
+        // Spawn a background thread to receive ICMP replies and do the
+        // necessary accounting.
+        spawn(move || loop {
+            let mut ubuf = [MaybeUninit::new(0); 10240];
+            if let Ok((sz, _)) = self.sock.recv_from(&mut ubuf) {
+                let buf = unsafe { &slice_assume_init_ref(&ubuf[..sz]) };
+                let msg: EchoRequest = match ispf::from_bytes_be(&buf[20..sz]) {
+                    Ok(msg) => msg,
+                    Err(_) => {
+                        continue;
+                    }
+                };
+                // correlate the ICMP id with a target
+                match self.targets.lock().unwrap().get_mut(&msg.identifier) {
+                    Some(ref mut target) => match target.sent {
+                        Some(ref mut sent) => {
+                            let t1 = Instant::now();
+                            let dt = t1 - *sent;
+                            target.current = Some(dt);
+                            if target.low == Duration::ZERO || dt < target.low {
+                                target.low = dt;
+                            }
+                            if dt > target.high {
+                                target.high = dt;
+                            }
+                            target.sum += dt;
+                            target.current = Some(dt);
+                            target.rx_count += 1;
+                            if target.first == 0 {
+                                target.first = target.tx_count;
+                            }
+                        }
+                        None => {
+                            println!("no sent");
+                        }
+                    },
+                    None => {
+                        println!("no target {}", msg.identifier);
+                    }
+                }
+            }
+        });
+    }
+}
+
+// TODO: Use `MaybeUninit::slice_assume_init_ref` once it has stabilized
+unsafe fn slice_assume_init_ref<T>(slice: &[MaybeUninit<T>]) -> &[T] {
+    unsafe { &*(slice as *const [MaybeUninit<T>] as *const [T]) }
+}
diff --git a/end-to-end-tests/src/helpers/mod.rs b/end-to-end-tests/src/helpers/mod.rs
index db03973555..b7cd6d5574 100644
--- a/end-to-end-tests/src/helpers/mod.rs
+++ b/end-to-end-tests/src/helpers/mod.rs
@@ -1,4 +1,6 @@
+pub mod cli;
 pub mod ctx;
+pub mod icmp;
 
 use self::ctx::nexus_addr;
 use anyhow::{bail, Result};
diff --git a/end-to-end-tests/src/instance_launch.rs b/end-to-end-tests/src/instance_launch.rs
index f27261e82d..1aae46fe98 100644
--- a/end-to-end-tests/src/instance_launch.rs
+++ b/end-to-end-tests/src/instance_launch.rs
@@ -92,6 +92,7 @@ async fn instance_launch() -> Result<()> {
         .first()
         .context("no external IPs")?
         .clone();
+
     let ExternalIp::Ephemeral { ip: ip_addr } = ip_addr else {
         anyhow::bail!("IP bound to instance was not ephemeral as required.")
     };
diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs
index 02302347cd..0dd8f85e4e 100644
--- a/illumos-utils/src/running_zone.rs
+++ b/illumos-utils/src/running_zone.rs
@@ -912,10 +912,16 @@ impl RunningZone {
         Ok(())
     }
 
+    /// Return a reference to the links for this zone.
     pub fn links(&self) -> &Vec<Link> {
         &self.inner.links
     }
 
+    /// Return a mutable reference to the links for this zone.
+    pub fn links_mut(&mut self) -> &mut Vec<Link> {
+        &mut self.inner.links
+    }
+
     /// Return the running processes associated with all the SMF services this
     /// zone is intended to run.
     pub fn service_processes(
diff --git a/installinator/src/dispatch.rs b/installinator/src/dispatch.rs
index 9bec14664c..1fcf351a9b 100644
--- a/installinator/src/dispatch.rs
+++ b/installinator/src/dispatch.rs
@@ -151,13 +151,13 @@ struct InstallOpts {
     #[clap(long)]
     install_on_gimlet: bool,
 
-    //TODO(ry) this probably needs to get plumbed somewhere instead of relying
+    //TODO this probably needs to get plumbed somewhere instead of relying
     //on a default.
     /// The first gimlet data link to use.
     #[clap(long, default_value = "cxgbe0")]
     data_link0: String,
 
-    //TODO(ry) this probably needs to get plumbed somewhere instead of relying
+    //TODO this probably needs to get plumbed somewhere instead of relying
     //on a default.
     /// The second gimlet data link to use.
     #[clap(long, default_value = "cxgbe1")]
diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml
index 45d1c34382..0712ba6743 100644
--- a/nexus/Cargo.toml
+++ b/nexus/Cargo.toml
@@ -35,6 +35,7 @@ hyper.workspace = true
 illumos-utils.workspace = true
 internal-dns.workspace = true
 ipnetwork.workspace = true
+itertools.workspace = true
 macaddr.workspace = true
 mime_guess.workspace = true
 # Not under "dev-dependencies"; these also need to be implemented for
@@ -104,7 +105,6 @@ diesel.workspace = true
 dns-server.workspace = true
 expectorate.workspace = true
 hyper-rustls.workspace = true
-itertools.workspace = true
 gateway-messages.workspace = true
 gateway-test-utils.workspace = true
 hubtools.workspace = true
diff --git a/nexus/db-model/src/deployment.rs b/nexus/db-model/src/deployment.rs
index a1f285fbef..e9a650812b 100644
--- a/nexus/db-model/src/deployment.rs
+++ b/nexus/db-model/src/deployment.rs
@@ -15,6 +15,7 @@ use crate::{ipv6, Generation, MacAddr, Name, SqlU16, SqlU32, SqlU8};
 use chrono::{DateTime, Utc};
 use ipnetwork::IpNetwork;
 use nexus_types::deployment::BlueprintTarget;
+use omicron_common::api::internal::shared::NetworkInterface;
 use uuid::Uuid;
 
 /// See [`nexus_types::deployment::Blueprint`].
@@ -249,7 +250,7 @@ impl BpOmicronZoneNic {
     pub fn into_network_interface_for_zone(
         self,
         zone_id: Uuid,
-    ) -> Result<nexus_types::inventory::NetworkInterface, anyhow::Error> {
+    ) -> Result<NetworkInterface, anyhow::Error> {
         let zone_nic = OmicronZoneNic::from(self);
         zone_nic.into_network_interface_for_zone(zone_id)
     }
diff --git a/nexus/db-model/src/external_ip.rs b/nexus/db-model/src/external_ip.rs
index 0f484f7610..337e7ef2a7 100644
--- a/nexus/db-model/src/external_ip.rs
+++ b/nexus/db-model/src/external_ip.rs
@@ -22,6 +22,7 @@ use nexus_types::external_api::views;
 use omicron_common::address::NUM_SOURCE_NAT_PORTS;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::IdentityMetadata;
+use schemars::JsonSchema;
 use serde::Deserialize;
 use serde::Serialize;
 use sled_agent_client::types::InstanceExternalIpBody;
@@ -34,7 +35,7 @@ impl_enum_type!(
     #[diesel(postgres_type(name = "ip_kind", schema = "public"))]
      pub struct IpKindEnum;
 
-     #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Deserialize, Serialize)]
+     #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
      #[diesel(sql_type = IpKindEnum)]
      pub enum IpKind;
 
@@ -48,7 +49,7 @@ impl_enum_type!(
     #[diesel(postgres_type(name = "ip_attach_state"))]
      pub struct IpAttachStateEnum;
 
-     #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Deserialize, Serialize)]
+     #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Deserialize, Serialize, JsonSchema)]
      #[diesel(sql_type = IpAttachStateEnum)]
      pub enum IpAttachState;
 
@@ -95,10 +96,11 @@ impl std::fmt::Display for IpKind {
     Selectable,
     Queryable,
     Insertable,
-    Deserialize,
-    Serialize,
     PartialEq,
     Eq,
+    Serialize,
+    Deserialize,
+    JsonSchema,
 )]
 #[diesel(table_name = external_ip)]
 pub struct ExternalIp {
@@ -125,6 +127,7 @@ pub struct ExternalIp {
     // Only Some(_) for instance Floating IPs
     pub project_id: Option<Uuid>,
     pub state: IpAttachState,
+    pub is_probe: bool,
 }
 
 /// A view type constructed from `ExternalIp` used to represent Floating IP
@@ -171,6 +174,7 @@ pub struct IncompleteExternalIp {
     time_created: DateTime<Utc>,
     kind: IpKind,
     is_service: bool,
+    is_probe: bool,
     parent_id: Option<Uuid>,
     pool_id: Uuid,
     project_id: Option<Uuid>,
@@ -195,6 +199,7 @@ impl IncompleteExternalIp {
             time_created: Utc::now(),
             kind,
             is_service: false,
+            is_probe: false,
             parent_id: Some(instance_id),
             pool_id,
             project_id: None,
@@ -214,6 +219,30 @@ impl IncompleteExternalIp {
             kind,
             is_service: false,
             parent_id: None,
+            is_probe: false,
+            pool_id,
+            project_id: None,
+            explicit_ip: None,
+            explicit_port_range: None,
+            state: kind.initial_state(),
+        }
+    }
+
+    pub fn for_ephemeral_probe(
+        id: Uuid,
+        instance_id: Uuid,
+        pool_id: Uuid,
+    ) -> Self {
+        let kind = IpKind::Ephemeral;
+        Self {
+            id,
+            name: None,
+            description: None,
+            time_created: Utc::now(),
+            kind: IpKind::Ephemeral,
+            is_service: false,
+            is_probe: true,
+            parent_id: Some(instance_id),
             pool_id,
             project_id: None,
             explicit_ip: None,
@@ -237,6 +266,7 @@ impl IncompleteExternalIp {
             time_created: Utc::now(),
             kind,
             is_service: false,
+            is_probe: false,
             parent_id: None,
             pool_id,
             project_id: Some(project_id),
@@ -262,6 +292,7 @@ impl IncompleteExternalIp {
             time_created: Utc::now(),
             kind,
             is_service: false,
+            is_probe: false,
             parent_id: None,
             pool_id,
             project_id: Some(project_id),
@@ -286,6 +317,7 @@ impl IncompleteExternalIp {
             time_created: Utc::now(),
             kind: IpKind::Floating,
             is_service: true,
+            is_probe: false,
             parent_id: Some(service_id),
             pool_id,
             project_id: None,
@@ -317,6 +349,7 @@ impl IncompleteExternalIp {
             time_created: Utc::now(),
             kind,
             is_service: true,
+            is_probe: false,
             parent_id: Some(service_id),
             pool_id,
             project_id: None,
@@ -341,6 +374,7 @@ impl IncompleteExternalIp {
             time_created: Utc::now(),
             kind,
             is_service: true,
+            is_probe: false,
             parent_id: Some(service_id),
             pool_id,
             project_id: None,
@@ -359,6 +393,7 @@ impl IncompleteExternalIp {
             time_created: Utc::now(),
             kind,
             is_service: true,
+            is_probe: false,
             parent_id: Some(service_id),
             pool_id,
             project_id: None,
@@ -392,6 +427,10 @@ impl IncompleteExternalIp {
         &self.is_service
     }
 
+    pub fn is_probe(&self) -> &bool {
+        &self.is_probe
+    }
+
     pub fn parent_id(&self) -> &Option<Uuid> {
         &self.parent_id
     }
diff --git a/nexus/db-model/src/inventory.rs b/nexus/db-model/src/inventory.rs
index 0992eb60b5..2f69fd998c 100644
--- a/nexus/db-model/src/inventory.rs
+++ b/nexus/db-model/src/inventory.rs
@@ -28,6 +28,7 @@ use ipnetwork::IpNetwork;
 use nexus_types::inventory::{
     BaseboardId, Caboose, Collection, PowerState, RotPage, RotSlot,
 };
+use omicron_common::api::internal::shared::NetworkInterface;
 use uuid::Uuid;
 
 // See [`nexus_types::inventory::PowerState`].
@@ -872,7 +873,7 @@ impl InvOmicronZoneNic {
     pub fn into_network_interface_for_zone(
         self,
         zone_id: Uuid,
-    ) -> Result<nexus_types::inventory::NetworkInterface, anyhow::Error> {
+    ) -> Result<NetworkInterface, anyhow::Error> {
         let zone_nic = OmicronZoneNic::from(self);
         zone_nic.into_network_interface_for_zone(zone_id)
     }
diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs
index 07c9f5eec6..5c89134b78 100644
--- a/nexus/db-model/src/lib.rs
+++ b/nexus/db-model/src/lib.rs
@@ -45,6 +45,7 @@ mod network_interface;
 mod oximeter_info;
 mod physical_disk;
 mod physical_disk_kind;
+mod probe;
 mod producer_endpoint;
 mod project;
 mod semver_version;
@@ -145,6 +146,7 @@ pub use network_interface::*;
 pub use oximeter_info::*;
 pub use physical_disk::*;
 pub use physical_disk_kind::*;
+pub use probe::*;
 pub use producer_endpoint::*;
 pub use project::*;
 pub use quota::*;
diff --git a/nexus/db-model/src/network_interface.rs b/nexus/db-model/src/network_interface.rs
index 72752ae3f8..fdcfcbf588 100644
--- a/nexus/db-model/src/network_interface.rs
+++ b/nexus/db-model/src/network_interface.rs
@@ -14,7 +14,7 @@ use db_macros::Resource;
 use diesel::AsChangeset;
 use nexus_types::external_api::params;
 use nexus_types::identity::Resource;
-use omicron_common::api::external;
+use omicron_common::api::{external, internal};
 use uuid::Uuid;
 
 /// The max number of interfaces that may be associated with a resource,
@@ -35,6 +35,7 @@ impl_enum_type! {
 
     Instance => b"instance"
     Service => b"service"
+    Probe => b"probe"
 }
 
 /// Generic Network Interface DB model.
@@ -63,6 +64,41 @@ pub struct NetworkInterface {
     pub primary: bool,
 }
 
+impl NetworkInterface {
+    pub fn into_internal(
+        self,
+        subnet: external::IpNet,
+    ) -> internal::shared::NetworkInterface {
+        internal::shared::NetworkInterface {
+            id: self.id(),
+            kind: match self.kind {
+                NetworkInterfaceKind::Instance => {
+                    internal::shared::NetworkInterfaceKind::Instance {
+                        id: self.parent_id,
+                    }
+                }
+                NetworkInterfaceKind::Service => {
+                    internal::shared::NetworkInterfaceKind::Service {
+                        id: self.parent_id,
+                    }
+                }
+                NetworkInterfaceKind::Probe => {
+                    internal::shared::NetworkInterfaceKind::Probe {
+                        id: self.parent_id,
+                    }
+                }
+            },
+            name: self.name().clone(),
+            ip: self.ip.ip(),
+            mac: self.mac.into(),
+            subnet: subnet,
+            vni: external::Vni::try_from(0).unwrap(),
+            primary: self.primary,
+            slot: self.slot.try_into().unwrap(),
+        }
+    }
+}
+
 /// Instance Network Interface DB model.
 ///
 /// The underlying "table" (`instance_network_interface`) is actually a view
@@ -244,6 +280,13 @@ impl IncompleteNetworkInterface {
                         )));
                     }
                 }
+                NetworkInterfaceKind::Probe => {
+                    if !mac.is_guest() {
+                        return Err(external::Error::invalid_request(format!(
+                            "invalid MAC address {mac} for probe NIC",
+                        )));
+                    }
+                }
                 NetworkInterfaceKind::Service => {
                     if !mac.is_system() {
                         return Err(external::Error::invalid_request(format!(
@@ -312,6 +355,26 @@ impl IncompleteNetworkInterface {
             Some(slot),
         )
     }
+
+    pub fn new_probe(
+        interface_id: Uuid,
+        probe_id: Uuid,
+        subnet: VpcSubnet,
+        identity: external::IdentityMetadataCreateParams,
+        ip: Option<std::net::IpAddr>,
+        mac: Option<external::MacAddr>,
+    ) -> Result<Self, external::Error> {
+        Self::new(
+            interface_id,
+            NetworkInterfaceKind::Probe,
+            probe_id,
+            subnet,
+            identity,
+            ip,
+            mac,
+            None,
+        )
+    }
 }
 
 /// Describes a set of updates for the [`NetworkInterface`] model.
diff --git a/nexus/db-model/src/omicron_zone_config.rs b/nexus/db-model/src/omicron_zone_config.rs
index f4726ccd92..ce3127a9b3 100644
--- a/nexus/db-model/src/omicron_zone_config.rs
+++ b/nexus/db-model/src/omicron_zone_config.rs
@@ -18,6 +18,9 @@ use crate::{ipv6, MacAddr, Name, SqlU16, SqlU32, SqlU8};
 use anyhow::{anyhow, bail, ensure, Context};
 use ipnetwork::IpNetwork;
 use nexus_types::inventory::OmicronZoneType;
+use omicron_common::api::internal::shared::{
+    NetworkInterface, NetworkInterfaceKind,
+};
 use uuid::Uuid;
 
 #[derive(Debug)]
@@ -410,9 +413,7 @@ impl OmicronZoneNic {
                 ensure!(
                     matches!(
                         nic.kind,
-                        nexus_types::inventory::NetworkInterfaceKind::Service(
-                            id
-                        ) if id == zone.id
+                        NetworkInterfaceKind::Service{ id } if id == zone.id
                     ),
                     "expected zone's NIC kind to be \"service\" and the \
                     id to match the zone's id ({})",
@@ -424,7 +425,7 @@ impl OmicronZoneNic {
                     name: Name::from(nic.name.clone()),
                     ip: IpNetwork::from(nic.ip),
                     mac: MacAddr::from(nic.mac),
-                    subnet: IpNetwork::from(nic.subnet.clone()),
+                    subnet: IpNetwork::from(nic.subnet),
                     vni: SqlU32::from(u32::from(nic.vni)),
                     is_primary: nic.primary,
                     slot: SqlU8::from(nic.slot),
@@ -437,13 +438,11 @@ impl OmicronZoneNic {
     pub(crate) fn into_network_interface_for_zone(
         self,
         zone_id: Uuid,
-    ) -> anyhow::Result<nexus_types::inventory::NetworkInterface> {
-        Ok(nexus_types::inventory::NetworkInterface {
+    ) -> anyhow::Result<NetworkInterface> {
+        Ok(NetworkInterface {
             id: self.id,
             ip: self.ip.ip(),
-            kind: nexus_types::inventory::NetworkInterfaceKind::Service(
-                zone_id,
-            ),
+            kind: NetworkInterfaceKind::Service { id: zone_id },
             mac: *self.mac,
             name: self.name.into(),
             primary: self.is_primary,
diff --git a/nexus/db-model/src/probe.rs b/nexus/db-model/src/probe.rs
new file mode 100644
index 0000000000..be3576dfa0
--- /dev/null
+++ b/nexus/db-model/src/probe.rs
@@ -0,0 +1,50 @@
+use crate::schema::probe;
+use db_macros::Resource;
+use nexus_types::external_api::params;
+use nexus_types::identity::Resource;
+use omicron_common::api::external;
+use omicron_common::api::external::IdentityMetadataCreateParams;
+use serde::Deserialize;
+use serde::Serialize;
+use uuid::Uuid;
+
+#[derive(
+    Queryable,
+    Insertable,
+    Selectable,
+    Clone,
+    Debug,
+    Resource,
+    Serialize,
+    Deserialize,
+)]
+#[diesel(table_name = probe)]
+pub struct Probe {
+    #[diesel(embed)]
+    pub identity: ProbeIdentity,
+
+    pub project_id: Uuid,
+    pub sled: Uuid,
+}
+
+impl Probe {
+    pub fn from_create(p: &params::ProbeCreate, project_id: Uuid) -> Self {
+        Self {
+            identity: ProbeIdentity::new(
+                Uuid::new_v4(),
+                IdentityMetadataCreateParams {
+                    name: p.identity.name.clone(),
+                    description: p.identity.description.clone(),
+                },
+            ),
+            project_id,
+            sled: p.sled,
+        }
+    }
+}
+
+impl Into<external::Probe> for Probe {
+    fn into(self) -> external::Probe {
+        external::Probe { identity: self.identity().clone(), sled: self.sled }
+    }
+}
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index 09bc963936..e2b918d805 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(39, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(40, 0, 0);
 
 table! {
     disk (id) {
@@ -591,6 +591,7 @@ table! {
 
         project_id -> Nullable<Uuid>,
         state -> crate::IpAttachStateEnum,
+        is_probe -> Bool,
     }
 }
 
@@ -1520,6 +1521,19 @@ table! {
     }
 }
 
+table! {
+    probe (id) {
+        id -> Uuid,
+        name -> Text,
+        description -> Text,
+        time_created -> Timestamptz,
+        time_modified -> Timestamptz,
+        time_deleted -> Nullable<Timestamptz>,
+        project_id -> Uuid,
+        sled -> Uuid,
+    }
+}
+
 table! {
     db_metadata (singleton) {
         singleton -> Bool,
diff --git a/nexus/db-model/src/unsigned.rs b/nexus/db-model/src/unsigned.rs
index b4e9db2308..920cad1cff 100644
--- a/nexus/db-model/src/unsigned.rs
+++ b/nexus/db-model/src/unsigned.rs
@@ -7,6 +7,7 @@ use diesel::deserialize::{self, FromSql};
 use diesel::pg::Pg;
 use diesel::serialize::{self, ToSql};
 use diesel::sql_types;
+use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use std::convert::TryFrom;
 
@@ -76,6 +77,7 @@ where
     FromSqlRow,
     Serialize,
     Deserialize,
+    JsonSchema,
 )]
 #[diesel(sql_type = sql_types::Int4)]
 #[repr(transparent)]
diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml
index e673003036..a33ca24c84 100644
--- a/nexus/db-queries/Cargo.toml
+++ b/nexus/db-queries/Cargo.toml
@@ -35,6 +35,7 @@ pq-sys = "*"
 rand.workspace = true
 ref-cast.workspace = true
 samael.workspace = true
+schemars.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 serde_urlencoded.workspace = true
diff --git a/nexus/db-queries/src/db/datastore/external_ip.rs b/nexus/db-queries/src/db/datastore/external_ip.rs
index f561a024e8..017d2f22d2 100644
--- a/nexus/db-queries/src/db/datastore/external_ip.rs
+++ b/nexus/db-queries/src/db/datastore/external_ip.rs
@@ -46,6 +46,7 @@ use omicron_common::api::external::Error;
 use omicron_common::api::external::IdentityMetadataCreateParams;
 use omicron_common::api::external::ListResultVec;
 use omicron_common::api::external::LookupResult;
+use omicron_common::api::external::NameOrId;
 use omicron_common::api::external::ResourceType;
 use omicron_common::api::external::UpdateResult;
 use ref_cast::RefCast;
@@ -71,6 +72,42 @@ impl DataStore {
         self.allocate_external_ip(opctx, data).await
     }
 
+    /// Create an Ephemeral IP address for a probe.
+    pub async fn allocate_probe_ephemeral_ip(
+        &self,
+        opctx: &OpContext,
+        ip_id: Uuid,
+        probe_id: Uuid,
+        pool_name: Option<NameOrId>,
+    ) -> CreateResult<ExternalIp> {
+        let pool = match pool_name {
+            Some(NameOrId::Name(name)) => {
+                let (.., pool) = LookupPath::new(opctx, &self)
+                    .ip_pool_name(&name.into())
+                    .fetch_for(authz::Action::CreateChild)
+                    .await?;
+                pool
+            }
+            Some(NameOrId::Id(id)) => {
+                let (.., pool) = LookupPath::new(opctx, &self)
+                    .ip_pool_id(id)
+                    .fetch_for(authz::Action::CreateChild)
+                    .await?;
+                pool
+            }
+            // If no name given, use the default pool
+            None => {
+                let (.., pool) = self.ip_pools_fetch_default(&opctx).await?;
+                pool
+            }
+        };
+
+        let pool_id = pool.identity.id;
+        let data =
+            IncompleteExternalIp::for_ephemeral_probe(ip_id, probe_id, pool_id);
+        self.allocate_external_ip(opctx, data).await
+    }
+
     /// Create an Ephemeral IP address for an instance.
     ///
     /// For consistency between instance create and External IP attach/detach
@@ -725,6 +762,7 @@ impl DataStore {
         diesel::update(dsl::external_ip)
             .filter(dsl::time_deleted.is_null())
             .filter(dsl::is_service.eq(false))
+            .filter(dsl::is_probe.eq(false))
             .filter(dsl::parent_id.eq(instance_id))
             .filter(dsl::kind.ne(IpKind::Floating))
             .set((
@@ -736,7 +774,31 @@ impl DataStore {
             .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
     }
 
-    /// Detach all Floating IP address from their parent instance.
+    /// Delete all external IP addresses associated with the provided probe
+    /// ID.
+    ///
+    /// This method returns the number of records deleted, rather than the usual
+    /// `DeleteResult`. That's mostly useful for tests, but could be important
+    /// if callers have some invariants they'd like to check.
+    pub async fn deallocate_external_ip_by_probe_id(
+        &self,
+        opctx: &OpContext,
+        probe_id: Uuid,
+    ) -> Result<usize, Error> {
+        use db::schema::external_ip::dsl;
+        let now = Utc::now();
+        diesel::update(dsl::external_ip)
+            .filter(dsl::time_deleted.is_null())
+            .filter(dsl::is_probe.eq(true))
+            .filter(dsl::parent_id.eq(probe_id))
+            .filter(dsl::kind.ne(IpKind::Ephemeral))
+            .set(dsl::time_deleted.eq(now))
+            .execute_async(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
+    }
+
+    /// Detach an individual Floating IP address from their parent instance.
     ///
     /// As in `deallocate_external_ip_by_instance_id`, this method returns the
     /// number of records altered, rather than an `UpdateResult`.
@@ -774,6 +836,7 @@ impl DataStore {
         use db::schema::external_ip::dsl;
         dsl::external_ip
             .filter(dsl::is_service.eq(false))
+            .filter(dsl::is_probe.eq(false))
             .filter(dsl::parent_id.eq(instance_id))
             .filter(dsl::time_deleted.is_null())
             .select(ExternalIp::as_select())
@@ -796,6 +859,23 @@ impl DataStore {
             .find(|v| v.kind == IpKind::Ephemeral))
     }
 
+    /// Fetch all external IP addresses of any kind for the provided probe
+    pub async fn probe_lookup_external_ips(
+        &self,
+        opctx: &OpContext,
+        probe_id: Uuid,
+    ) -> LookupResult<Vec<ExternalIp>> {
+        use db::schema::external_ip::dsl;
+        dsl::external_ip
+            .filter(dsl::is_probe.eq(true))
+            .filter(dsl::parent_id.eq(probe_id))
+            .filter(dsl::time_deleted.is_null())
+            .select(ExternalIp::as_select())
+            .get_results_async(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
+    }
+
     /// Fetch all Floating IP addresses for the provided project.
     pub async fn floating_ips_list(
         &self,
diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs
index b164186fdf..475fb27df1 100644
--- a/nexus/db-queries/src/db/datastore/mod.rs
+++ b/nexus/db-queries/src/db/datastore/mod.rs
@@ -70,6 +70,7 @@ mod ipv4_nat_entry;
 mod network_interface;
 mod oximeter;
 mod physical_disk;
+mod probe;
 mod project;
 mod quota;
 mod rack;
@@ -106,6 +107,7 @@ pub use db_metadata::{
 pub use dns::DnsVersionUpdateBuilder;
 pub use instance::InstanceAndActiveVmm;
 pub use inventory::DataStoreInventoryTest;
+pub use probe::ProbeInfo;
 pub use rack::RackInit;
 pub use silo::Discoverability;
 pub use switch_port::SwitchPortSettingsCombinedResult;
@@ -1681,6 +1683,7 @@ mod test {
                 first_port: crate::db::model::SqlU16(0),
                 last_port: crate::db::model::SqlU16(10),
                 state: nexus_db_model::IpAttachState::Attached,
+                is_probe: false,
             })
             .collect::<Vec<_>>();
         diesel::insert_into(dsl::external_ip)
@@ -1743,6 +1746,7 @@ mod test {
             first_port: crate::db::model::SqlU16(0),
             last_port: crate::db::model::SqlU16(10),
             state: nexus_db_model::IpAttachState::Attached,
+            is_probe: false,
         };
         diesel::insert_into(dsl::external_ip)
             .values(ip.clone())
@@ -1814,6 +1818,7 @@ mod test {
             first_port: crate::db::model::SqlU16(0),
             last_port: crate::db::model::SqlU16(10),
             state: nexus_db_model::IpAttachState::Attached,
+            is_probe: false,
         };
 
         // Combinations of NULL and non-NULL for:
diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs
index f2782e8f67..1bccca4e97 100644
--- a/nexus/db-queries/src/db/datastore/network_interface.rs
+++ b/nexus/db-queries/src/db/datastore/network_interface.rs
@@ -35,11 +35,11 @@ use omicron_common::api::external::http_pagination::PaginatedBy;
 use omicron_common::api::external::DeleteResult;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::ListResultVec;
+use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::LookupType;
 use omicron_common::api::external::ResourceType;
 use omicron_common::api::external::UpdateResult;
 use ref_cast::RefCast;
-use sled_agent_client::types as sled_client_types;
 use uuid::Uuid;
 
 /// OPTE requires information that's currently split across the network
@@ -59,8 +59,10 @@ struct NicInfo {
     slot: i16,
 }
 
-impl From<NicInfo> for sled_client_types::NetworkInterface {
-    fn from(nic: NicInfo) -> sled_client_types::NetworkInterface {
+impl From<NicInfo> for omicron_common::api::internal::shared::NetworkInterface {
+    fn from(
+        nic: NicInfo,
+    ) -> omicron_common::api::internal::shared::NetworkInterface {
         let ip_subnet = if nic.ip.is_ipv4() {
             external::IpNet::V4(nic.ipv4_block.0)
         } else {
@@ -68,19 +70,22 @@ impl From<NicInfo> for sled_client_types::NetworkInterface {
         };
         let kind = match nic.kind {
             NetworkInterfaceKind::Instance => {
-                sled_client_types::NetworkInterfaceKind::Instance(nic.parent_id)
+                omicron_common::api::internal::shared::NetworkInterfaceKind::Instance{ id: nic.parent_id }
             }
             NetworkInterfaceKind::Service => {
-                sled_client_types::NetworkInterfaceKind::Service(nic.parent_id)
+                omicron_common::api::internal::shared::NetworkInterfaceKind::Service{ id: nic.parent_id }
+            }
+            NetworkInterfaceKind::Probe => {
+                omicron_common::api::internal::shared::NetworkInterfaceKind::Probe{ id: nic.parent_id }
             }
         };
-        sled_client_types::NetworkInterface {
+        omicron_common::api::internal::shared::NetworkInterface {
             id: nic.id,
             kind,
             name: nic.name.into(),
             ip: nic.ip.ip(),
             mac: nic.mac.0,
-            subnet: sled_client_types::IpNet::from(ip_subnet),
+            subnet: ip_subnet,
             vni: nic.vni.0,
             primary: nic.primary,
             slot: u8::try_from(nic.slot).unwrap(),
@@ -108,6 +113,14 @@ impl DataStore {
         self.instance_create_network_interface_raw(&opctx, interface).await
     }
 
+    pub async fn probe_create_network_interface(
+        &self,
+        opctx: &OpContext,
+        interface: IncompleteNetworkInterface,
+    ) -> Result<NetworkInterface, network_interface::InsertError> {
+        self.create_network_interface_raw(&opctx, interface).await
+    }
+
     pub(crate) async fn instance_create_network_interface_raw(
         &self,
         opctx: &OpContext,
@@ -271,6 +284,33 @@ impl DataStore {
         Ok(())
     }
 
+    /// Delete all network interfaces attached to the given probe.
+    pub async fn probe_delete_all_network_interfaces(
+        &self,
+        opctx: &OpContext,
+        probe_id: Uuid,
+    ) -> DeleteResult {
+        use db::schema::network_interface::dsl;
+        let now = Utc::now();
+        diesel::update(dsl::network_interface)
+            .filter(dsl::parent_id.eq(probe_id))
+            .filter(dsl::kind.eq(NetworkInterfaceKind::Probe))
+            .filter(dsl::time_deleted.is_null())
+            .set(dsl::time_deleted.eq(now))
+            .execute_async(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| {
+                public_error_from_diesel(
+                    e,
+                    ErrorHandler::NotFoundByLookup(
+                        ResourceType::Probe,
+                        LookupType::ById(probe_id),
+                    ),
+                )
+            })?;
+        Ok(())
+    }
+
     /// Delete an `InstanceNetworkInterface` attached to a provided instance.
     ///
     /// Note that the primary interface for an instance cannot be deleted if
@@ -313,7 +353,8 @@ impl DataStore {
         &self,
         opctx: &OpContext,
         partial_query: BoxedQuery<db::schema::network_interface::table>,
-    ) -> ListResultVec<sled_client_types::NetworkInterface> {
+    ) -> ListResultVec<omicron_common::api::internal::shared::NetworkInterface>
+    {
         use db::schema::network_interface;
         use db::schema::vpc;
         use db::schema::vpc_subnet;
@@ -349,7 +390,7 @@ impl DataStore {
             .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
         Ok(rows
             .into_iter()
-            .map(sled_client_types::NetworkInterface::from)
+            .map(omicron_common::api::internal::shared::NetworkInterface::from)
             .collect())
     }
 
@@ -359,7 +400,8 @@ impl DataStore {
         &self,
         opctx: &OpContext,
         authz_instance: &authz::Instance,
-    ) -> ListResultVec<sled_client_types::NetworkInterface> {
+    ) -> ListResultVec<omicron_common::api::internal::shared::NetworkInterface>
+    {
         opctx.authorize(authz::Action::ListChildren, authz_instance).await?;
 
         use db::schema::network_interface;
@@ -375,13 +417,31 @@ impl DataStore {
         .await
     }
 
+    pub async fn derive_probe_network_interface_info(
+        &self,
+        opctx: &OpContext,
+        probe_id: Uuid,
+    ) -> ListResultVec<omicron_common::api::internal::shared::NetworkInterface>
+    {
+        use db::schema::network_interface;
+        self.derive_network_interface_info(
+            opctx,
+            network_interface::table
+                .filter(network_interface::parent_id.eq(probe_id))
+                .filter(network_interface::kind.eq(NetworkInterfaceKind::Probe))
+                .into_boxed(),
+        )
+        .await
+    }
+
     /// Return information about all VNICs connected to a VPC required
     /// for the sled agent to instantiate firewall rules via OPTE.
     pub async fn derive_vpc_network_interface_info(
         &self,
         opctx: &OpContext,
         authz_vpc: &authz::Vpc,
-    ) -> ListResultVec<sled_client_types::NetworkInterface> {
+    ) -> ListResultVec<omicron_common::api::internal::shared::NetworkInterface>
+    {
         opctx.authorize(authz::Action::ListChildren, authz_vpc).await?;
 
         use db::schema::network_interface;
@@ -400,7 +460,8 @@ impl DataStore {
         &self,
         opctx: &OpContext,
         authz_subnet: &authz::VpcSubnet,
-    ) -> ListResultVec<sled_client_types::NetworkInterface> {
+    ) -> ListResultVec<omicron_common::api::internal::shared::NetworkInterface>
+    {
         opctx.authorize(authz::Action::ListChildren, authz_subnet).await?;
 
         use db::schema::network_interface;
@@ -443,6 +504,25 @@ impl DataStore {
         .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
     }
 
+    /// Get network interface associated with a given probe.
+    pub async fn probe_get_network_interface(
+        &self,
+        opctx: &OpContext,
+        probe_id: Uuid,
+    ) -> LookupResult<NetworkInterface> {
+        use db::schema::network_interface::dsl;
+
+        dsl::network_interface
+            .filter(dsl::time_deleted.is_null())
+            .filter(dsl::parent_id.eq(probe_id))
+            .select(NetworkInterface::as_select())
+            .first_async::<NetworkInterface>(
+                &*self.pool_connection_authorized(opctx).await?,
+            )
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
+    }
+
     /// Update a network interface associated with a given instance.
     pub async fn instance_update_network_interface(
         &self,
diff --git a/nexus/db-queries/src/db/datastore/probe.rs b/nexus/db-queries/src/db/datastore/probe.rs
new file mode 100644
index 0000000000..f1e737e353
--- /dev/null
+++ b/nexus/db-queries/src/db/datastore/probe.rs
@@ -0,0 +1,390 @@
+use std::net::IpAddr;
+
+use crate::authz;
+use crate::context::OpContext;
+use crate::db;
+use crate::db::datastore::DataStoreConnection;
+use crate::db::error::public_error_from_diesel;
+use crate::db::error::ErrorHandler;
+use crate::db::lookup::LookupPath;
+use crate::db::model::Name;
+use crate::db::pagination::paginated;
+use async_bb8_diesel::AsyncRunQueryDsl;
+use chrono::Utc;
+use diesel::{ExpressionMethods, QueryDsl, SelectableHelper};
+use nexus_db_model::IncompleteNetworkInterface;
+use nexus_db_model::Probe;
+use nexus_db_model::VpcSubnet;
+use nexus_types::external_api::params;
+use nexus_types::identity::Resource;
+use omicron_common::api::external::http_pagination::PaginatedBy;
+use omicron_common::api::external::CreateResult;
+use omicron_common::api::external::DataPageParams;
+use omicron_common::api::external::DeleteResult;
+use omicron_common::api::external::IdentityMetadataCreateParams;
+use omicron_common::api::external::ListResultVec;
+use omicron_common::api::external::LookupResult;
+use omicron_common::api::external::LookupType;
+use omicron_common::api::external::NameOrId;
+use omicron_common::api::external::ResourceType;
+use omicron_common::api::internal::shared::NetworkInterface;
+use ref_cast::RefCast;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use uuid::Uuid;
+
+#[derive(Debug, Clone, JsonSchema, Serialize, Deserialize)]
+pub struct ProbeInfo {
+    pub id: Uuid,
+    pub name: Name,
+    sled: Uuid,
+    pub external_ips: Vec<ProbeExternalIp>,
+    pub interface: NetworkInterface,
+}
+
+#[derive(Debug, Clone, JsonSchema, Serialize, Deserialize)]
+pub struct ProbeExternalIp {
+    ip: IpAddr,
+    first_port: u16,
+    last_port: u16,
+    kind: IpKind,
+}
+
+impl From<nexus_db_model::ExternalIp> for ProbeExternalIp {
+    fn from(value: nexus_db_model::ExternalIp) -> Self {
+        Self {
+            ip: value.ip.ip(),
+            first_port: value.first_port.0,
+            last_port: value.last_port.0,
+            kind: value.kind.into(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, JsonSchema, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum IpKind {
+    Snat,
+    Floating,
+    Ephemeral,
+}
+
+impl From<nexus_db_model::IpKind> for IpKind {
+    fn from(value: nexus_db_model::IpKind) -> Self {
+        match value {
+            nexus_db_model::IpKind::SNat => Self::Snat,
+            nexus_db_model::IpKind::Ephemeral => Self::Ephemeral,
+            nexus_db_model::IpKind::Floating => Self::Floating,
+        }
+    }
+}
+
+impl super::DataStore {
+    /// List the probes for the given project.
+    pub async fn probe_list(
+        &self,
+        opctx: &OpContext,
+        authz_project: &authz::Project,
+        pagparams: &PaginatedBy<'_>,
+    ) -> ListResultVec<ProbeInfo> {
+        opctx.authorize(authz::Action::ListChildren, authz_project).await?;
+
+        use db::schema::probe::dsl;
+        use db::schema::vpc_subnet::dsl as vpc_subnet_dsl;
+
+        let pool = self.pool_connection_authorized(opctx).await?;
+
+        let probes = match pagparams {
+            PaginatedBy::Id(pagparams) => {
+                paginated(dsl::probe, dsl::id, &pagparams)
+            }
+            PaginatedBy::Name(pagparams) => paginated(
+                dsl::probe,
+                dsl::name,
+                &pagparams.map_name(|n| Name::ref_cast(n)),
+            ),
+        }
+        .filter(dsl::project_id.eq(authz_project.id()))
+        .filter(dsl::time_deleted.is_null())
+        .select(Probe::as_select())
+        .load_async(&*pool)
+        .await
+        .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        let mut result = Vec::with_capacity(probes.len());
+
+        for probe in probes.into_iter() {
+            let external_ips = self
+                .probe_lookup_external_ips(opctx, probe.id())
+                .await?
+                .into_iter()
+                .map(Into::into)
+                .collect();
+
+            let interface =
+                self.probe_get_network_interface(opctx, probe.id()).await?;
+
+            let vni = self.resolve_vpc_to_vni(opctx, interface.vpc_id).await?;
+
+            let db_subnet = vpc_subnet_dsl::vpc_subnet
+                .filter(vpc_subnet_dsl::id.eq(interface.subnet_id))
+                .select(VpcSubnet::as_select())
+                .first_async(&*pool)
+                .await
+                .map_err(|e| {
+                    public_error_from_diesel(e, ErrorHandler::Server)
+                })?;
+
+            let mut interface: NetworkInterface =
+                interface.into_internal(db_subnet.ipv4_block.0.into());
+
+            interface.vni = vni.0;
+
+            result.push(ProbeInfo {
+                id: probe.id(),
+                name: probe.name().clone().into(),
+                sled: probe.sled,
+                interface,
+                external_ips,
+            })
+        }
+
+        Ok(result)
+    }
+
+    async fn resolve_probe_info(
+        &self,
+        opctx: &OpContext,
+        probe: &Probe,
+        pool: &DataStoreConnection<'_>,
+    ) -> LookupResult<ProbeInfo> {
+        use db::schema::vpc_subnet::dsl as vpc_subnet_dsl;
+
+        let external_ips = self
+            .probe_lookup_external_ips(opctx, probe.id())
+            .await?
+            .into_iter()
+            .map(Into::into)
+            .collect();
+
+        let interface =
+            self.probe_get_network_interface(opctx, probe.id()).await?;
+
+        let db_subnet = vpc_subnet_dsl::vpc_subnet
+            .filter(vpc_subnet_dsl::id.eq(interface.subnet_id))
+            .select(VpcSubnet::as_select())
+            .first_async(&**pool)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        let vni = self.resolve_vpc_to_vni(opctx, interface.vpc_id).await?;
+
+        let mut interface: NetworkInterface =
+            interface.into_internal(db_subnet.ipv4_block.0.into());
+        interface.vni = vni.0;
+
+        Ok(ProbeInfo {
+            id: probe.id(),
+            name: probe.name().clone().into(),
+            sled: probe.sled,
+            interface,
+            external_ips,
+        })
+    }
+
+    /// List the probes for a given sled. This is used by sled agents for
+    /// determining what probes they should be running.
+    pub async fn probe_list_for_sled(
+        &self,
+        sled: Uuid,
+        opctx: &OpContext,
+        pagparams: &DataPageParams<'_, Uuid>,
+    ) -> ListResultVec<ProbeInfo> {
+        use db::schema::probe::dsl;
+
+        let pool = self.pool_connection_authorized(opctx).await?;
+
+        let probes = paginated(dsl::probe, dsl::id, pagparams)
+            .filter(dsl::time_deleted.is_null())
+            .filter(dsl::sled.eq(sled))
+            .select(Probe::as_select())
+            .load_async(&*pool)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        let mut result = Vec::with_capacity(probes.len());
+
+        for probe in probes.into_iter() {
+            result.push(self.resolve_probe_info(opctx, &probe, &pool).await?);
+        }
+
+        Ok(result)
+    }
+
+    /// Get information about a particular probe given its name or id.
+    pub async fn probe_get(
+        &self,
+        opctx: &OpContext,
+        authz_project: &authz::Project,
+        name_or_id: &NameOrId,
+    ) -> LookupResult<ProbeInfo> {
+        use db::schema::probe;
+        use db::schema::probe::dsl;
+        let pool = self.pool_connection_authorized(opctx).await?;
+
+        let name_or_id = name_or_id.clone();
+
+        let probe = match name_or_id {
+            NameOrId::Name(name) => dsl::probe
+                .filter(probe::name.eq(name.to_string()))
+                .filter(probe::time_deleted.is_null())
+                .filter(probe::project_id.eq(authz_project.id()))
+                .select(Probe::as_select())
+                .limit(1)
+                .first_async::<Probe>(&*pool)
+                .await
+                .map_err(|e| {
+                    public_error_from_diesel(
+                        e,
+                        ErrorHandler::NotFoundByLookup(
+                            ResourceType::Probe,
+                            LookupType::ByName(name.to_string()),
+                        ),
+                    )
+                }),
+            NameOrId::Id(id) => dsl::probe
+                .filter(probe::id.eq(id))
+                .filter(probe::project_id.eq(authz_project.id()))
+                .select(Probe::as_select())
+                .limit(1)
+                .first_async::<Probe>(&*pool)
+                .await
+                .map_err(|e| {
+                    public_error_from_diesel(
+                        e,
+                        ErrorHandler::NotFoundByLookup(
+                            ResourceType::Probe,
+                            LookupType::ById(id),
+                        ),
+                    )
+                }),
+        }?;
+
+        self.resolve_probe_info(opctx, &probe, &pool).await
+    }
+
+    /// Add a probe to the data store.
+    pub async fn probe_create(
+        &self,
+        opctx: &OpContext,
+        authz_project: &authz::Project,
+        new_probe: &params::ProbeCreate,
+    ) -> CreateResult<Probe> {
+        //TODO in transaction
+        use db::schema::probe::dsl;
+        let pool = self.pool_connection_authorized(opctx).await?;
+
+        let probe = Probe::from_create(new_probe, authz_project.id());
+
+        let _eip = self
+            .allocate_probe_ephemeral_ip(
+                opctx,
+                Uuid::new_v4(),
+                probe.id(),
+                new_probe.ip_pool.clone().map(Into::into),
+            )
+            .await?;
+
+        let default_name = omicron_common::api::external::Name::try_from(
+            "default".to_string(),
+        )
+        .unwrap();
+        let internal_default_name = db::model::Name::from(default_name.clone());
+
+        let (.., db_subnet) = LookupPath::new(opctx, self)
+            .project_id(authz_project.id())
+            .vpc_name(&internal_default_name)
+            .vpc_subnet_name(&internal_default_name)
+            .fetch()
+            .await?;
+
+        let incomplete = IncompleteNetworkInterface::new_probe(
+            Uuid::new_v4(),
+            probe.id(),
+            db_subnet,
+            IdentityMetadataCreateParams {
+                name: probe.name().clone(),
+                description: format!(
+                    "default primary interface for {}",
+                    probe.name(),
+                ),
+            },
+            None, //Request IP address assignment
+            None, //Request MAC address assignment
+        )?;
+
+        let _ifx = self
+            .probe_create_network_interface(opctx, incomplete)
+            .await
+            .map_err(|e| {
+                omicron_common::api::external::Error::InternalError {
+                    internal_message: format!(
+                        "create network interface: {e:?}"
+                    ),
+                }
+            })?;
+
+        let result = diesel::insert_into(dsl::probe)
+            .values(probe.clone())
+            .returning(Probe::as_returning())
+            .get_result_async(&*pool)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        Ok(result)
+    }
+
+    /// Remove a probe from the data store.
+    pub async fn probe_delete(
+        &self,
+        opctx: &OpContext,
+        authz_project: &authz::Project,
+        name_or_id: &NameOrId,
+    ) -> DeleteResult {
+        use db::schema::probe;
+        use db::schema::probe::dsl;
+        let pool = self.pool_connection_authorized(opctx).await?;
+
+        let name_or_id = name_or_id.clone();
+
+        //TODO in transaction
+        let id = match name_or_id {
+            NameOrId::Name(name) => dsl::probe
+                .filter(probe::name.eq(name.to_string()))
+                .filter(probe::time_deleted.is_null())
+                .filter(probe::project_id.eq(authz_project.id()))
+                .select(probe::id)
+                .limit(1)
+                .first_async::<Uuid>(&*pool)
+                .await
+                .map_err(|e| {
+                    public_error_from_diesel(e, ErrorHandler::Server)
+                })?,
+            NameOrId::Id(id) => id,
+        };
+
+        self.deallocate_external_ip_by_probe_id(opctx, id).await?;
+
+        self.probe_delete_all_network_interfaces(opctx, id).await?;
+
+        diesel::update(dsl::probe)
+            .filter(dsl::id.eq(id))
+            .filter(dsl::project_id.eq(authz_project.id()))
+            .set(dsl::time_deleted.eq(Utc::now()))
+            .execute_async(&*pool)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        Ok(())
+    }
+}
diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs
index cd972c0941..8c5d5f4f46 100644
--- a/nexus/db-queries/src/db/datastore/vpc.rs
+++ b/nexus/db-queries/src/db/datastore/vpc.rs
@@ -1212,6 +1212,30 @@ impl DataStore {
                 )
             })
     }
+
+    /// Look up a VNI by VPC.
+    pub async fn resolve_vpc_to_vni(
+        &self,
+        opctx: &OpContext,
+        vpc_id: Uuid,
+    ) -> LookupResult<Vni> {
+        use db::schema::vpc::dsl;
+        dsl::vpc
+            .filter(dsl::id.eq(vpc_id))
+            .filter(dsl::time_deleted.is_null())
+            .select(dsl::vni)
+            .get_result_async(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| {
+                public_error_from_diesel(
+                    e,
+                    ErrorHandler::NotFoundByLookup(
+                        ResourceType::Vpc,
+                        LookupType::ByCompositeId("VNI".to_string()),
+                    ),
+                )
+            })
+    }
 }
 
 #[cfg(test)]
@@ -1228,8 +1252,6 @@ mod tests {
     use nexus_test_utils::db::test_setup_database;
     use nexus_types::deployment::Blueprint;
     use nexus_types::deployment::BlueprintTarget;
-    use nexus_types::deployment::NetworkInterface;
-    use nexus_types::deployment::NetworkInterfaceKind;
     use nexus_types::deployment::OmicronZoneConfig;
     use nexus_types::deployment::OmicronZoneType;
     use nexus_types::deployment::OmicronZonesConfig;
@@ -1241,6 +1263,8 @@ mod tests {
     use omicron_common::api::external::IpNet;
     use omicron_common::api::external::MacAddr;
     use omicron_common::api::external::Vni;
+    use omicron_common::api::internal::shared::NetworkInterface;
+    use omicron_common::api::internal::shared::NetworkInterfaceKind;
     use omicron_test_utils::dev;
     use slog::info;
     use std::collections::BTreeMap;
@@ -1558,13 +1582,15 @@ mod tests {
                         external_ip: "::1".parse().unwrap(),
                         nic: NetworkInterface {
                             id: nic.identity.id,
-                            kind: NetworkInterfaceKind::Service(service.id()),
+                            kind: NetworkInterfaceKind::Service {
+                                id: service.id(),
+                            },
                             name: format!("test-nic-{}", nic.identity.id)
                                 .parse()
                                 .unwrap(),
                             ip: nic.ip.unwrap(),
                             mac: nic.mac.unwrap(),
-                            subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET).into(),
+                            subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET),
                             vni: Vni::SERVICES_VNI,
                             primary: true,
                             slot: nic.slot.unwrap(),
diff --git a/nexus/db-queries/src/db/queries/external_ip.rs b/nexus/db-queries/src/db/queries/external_ip.rs
index 739c0b0809..0502450121 100644
--- a/nexus/db-queries/src/db/queries/external_ip.rs
+++ b/nexus/db-queries/src/db/queries/external_ip.rs
@@ -132,6 +132,7 @@ const MAX_PORT: u16 = u16::MAX;
 ///         CAST(candidate_first_port AS INT4) AS first_port,
 ///         CAST(candidate_last_port AS INT4) AS last_port,
 ///         <project_id> AS project_id,
+///         <is_probe> AS is_probe,
 ///         <state> AS state
 ///     FROM
 ///         SELECT * FROM (
@@ -419,6 +420,12 @@ impl NextExternalIp {
         )?;
         out.push_sql(" AS ");
         out.push_identifier(dsl::state::NAME)?;
+        out.push_sql(", ");
+
+        // is_probe flag
+        out.push_bind_param::<sql_types::Bool, bool>(self.ip.is_probe())?;
+        out.push_sql(" AS ");
+        out.push_identifier(dsl::is_probe::NAME)?;
 
         out.push_sql(" FROM (");
         self.push_address_sequence_subquery(out.reborrow())?;
diff --git a/nexus/db-queries/src/db/queries/network_interface.rs b/nexus/db-queries/src/db/queries/network_interface.rs
index f6ce3e31e3..c0fc18aca1 100644
--- a/nexus/db-queries/src/db/queries/network_interface.rs
+++ b/nexus/db-queries/src/db/queries/network_interface.rs
@@ -159,6 +159,9 @@ impl InsertError {
             InsertError::InterfaceAlreadyExists(_name, NetworkInterfaceKind::Service) => {
                 unimplemented!("service network interface")
             }
+            InsertError::InterfaceAlreadyExists(_name, NetworkInterfaceKind::Probe) => {
+                unimplemented!("probe network interface")
+            }
             InsertError::NoAvailableIpAddresses => {
                 external::Error::invalid_request(
                     "No available IP addresses for interface",
@@ -408,6 +411,9 @@ fn decode_database_error(
                     NetworkInterfaceKind::Service => {
                         external::ResourceType::ServiceNetworkInterface
                     }
+                    NetworkInterfaceKind::Probe => {
+                        external::ResourceType::ProbeNetworkInterface
+                    }
                 };
                 InsertError::External(error::public_error_from_diesel(
                     err,
@@ -647,7 +653,7 @@ impl NextMacShifts {
 impl NextMacAddress {
     pub fn new(vpc_id: Uuid, kind: NetworkInterfaceKind) -> Self {
         let (base, max_shift, min_shift) = match kind {
-            NetworkInterfaceKind::Instance => {
+            NetworkInterfaceKind::Instance | NetworkInterfaceKind::Probe => {
                 let NextMacShifts { base, min_shift, max_shift } =
                     NextMacShifts::for_guest();
                 (base.into(), max_shift, min_shift)
@@ -2730,6 +2736,7 @@ mod tests {
             NetworkInterfaceKind::Service => {
                 (inserted.mac.is_system(), "system")
             }
+            NetworkInterfaceKind::Probe => (inserted.mac.is_system(), "probe"),
         };
         assert!(
             mac_in_range,
diff --git a/nexus/reconfigurator/execution/src/resource_allocation.rs b/nexus/reconfigurator/execution/src/resource_allocation.rs
index 8ca44df39e..83a484baa4 100644
--- a/nexus/reconfigurator/execution/src/resource_allocation.rs
+++ b/nexus/reconfigurator/execution/src/resource_allocation.rs
@@ -15,12 +15,12 @@ use nexus_db_queries::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET;
 use nexus_db_queries::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET;
 use nexus_db_queries::db::fixed_data::vpc_subnet::NTP_VPC_SUBNET;
 use nexus_db_queries::db::DataStore;
-use nexus_types::deployment::NetworkInterface;
-use nexus_types::deployment::NetworkInterfaceKind;
 use nexus_types::deployment::OmicronZoneType;
 use nexus_types::deployment::OmicronZonesConfig;
 use nexus_types::deployment::SourceNatConfig;
 use omicron_common::api::external::IdentityMetadataCreateParams;
+use omicron_common::api::internal::shared::NetworkInterface;
+use omicron_common::api::internal::shared::NetworkInterfaceKind;
 use slog::info;
 use slog::warn;
 use std::collections::BTreeMap;
@@ -345,6 +345,7 @@ impl<'a> ResourceAllocator<'a> {
                 bail!("invalid NIC kind (expected service, got instance)")
             }
             NetworkInterfaceKind::Service { .. } => (),
+            NetworkInterfaceKind::Probe { .. } => (),
         }
 
         // Only attempt to allocate `nic` if it isn't already assigned to this
@@ -546,7 +547,7 @@ mod tests {
             external_ips.next().expect("exhausted external_ips");
         let nexus_nic = NetworkInterface {
             id: Uuid::new_v4(),
-            kind: NetworkInterfaceKind::Service(nexus_id),
+            kind: NetworkInterfaceKind::Service { id: nexus_id },
             name: "test-nexus".parse().expect("bad name"),
             ip: NEXUS_OPTE_IPV4_SUBNET
                 .iter()
@@ -554,7 +555,7 @@ mod tests {
                 .unwrap()
                 .into(),
             mac: MacAddr::random_system(),
-            subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET).into(),
+            subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET),
             vni: Vni::SERVICES_VNI,
             primary: true,
             slot: 0,
@@ -566,7 +567,7 @@ mod tests {
             external_ips.next().expect("exhausted external_ips");
         let dns_nic = NetworkInterface {
             id: Uuid::new_v4(),
-            kind: NetworkInterfaceKind::Service(dns_id),
+            kind: NetworkInterfaceKind::Service { id: dns_id },
             name: "test-external-dns".parse().expect("bad name"),
             ip: DNS_OPTE_IPV4_SUBNET
                 .iter()
@@ -574,7 +575,7 @@ mod tests {
                 .unwrap()
                 .into(),
             mac: MacAddr::random_system(),
-            subnet: IpNet::from(*DNS_OPTE_IPV4_SUBNET).into(),
+            subnet: IpNet::from(*DNS_OPTE_IPV4_SUBNET),
             vni: Vni::SERVICES_VNI,
             primary: true,
             slot: 0,
@@ -589,7 +590,7 @@ mod tests {
         };
         let ntp_nic = NetworkInterface {
             id: Uuid::new_v4(),
-            kind: NetworkInterfaceKind::Service(ntp_id),
+            kind: NetworkInterfaceKind::Service { id: ntp_id },
             name: "test-external-ntp".parse().expect("bad name"),
             ip: NTP_OPTE_IPV4_SUBNET
                 .iter()
@@ -597,7 +598,7 @@ mod tests {
                 .unwrap()
                 .into(),
             mac: MacAddr::random_system(),
-            subnet: IpNet::from(*NTP_OPTE_IPV4_SUBNET).into(),
+            subnet: IpNet::from(*NTP_OPTE_IPV4_SUBNET),
             vni: Vni::SERVICES_VNI,
             primary: true,
             slot: 0,
@@ -888,9 +889,14 @@ mod tests {
                             "invalid NIC kind (expected service, got instance)"
                         )
                     }
-                    NetworkInterfaceKind::Service(id) => {
+                    NetworkInterfaceKind::Probe { .. } => {
+                        panic!(
+                            "invalid NIC kind (expected service, got instance)"
+                        )
+                    }
+                    NetworkInterfaceKind::Service { id } => {
                         let id = *id;
-                        nic.kind = NetworkInterfaceKind::Instance(id);
+                        nic.kind = NetworkInterfaceKind::Instance { id };
                     }
                 }
                 "invalid NIC kind".to_string()
diff --git a/nexus/reconfigurator/planning/src/blueprint_builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder.rs
index d541e112d5..26d0d1e23f 100644
--- a/nexus/reconfigurator/planning/src/blueprint_builder.rs
+++ b/nexus/reconfigurator/planning/src/blueprint_builder.rs
@@ -13,8 +13,6 @@ use ipnet::IpAdd;
 use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use nexus_inventory::now_db_precision;
 use nexus_types::deployment::Blueprint;
-use nexus_types::deployment::NetworkInterface;
-use nexus_types::deployment::NetworkInterfaceKind;
 use nexus_types::deployment::OmicronZoneConfig;
 use nexus_types::deployment::OmicronZoneDataset;
 use nexus_types::deployment::OmicronZoneType;
@@ -35,6 +33,8 @@ use omicron_common::api::external::Generation;
 use omicron_common::api::external::IpNet;
 use omicron_common::api::external::MacAddr;
 use omicron_common::api::external::Vni;
+use omicron_common::api::internal::shared::NetworkInterface;
+use omicron_common::api::internal::shared::NetworkInterfaceKind;
 use slog::o;
 use slog::Logger;
 use std::collections::BTreeMap;
@@ -532,14 +532,14 @@ impl<'a> BlueprintBuilder<'a> {
                             .next()
                             .ok_or(Error::ExhaustedNexusIps)?
                             .into(),
-                        IpNet::from(*NEXUS_OPTE_IPV4_SUBNET).into(),
+                        IpNet::from(*NEXUS_OPTE_IPV4_SUBNET),
                     ),
                     IpAddr::V6(_) => (
                         self.nexus_v6_ips
                             .next()
                             .ok_or(Error::ExhaustedNexusIps)?
                             .into(),
-                        IpNet::from(*NEXUS_OPTE_IPV6_SUBNET).into(),
+                        IpNet::from(*NEXUS_OPTE_IPV6_SUBNET),
                     ),
                 };
                 let mac = self
@@ -548,7 +548,7 @@ impl<'a> BlueprintBuilder<'a> {
                     .ok_or(Error::NoSystemMacAddressAvailable)?;
                 NetworkInterface {
                     id: Uuid::new_v4(),
-                    kind: NetworkInterfaceKind::Service(nexus_id),
+                    kind: NetworkInterfaceKind::Service { id: nexus_id },
                     name: format!("nexus-{nexus_id}").parse().unwrap(),
                     ip,
                     mac,
diff --git a/nexus/src/app/instance_network.rs b/nexus/src/app/instance_network.rs
index c0bc5d237b..eb5f83470f 100644
--- a/nexus/src/app/instance_network.rs
+++ b/nexus/src/app/instance_network.rs
@@ -22,11 +22,13 @@ use omicron_common::api::external::Error;
 use omicron_common::api::external::Ipv4Net;
 use omicron_common::api::external::Ipv6Net;
 use omicron_common::api::internal::nexus;
+use omicron_common::api::internal::shared::NetworkInterface;
 use omicron_common::api::internal::shared::SwitchLocation;
 use sled_agent_client::types::DeleteVirtualNetworkInterfaceHost;
 use sled_agent_client::types::SetVirtualNetworkInterfaceHost;
 use std::collections::HashSet;
 use std::str::FromStr;
+use std::sync::Arc;
 use uuid::Uuid;
 
 impl super::Nexus {
@@ -452,11 +454,105 @@ impl super::Nexus {
         Ok(nat_entries)
     }
 
+    // The logic of this function should follow very closely what
+    // `instance_ensure_dpd_config` does. However, there are enough differences
+    // in the mechanics of how the logic is being carried out to justify having
+    // this separate function, it seems.
+    pub(crate) async fn probe_ensure_dpd_config(
+        &self,
+        opctx: &OpContext,
+        probe_id: Uuid,
+        sled_ip_address: std::net::Ipv6Addr,
+        ip_index_filter: Option<usize>,
+        dpd_client: &Arc<dpd_client::Client>,
+    ) -> Result<(), Error> {
+        let log = &self.log;
+
+        // All external IPs map to the primary network interface, so find that
+        // interface. If there is no such interface, there's no way to route
+        // traffic destined to those IPs, so there's nothing to configure and
+        // it's safe to return early.
+        let network_interface = match self
+            .db_datastore
+            .derive_probe_network_interface_info(&opctx, probe_id)
+            .await?
+            .into_iter()
+            .find(|interface| interface.primary)
+        {
+            Some(interface) => interface,
+            None => {
+                info!(log, "probe has no primary network interface";
+                      "probe_id" => %probe_id);
+                return Ok(());
+            }
+        };
+
+        let mac_address =
+            macaddr::MacAddr6::from_str(&network_interface.mac.to_string())
+                .map_err(|e| {
+                    Error::internal_error(&format!(
+                        "failed to convert mac address: {e}"
+                    ))
+                })?;
+
+        info!(log, "looking up probe's external IPs";
+              "probe_id" => %probe_id);
+
+        let ips = self
+            .db_datastore
+            .probe_lookup_external_ips(&opctx, probe_id)
+            .await?;
+
+        if let Some(wanted_index) = ip_index_filter {
+            if let None = ips.get(wanted_index) {
+                return Err(Error::internal_error(&format!(
+                    "failed to find external ip address at index: {}",
+                    wanted_index
+                )));
+            }
+        }
+
+        let sled_address =
+            Ipv6Net(Ipv6Network::new(sled_ip_address, 128).unwrap());
+
+        for target_ip in ips
+            .iter()
+            .enumerate()
+            .filter(|(index, _)| {
+                if let Some(wanted_index) = ip_index_filter {
+                    *index == wanted_index
+                } else {
+                    true
+                }
+            })
+            .map(|(_, ip)| ip)
+        {
+            // For each external ip, add a nat entry to the database
+            self.ensure_nat_entry(
+                target_ip,
+                sled_address,
+                &network_interface,
+                mac_address,
+                opctx,
+            )
+            .await?;
+        }
+
+        // Notify dendrite that there are changes for it to reconcile.
+        // In the event of a failure to notify dendrite, we'll log an error
+        // and rely on dendrite's RPW timer to catch it up.
+        if let Err(e) = dpd_client.ipv4_nat_trigger_update().await {
+            error!(self.log, "failed to notify dendrite of nat updates"; "error" => ?e);
+        };
+
+        Ok(())
+    }
+
     async fn ensure_nat_entry(
         &self,
         target_ip: &nexus_db_model::ExternalIp,
         sled_address: Ipv6Net,
-        network_interface: &sled_agent_client::types::NetworkInterface,
+        network_interface: &NetworkInterface,
         mac_address: macaddr::MacAddr6,
         opctx: &OpContext,
     ) -> Result<Ipv4NatEntry, Error> {
@@ -675,6 +771,87 @@ impl super::Nexus {
         Ok(())
     }
 
+    // The logic of this function should follow very closely what
+    // `instance_delete_dpd_config` does. However, there are enough differences
+    // in the mechanics of how the logic is being carried out to justify having
+    // this separate function, it seems.
+    pub(crate) async fn probe_delete_dpd_config(
+        &self,
+        opctx: &OpContext,
+        probe_id: Uuid,
+    ) -> Result<(), Error> {
+        let log = &self.log;
+
+        info!(log, "deleting probe dpd configuration";
+              "probe_id" => %probe_id);
+
+        let external_ips = self
+            .db_datastore
+            .probe_lookup_external_ips(opctx, probe_id)
+            .await?;
+
+        let mut errors = vec![];
+        for entry in external_ips {
+            // Soft delete the NAT entry
+            match self
+                .db_datastore
+                .ipv4_nat_delete_by_external_ip(&opctx, &entry)
+                .await
+            {
+                Ok(_) => Ok(()),
+                Err(err) => match err {
+                    Error::ObjectNotFound { .. } => {
+                        warn!(log, "no matching nat entries to soft delete");
+                        Ok(())
+                    }
+                    _ => {
+                        let message = format!(
+                            "failed to delete nat entry due to error: {err:?}"
+                        );
+                        error!(log, "{}", message);
+                        Err(Error::internal_error(&message))
+                    }
+                },
+            }?;
+        }
+
+        let boundary_switches =
+            self.boundary_switches(&self.opctx_alloc).await?;
+
+        for switch in &boundary_switches {
+            debug!(&self.log, "notifying dendrite of updates";
+                       "probe_id" => %probe_id,
+                       "switch" => switch.to_string());
+
+            let client_result = self.dpd_clients.get(switch).ok_or_else(|| {
+                Error::internal_error(&format!(
+                    "unable to find dendrite client for {switch}"
+                ))
+            });
+
+            let dpd_client = match client_result {
+                Ok(client) => client,
+                Err(new_error) => {
+                    errors.push(new_error);
+                    continue;
+                }
+            };
+
+            // Notify dendrite that there are changes for it to reconcile.
+            // In the event of a failure to notify dendrite, we'll log an error
+            // and rely on dendrite's RPW timer to catch it up.
+            if let Err(e) = dpd_client.ipv4_nat_trigger_update().await {
+                error!(self.log, "failed to notify dendrite of nat updates"; "error" => ?e);
+            };
+        }
+
+        if let Some(e) = errors.into_iter().nth(0) {
+            return Err(e);
+        }
+
+        Ok(())
+    }
+
     /// Deletes an instance's OPTE V2P mappings and the boundary switch NAT
     /// entries for its external IPs.
     ///
diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs
index 781e39ac83..68f57d31f3 100644
--- a/nexus/src/app/mod.rs
+++ b/nexus/src/app/mod.rs
@@ -55,6 +55,7 @@ mod ip_pool;
 mod metrics;
 mod network_interface;
 mod oximeter;
+mod probe;
 mod project;
 mod quota;
 mod rack;
diff --git a/nexus/src/app/probe.rs b/nexus/src/app/probe.rs
new file mode 100644
index 0000000000..0fce9d3431
--- /dev/null
+++ b/nexus/src/app/probe.rs
@@ -0,0 +1,109 @@
+use nexus_db_model::Probe;
+use nexus_db_queries::authz;
+use nexus_db_queries::context::OpContext;
+use nexus_db_queries::db::datastore::ProbeInfo;
+use nexus_db_queries::db::lookup;
+use nexus_types::external_api::params;
+use nexus_types::identity::Resource;
+use omicron_common::api::external::Error;
+use omicron_common::api::external::{
+    http_pagination::PaginatedBy, CreateResult, DataPageParams, DeleteResult,
+    ListResultVec, LookupResult, NameOrId,
+};
+use uuid::Uuid;
+
+impl super::Nexus {
+    /// List the probes in the given project.
+    pub(crate) async fn probe_list(
+        &self,
+        opctx: &OpContext,
+        project_lookup: &lookup::Project<'_>,
+        pagparams: &PaginatedBy<'_>,
+    ) -> ListResultVec<ProbeInfo> {
+        let (.., authz_project) =
+            project_lookup.lookup_for(authz::Action::ListChildren).await?;
+        self.db_datastore.probe_list(opctx, &authz_project, pagparams).await
+    }
+
+    /// List the probes for the given sled. This is used by sled agents to
+    /// determine what probes they should be running.
+    pub(crate) async fn probe_list_for_sled(
+        &self,
+        opctx: &OpContext,
+        pagparams: &DataPageParams<'_, Uuid>,
+        sled: Uuid,
+    ) -> ListResultVec<ProbeInfo> {
+        self.db_datastore.probe_list_for_sled(sled, opctx, pagparams).await
+    }
+
+    /// Get info about a particular probe.
+    pub(crate) async fn probe_get(
+        &self,
+        opctx: &OpContext,
+        project_lookup: &lookup::Project<'_>,
+        name_or_id: &NameOrId,
+    ) -> LookupResult<ProbeInfo> {
+        let (.., authz_project) =
+            project_lookup.lookup_for(authz::Action::CreateChild).await?;
+        self.db_datastore.probe_get(opctx, &authz_project, &name_or_id).await
+    }
+
+    /// Create a probe. This adds the probe to the data store and sets up the
+    /// NAT state on the switch. Actual launching of the probe is done by the
+    /// target sled agent asynchronously.
+    pub(crate) async fn probe_create(
+        &self,
+        opctx: &OpContext,
+        project_lookup: &lookup::Project<'_>,
+        new_probe_params: &params::ProbeCreate,
+    ) -> CreateResult<Probe> {
+        let (.., authz_project) =
+            project_lookup.lookup_for(authz::Action::CreateChild).await?;
+
+        let probe = self
+            .db_datastore
+            .probe_create(opctx, &authz_project, new_probe_params)
+            .await?;
+
+        let (.., sled) =
+            self.sled_lookup(opctx, &new_probe_params.sled)?.fetch().await?;
+
+        let boundary_switches =
+            self.boundary_switches(&self.opctx_alloc).await?;
+
+        for switch in &boundary_switches {
+            let dpd_client = self.dpd_clients.get(switch).ok_or_else(|| {
+                Error::internal_error(&format!(
+                    "could not find dpd client for {switch}"
+                ))
+            })?;
+            self.probe_ensure_dpd_config(
+                opctx,
+                probe.id(),
+                sled.ip.into(),
+                None,
+                dpd_client,
+            )
+            .await?;
+        }
+
+        Ok(probe)
+    }
+
+    /// Delete a probe. This deletes the probe from the data store and tears
+    /// down the associated NAT state.
+    pub(crate) async fn probe_delete(
+        &self,
+        opctx: &OpContext,
+        project_lookup: &lookup::Project<'_>,
+        name_or_id: NameOrId,
+    ) -> DeleteResult {
+        let probe = self.probe_get(opctx, project_lookup, &name_or_id).await?;
+
+        self.probe_delete_dpd_config(opctx, probe.id).await?;
+
+        let (.., authz_project) =
+            project_lookup.lookup_for(authz::Action::CreateChild).await?;
+        self.db_datastore.probe_delete(opctx, &authz_project, &name_or_id).await
+    }
+}
diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs
index a137f19434..4030fce31d 100644
--- a/nexus/src/app/rack.rs
+++ b/nexus/src/app/rack.rs
@@ -760,8 +760,9 @@ impl super::Nexus {
                 ntp_servers: Vec::new(), //TODO
                 rack_network_config: Some(RackNetworkConfigV1 {
                     rack_subnet: subnet,
-                    //TODO(ry) you are here. We need to remove these too. They are
-                    // inconsistent with a generic set of addresses on ports.
+                    //TODO: We need to remove these. They are inconsistent with
+                    // a generic set of addresses on ports that may not be
+                    // contiguous.
                     infra_ip_first: Ipv4Addr::UNSPECIFIED,
                     infra_ip_last: Ipv4Addr::UNSPECIFIED,
                     ports,
diff --git a/nexus/src/app/sagas/project_create.rs b/nexus/src/app/sagas/project_create.rs
index 40acc822c0..b31dd821f0 100644
--- a/nexus/src/app/sagas/project_create.rs
+++ b/nexus/src/app/sagas/project_create.rs
@@ -245,7 +245,6 @@ mod test {
                         .filter(dsl::collection_type.eq(nexus_db_queries::db::model::CollectionTypeProvisioned::Project.to_string()))
                         // ignore built-in services project
                         .filter(dsl::id.ne(*SERVICES_PROJECT_ID))
-
                         .select(VirtualProvisioningCollection::as_select())
                         .get_results_async::<VirtualProvisioningCollection>(&conn)
                         .await
diff --git a/nexus/src/app/switch_port.rs b/nexus/src/app/switch_port.rs
index b9f0f94fa0..fc9ad2866a 100644
--- a/nexus/src/app/switch_port.rs
+++ b/nexus/src/app/switch_port.rs
@@ -40,9 +40,9 @@ impl super::Nexus {
     ) -> CreateResult<SwitchPortSettingsCombinedResult> {
         opctx.authorize(authz::Action::Modify, &authz::FLEET).await?;
 
-        //TODO(ry) race conditions on exists check versus update/create.
-        //         Normally I would use a DB lock here, but not sure what
-        //         the Omicron way of doing things here is.
+        //TODO race conditions on exists check versus update/create.
+        //     Normally I would use a DB lock here, but not sure what
+        //     the Omicron way of doing things here is.
 
         match self
             .db_datastore
diff --git a/nexus/src/app/vpc.rs b/nexus/src/app/vpc.rs
index 3a6278053a..44b676b853 100644
--- a/nexus/src/app/vpc.rs
+++ b/nexus/src/app/vpc.rs
@@ -30,10 +30,10 @@ use omicron_common::api::external::NameOrId;
 use omicron_common::api::external::UpdateResult;
 use omicron_common::api::external::VpcFirewallRuleUpdateParams;
 use omicron_common::api::internal::nexus::HostIdentifier;
-use sled_agent_client::types::NetworkInterface;
 
 use futures::future::join_all;
 use ipnetwork::IpNetwork;
+use omicron_common::api::internal::shared::NetworkInterface;
 use std::collections::{HashMap, HashSet};
 use std::net::IpAddr;
 use std::sync::Arc;
diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs
index 8fe02a0fd8..aa037e072f 100644
--- a/nexus/src/external_api/http_entrypoints.rs
+++ b/nexus/src/external_api/http_entrypoints.rs
@@ -15,7 +15,6 @@ use super::{
 };
 use crate::external_api::shared;
 use crate::ServerContext;
-use dropshot::EmptyScanParams;
 use dropshot::HttpError;
 use dropshot::HttpResponseAccepted;
 use dropshot::HttpResponseCreated;
@@ -34,18 +33,20 @@ use dropshot::{
     channel, endpoint, WebsocketChannelResult, WebsocketConnection,
 };
 use dropshot::{ApiDescription, StreamingBody};
+use dropshot::{ApiEndpoint, EmptyScanParams};
 use ipnetwork::IpNetwork;
-use nexus_db_queries::authz;
 use nexus_db_queries::db;
 use nexus_db_queries::db::identity::Resource;
 use nexus_db_queries::db::lookup::ImageLookup;
 use nexus_db_queries::db::lookup::ImageParentLookup;
 use nexus_db_queries::db::model::Name;
+use nexus_db_queries::{authz, db::datastore::ProbeInfo};
 use nexus_types::external_api::shared::BfdStatus;
 use omicron_common::api::external::http_pagination::data_page_params_for;
 use omicron_common::api::external::http_pagination::marker_for_name;
 use omicron_common::api::external::http_pagination::marker_for_name_or_id;
 use omicron_common::api::external::http_pagination::name_or_id_pagination;
+use omicron_common::api::external::http_pagination::PaginatedBy;
 use omicron_common::api::external::http_pagination::PaginatedById;
 use omicron_common::api::external::http_pagination::PaginatedByName;
 use omicron_common::api::external::http_pagination::PaginatedByNameOrId;
@@ -69,6 +70,7 @@ use omicron_common::api::external::InstanceNetworkInterface;
 use omicron_common::api::external::InternalContext;
 use omicron_common::api::external::LoopbackAddress;
 use omicron_common::api::external::NameOrId;
+use omicron_common::api::external::Probe;
 use omicron_common::api::external::RouterRoute;
 use omicron_common::api::external::RouterRouteKind;
 use omicron_common::api::external::SwitchPort;
@@ -353,12 +355,40 @@ pub(crate) fn external_api() -> NexusApiDescription {
         Ok(())
     }
 
+    fn register_experimental<T>(
+        api: &mut NexusApiDescription,
+        endpoint: T,
+    ) -> Result<(), String>
+    where
+        T: Into<ApiEndpoint<Arc<ServerContext>>>,
+    {
+        let mut ep: ApiEndpoint<Arc<ServerContext>> = endpoint.into();
+        // only one tag is allowed
+        ep.tags = vec![String::from("hidden")];
+        ep.path = String::from("/experimental") + &ep.path;
+        api.register(ep)
+    }
+
+    fn register_experimental_endpoints(
+        api: &mut NexusApiDescription,
+    ) -> Result<(), String> {
+        register_experimental(api, probe_list)?;
+        register_experimental(api, probe_view)?;
+        register_experimental(api, probe_create)?;
+        register_experimental(api, probe_delete)?;
+
+        Ok(())
+    }
+
     let conf = serde_json::from_str(include_str!("./tag-config.json")).unwrap();
     let mut api = NexusApiDescription::new().tag_config(conf);
 
     if let Err(err) = register_endpoints(&mut api) {
         panic!("failed to register entrypoints: {}", err);
     }
+    if let Err(err) = register_experimental_endpoints(&mut api) {
+        panic!("failed to register experimental entrypoints: {}", err);
+    }
     api
 }
 
@@ -5999,6 +6029,125 @@ async fn current_user_ssh_key_delete(
     apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
 }
 
+/// List instrumentation probes
+#[endpoint {
+    method = GET,
+    path = "/v1/probes",
+    tags = ["system/probes"],
+}]
+async fn probe_list(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    query_params: Query<PaginatedByNameOrId<params::ProjectSelector>>,
+) -> Result<HttpResponseOk<ResultsPage<ProbeInfo>>, HttpError> {
+    let apictx = rqctx.context();
+    let handler = async {
+        let opctx = crate::context::op_context_for_external_api(&rqctx).await?;
+        opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?;
+
+        let nexus = &apictx.nexus;
+        let query = query_params.into_inner();
+        let pag_params = data_page_params_for(&rqctx, &query)?;
+        let scan_params = ScanByNameOrId::from_query(&query)?;
+        let paginated_by = name_or_id_pagination(&pag_params, scan_params)?;
+        let project_lookup =
+            nexus.project_lookup(&opctx, scan_params.selector.clone())?;
+
+        let probes =
+            nexus.probe_list(&opctx, &project_lookup, &paginated_by).await?;
+
+        Ok(HttpResponseOk(ScanByNameOrId::results_page(
+            &query,
+            probes,
+            &|_, p: &ProbeInfo| match paginated_by {
+                PaginatedBy::Id(_) => NameOrId::Id(p.id),
+                PaginatedBy::Name(_) => NameOrId::Name(p.name.clone().into()),
+            },
+        )?))
+    };
+    apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
+/// View instrumentation probe
+#[endpoint {
+    method = GET,
+    path = "/v1/probes/{probe}",
+    tags = ["system/probes"],
+}]
+async fn probe_view(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    path_params: Path<params::ProbePath>,
+    query_params: Query<params::ProjectSelector>,
+) -> Result<HttpResponseOk<ProbeInfo>, HttpError> {
+    let apictx = rqctx.context();
+    let handler = async {
+        let opctx = crate::context::op_context_for_external_api(&rqctx).await?;
+        opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?;
+
+        let nexus = &apictx.nexus;
+        let path = path_params.into_inner();
+        let project_selector = query_params.into_inner();
+        let project_lookup = nexus.project_lookup(&opctx, project_selector)?;
+        let probe =
+            nexus.probe_get(&opctx, &project_lookup, &path.probe).await?;
+        Ok(HttpResponseOk(probe))
+    };
+    apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
+/// Create instrumentation probe
+#[endpoint {
+    method = POST,
+    path = "/v1/probes",
+    tags = ["system/probes"],
+}]
+async fn probe_create(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    query_params: Query<params::ProjectSelector>,
+    new_probe: TypedBody<params::ProbeCreate>,
+) -> Result<HttpResponseCreated<Probe>, HttpError> {
+    let apictx = rqctx.context();
+    let handler = async {
+        let opctx = crate::context::op_context_for_external_api(&rqctx).await?;
+        opctx.authorize(authz::Action::Modify, &authz::FLEET).await?;
+
+        let nexus = &apictx.nexus;
+        let new_probe_params = &new_probe.into_inner();
+        let project_selector = query_params.into_inner();
+        let project_lookup = nexus.project_lookup(&opctx, project_selector)?;
+        let probe = nexus
+            .probe_create(&opctx, &project_lookup, &new_probe_params)
+            .await?;
+        Ok(HttpResponseCreated(probe.into()))
+    };
+    apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
+/// Delete instrumentation probe
+#[endpoint {
+    method = DELETE,
+    path = "/v1/probes/{probe}",
+    tags = ["system/probes"],
+}]
+async fn probe_delete(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    query_params: Query<params::ProjectSelector>,
+    path_params: Path<params::ProbePath>,
+) -> Result<HttpResponseDeleted, HttpError> {
+    let apictx = rqctx.context();
+    let handler = async {
+        let opctx = crate::context::op_context_for_external_api(&rqctx).await?;
+        opctx.authorize(authz::Action::Modify, &authz::FLEET).await?;
+
+        let nexus = &apictx.nexus;
+        let path = path_params.into_inner();
+        let project_selector = query_params.into_inner();
+        let project_lookup = nexus.project_lookup(&opctx, project_selector)?;
+        nexus.probe_delete(&opctx, &project_lookup, path.probe).await?;
+        Ok(HttpResponseDeleted())
+    };
+    apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
 #[cfg(test)]
 mod test {
     use super::external_api;
diff --git a/nexus/src/external_api/tag-config.json b/nexus/src/external_api/tag-config.json
index 3bc8006cee..6974906507 100644
--- a/nexus/src/external_api/tag-config.json
+++ b/nexus/src/external_api/tag-config.json
@@ -86,6 +86,12 @@
         "url": "http://docs.oxide.computer/api/vpcs"
       }
     },
+    "system/probes": {
+      "description": "Probes for testing network connectivity",
+      "external_docs": {
+        "url": "http://docs.oxide.computer/api/probes"
+      }
+    },
     "system/status": {
       "description": "Endpoints related to system health",
       "external_docs": {
diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs
index e298935fee..7f8211dc8e 100644
--- a/nexus/src/internal_api/http_entrypoints.rs
+++ b/nexus/src/internal_api/http_entrypoints.rs
@@ -25,6 +25,7 @@ use dropshot::ResultsPage;
 use dropshot::TypedBody;
 use hyper::Body;
 use nexus_db_model::Ipv4NatEntryView;
+use nexus_db_queries::db::datastore::ProbeInfo;
 use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::BlueprintMetadata;
 use nexus_types::deployment::BlueprintTarget;
@@ -94,6 +95,8 @@ pub(crate) fn internal_api() -> NexusApiDescription {
         api.register(sled_list_uninitialized)?;
         api.register(sled_add)?;
 
+        api.register(probes_get)?;
+
         Ok(())
     }
 
@@ -635,7 +638,7 @@ struct RpwNatQueryParam {
 /// change or until the `limit` is reached. If there are no changes, an
 /// empty vec is returned.
 #[endpoint {
-   method = GET,
+    method = GET,
     path = "/nat/ipv4/changeset/{from_gen}"
 }]
 async fn ipv4_nat_changeset(
@@ -864,3 +867,33 @@ async fn sled_add(
     };
     apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
 }
+
+/// Path parameters for probes
+#[derive(Deserialize, JsonSchema)]
+struct ProbePathParam {
+    sled: Uuid,
+}
+
+/// Get all the probes associated with a given sled.
+#[endpoint {
+    method = GET,
+    path = "/probes/{sled}"
+}]
+async fn probes_get(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    path_params: Path<ProbePathParam>,
+    query_params: Query<PaginatedById>,
+) -> Result<HttpResponseOk<Vec<ProbeInfo>>, HttpError> {
+    let apictx = rqctx.context();
+    let handler = async {
+        let query = query_params.into_inner();
+        let path = path_params.into_inner();
+        let nexus = &apictx.nexus;
+        let opctx = crate::context::op_context_for_internal_api(&rqctx).await;
+        let pagparams = data_page_params_for(&rqctx, &query)?;
+        Ok(HttpResponseOk(
+            nexus.probe_list_for_sled(&opctx, &pagparams, path.sled).await?,
+        ))
+    };
+    apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
diff --git a/nexus/tests/integration_tests/mod.rs b/nexus/tests/integration_tests/mod.rs
index 4b68a6c4f2..84b867252f 100644
--- a/nexus/tests/integration_tests/mod.rs
+++ b/nexus/tests/integration_tests/mod.rs
@@ -23,6 +23,7 @@ mod metrics;
 mod oximeter;
 mod pantry;
 mod password_login;
+mod probe;
 mod projects;
 mod quotas;
 mod rack;
diff --git a/nexus/tests/integration_tests/oximeter.rs b/nexus/tests/integration_tests/oximeter.rs
index 7dc453d713..9663e10fa0 100644
--- a/nexus/tests/integration_tests/oximeter.rs
+++ b/nexus/tests/integration_tests/oximeter.rs
@@ -152,7 +152,7 @@ async fn test_oximeter_reregistration() {
 
     // Timeouts for checks
     const POLL_INTERVAL: Duration = Duration::from_millis(100);
-    const POLL_DURATION: Duration = Duration::from_secs(30);
+    const POLL_DURATION: Duration = Duration::from_secs(60);
 
     // We must have at exactly one timeseries, with at least one sample.
     let timeseries =
diff --git a/nexus/tests/integration_tests/probe.rs b/nexus/tests/integration_tests/probe.rs
new file mode 100644
index 0000000000..71a695bf8c
--- /dev/null
+++ b/nexus/tests/integration_tests/probe.rs
@@ -0,0 +1,127 @@
+use dropshot::HttpErrorResponseBody;
+use http::{Method, StatusCode};
+use nexus_db_queries::db::datastore::ProbeInfo;
+use nexus_test_utils::{
+    http_testing::{AuthnMode, NexusRequest},
+    resource_helpers::{create_default_ip_pool, create_project},
+    SLED_AGENT_UUID,
+};
+use nexus_test_utils_macros::nexus_test;
+use nexus_types::external_api::params::ProbeCreate;
+use omicron_common::api::external::{IdentityMetadataCreateParams, Probe};
+
+type ControlPlaneTestContext =
+    nexus_test_utils::ControlPlaneTestContext<omicron_nexus::Server>;
+
+#[nexus_test]
+async fn test_probe_basic_crud(ctx: &ControlPlaneTestContext) {
+    let client = &ctx.external_client;
+
+    create_default_ip_pool(&client).await;
+    create_project(&client, "nebula").await;
+
+    let probes = NexusRequest::iter_collection_authn::<ProbeInfo>(
+        client,
+        "/experimental/v1/probes?project=nebula",
+        "",
+        None,
+    )
+    .await
+    .expect("Failed to list probes")
+    .all_items;
+
+    assert_eq!(probes.len(), 0, "Expected zero probes");
+
+    let params = ProbeCreate {
+        identity: IdentityMetadataCreateParams {
+            name: "class1".parse().unwrap(),
+            description: "subspace relay probe".to_owned(),
+        },
+        ip_pool: None,
+        sled: SLED_AGENT_UUID.parse().unwrap(),
+    };
+
+    let created: Probe = NexusRequest::objects_post(
+        client,
+        "/experimental/v1/probes?project=nebula",
+        &params,
+    )
+    .authn_as(AuthnMode::PrivilegedUser)
+    .execute()
+    .await
+    .unwrap()
+    .parsed_body()
+    .unwrap();
+
+    let probes = NexusRequest::iter_collection_authn::<ProbeInfo>(
+        client,
+        "/experimental/v1/probes?project=nebula",
+        "",
+        None,
+    )
+    .await
+    .expect("Failed to list probes")
+    .all_items;
+
+    assert_eq!(probes.len(), 1, "Expected one probe");
+    assert_eq!(probes[0].id, created.identity.id);
+
+    let error: HttpErrorResponseBody = NexusRequest::expect_failure(
+        client,
+        StatusCode::NOT_FOUND,
+        Method::GET,
+        "/experimental/v1/probes/class2?project=nebula",
+    )
+    .authn_as(AuthnMode::PrivilegedUser)
+    .execute()
+    .await
+    .unwrap()
+    .parsed_body()
+    .unwrap();
+    assert_eq!(error.message, "not found: probe with name \"class2\"");
+
+    NexusRequest::object_get(
+        client,
+        "/experimental/v1/probes/class1?project=nebula",
+    )
+    .authn_as(AuthnMode::PrivilegedUser)
+    .execute()
+    .await
+    .expect("failed to view probe")
+    .parsed_body::<ProbeInfo>()
+    .expect("failed to parse probe info");
+
+    let fetched: ProbeInfo = NexusRequest::object_get(
+        client,
+        "/experimental/v1/probes/class1?project=nebula",
+    )
+    .authn_as(AuthnMode::PrivilegedUser)
+    .execute()
+    .await
+    .unwrap()
+    .parsed_body()
+    .unwrap();
+
+    assert_eq!(fetched.id, created.identity.id);
+
+    NexusRequest::object_delete(
+        client,
+        "/experimental/v1/probes/class1?project=nebula",
+    )
+    .authn_as(AuthnMode::PrivilegedUser)
+    .execute()
+    .await
+    .unwrap();
+
+    let probes = NexusRequest::iter_collection_authn::<ProbeInfo>(
+        client,
+        "/experimental/v1/probes?project=nebula",
+        "",
+        None,
+    )
+    .await
+    .expect("Failed to list probes after delete")
+    .all_items;
+
+    assert_eq!(probes.len(), 0, "Expected zero probes");
+}
diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt
index ecffadcb4d..3b954ec6ec 100644
--- a/nexus/tests/output/nexus_tags.txt
+++ b/nexus/tests/output/nexus_tags.txt
@@ -26,6 +26,10 @@ device_access_token                      POST     /device/token
 device_auth_confirm                      POST     /device/confirm
 device_auth_request                      POST     /device/auth
 logout                                   POST     /v1/logout
+probe_create                             POST     /experimental/v1/probes
+probe_delete                             DELETE   /experimental/v1/probes/{probe}
+probe_list                               GET      /experimental/v1/probes
+probe_view                               GET      /experimental/v1/probes/{probe}
 
 API operations found with tag "images"
 OPERATION ID                             METHOD   URL PATH
diff --git a/nexus/tests/output/uncovered-authz-endpoints.txt b/nexus/tests/output/uncovered-authz-endpoints.txt
index d76d9c5495..d19c7970d0 100644
--- a/nexus/tests/output/uncovered-authz-endpoints.txt
+++ b/nexus/tests/output/uncovered-authz-endpoints.txt
@@ -1,8 +1,12 @@
 API endpoints with no coverage in authz tests:
+probe_delete                             (delete "/experimental/v1/probes/{probe}")
+probe_list                               (get    "/experimental/v1/probes")
+probe_view                               (get    "/experimental/v1/probes/{probe}")
 ping                                     (get    "/v1/ping")
 device_auth_request                      (post   "/device/auth")
 device_auth_confirm                      (post   "/device/confirm")
 device_access_token                      (post   "/device/token")
+probe_create                             (post   "/experimental/v1/probes")
 login_saml                               (post   "/login/{silo_name}/saml/{provider_name}")
 login_local                              (post   "/v1/login/{silo_name}/local")
 logout                                   (post   "/v1/logout")
diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs
index ac950d2ca3..7dcb843b7f 100644
--- a/nexus/types/src/deployment.rs
+++ b/nexus/types/src/deployment.rs
@@ -15,8 +15,6 @@
 use crate::external_api::views::SledPolicy;
 use crate::external_api::views::SledState;
 use crate::inventory::Collection;
-pub use crate::inventory::NetworkInterface;
-pub use crate::inventory::NetworkInterfaceKind;
 pub use crate::inventory::OmicronZoneConfig;
 pub use crate::inventory::OmicronZoneDataset;
 pub use crate::inventory::OmicronZoneType;
diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs
index 31cb1d3e5c..c6ae47d27c 100644
--- a/nexus/types/src/external_api/params.rs
+++ b/nexus/types/src/external_api/params.rs
@@ -80,6 +80,7 @@ path_param!(ProviderPath, provider, "SAML identity provider");
 path_param!(IpPoolPath, pool, "IP pool");
 path_param!(SshKeyPath, ssh_key, "SSH key");
 path_param!(AddressLotPath, address_lot, "address lot");
+path_param!(ProbePath, probe, "probe");
 
 id_path_param!(GroupPath, group_id, "group");
 
@@ -2042,3 +2043,21 @@ pub struct UpdatesGetRepositoryParams {
     /// The version to get.
     pub system_version: SemverVersion,
 }
+
+// Probes
+
+/// Create time parameters for probes.
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
+pub struct ProbeCreate {
+    #[serde(flatten)]
+    pub identity: IdentityMetadataCreateParams,
+    pub sled: Uuid,
+    pub ip_pool: Option<NameOrId>,
+}
+
+/// List probes with an optional name or id.
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)]
+pub struct ProbeListSelector {
+    /// A name or id to use when selecting a probe.
+    pub name_or_id: Option<NameOrId>,
+}
diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs
index 57fc7bd647..92985081dc 100644
--- a/nexus/types/src/inventory.rs
+++ b/nexus/types/src/inventory.rs
@@ -17,11 +17,11 @@ pub use gateway_client::types::PowerState;
 pub use gateway_client::types::RotSlot;
 pub use gateway_client::types::SpType;
 use omicron_common::api::external::ByteCount;
+pub use omicron_common::api::internal::shared::NetworkInterface;
+pub use omicron_common::api::internal::shared::NetworkInterfaceKind;
 pub use omicron_common::api::internal::shared::SourceNatConfig;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
-pub use sled_agent_client::types::NetworkInterface;
-pub use sled_agent_client::types::NetworkInterfaceKind;
 pub use sled_agent_client::types::OmicronZoneConfig;
 pub use sled_agent_client::types::OmicronZoneDataset;
 pub use sled_agent_client::types::OmicronZoneType;
diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json
index fb3ff976ae..9e18c7d6bc 100644
--- a/openapi/nexus-internal.json
+++ b/openapi/nexus-internal.json
@@ -667,6 +667,75 @@
         }
       }
     },
+    "/probes/{sled}": {
+      "get": {
+        "summary": "Get all the probes associated with a given sled.",
+        "operationId": "probes_get",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "sled",
+            "required": true,
+            "schema": {
+              "type": "string",
+              "format": "uuid"
+            }
+          },
+          {
+            "in": "query",
+            "name": "limit",
+            "description": "Maximum number of items returned by a single call",
+            "schema": {
+              "nullable": true,
+              "type": "integer",
+              "format": "uint32",
+              "minimum": 1
+            }
+          },
+          {
+            "in": "query",
+            "name": "page_token",
+            "description": "Token returned by previous call to retrieve the subsequent page",
+            "schema": {
+              "nullable": true,
+              "type": "string"
+            }
+          },
+          {
+            "in": "query",
+            "name": "sort_by",
+            "schema": {
+              "$ref": "#/components/schemas/IdSortMode"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "title": "Array_of_ProbeInfo",
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/components/schemas/ProbeInfo"
+                  }
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        },
+        "x-dropshot-pagination": {
+          "required": []
+        }
+      }
+    },
     "/racks/{rack_id}/initialization-complete": {
       "put": {
         "summary": "Report that the Rack Setup Service initialization is complete",
@@ -4358,14 +4427,31 @@
           }
         ]
       },
+      "IpKind": {
+        "type": "string",
+        "enum": [
+          "snat",
+          "floating",
+          "ephemeral"
+        ]
+      },
       "IpNet": {
-        "description": "IpNet\n\n<details><summary>JSON schema</summary>\n\n```json { \"oneOf\": [ { \"title\": \"v4\", \"allOf\": [ { \"$ref\": \"#/components/schemas/Ipv4Net\" } ] }, { \"title\": \"v6\", \"allOf\": [ { \"$ref\": \"#/components/schemas/Ipv6Net\" } ] } ] } ``` </details>",
-        "anyOf": [
+        "oneOf": [
           {
-            "$ref": "#/components/schemas/Ipv4Net"
+            "title": "v4",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Ipv4Net"
+              }
+            ]
           },
           {
-            "$ref": "#/components/schemas/Ipv6Net"
+            "title": "v6",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Ipv6Net"
+              }
+            ]
           }
         ]
       },
@@ -4457,8 +4543,11 @@
         ]
       },
       "Ipv4Net": {
-        "description": "An IPv4 subnet, including prefix and subnet mask\n\n<details><summary>JSON schema</summary>\n\n```json { \"title\": \"An IPv4 subnet\", \"description\": \"An IPv4 subnet, including prefix and subnet mask\", \"examples\": [ \"192.168.1.0/24\" ], \"type\": \"string\", \"pattern\": \"^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\\\\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])/([0-9]|1[0-9]|2[0-9]|3[0-2])$\" } ``` </details>",
-        "type": "string"
+        "example": "192.168.1.0/24",
+        "title": "An IPv4 subnet",
+        "description": "An IPv4 subnet, including prefix and subnet mask",
+        "type": "string",
+        "pattern": "^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])/([0-9]|1[0-9]|2[0-9]|3[0-2])$"
       },
       "Ipv4Network": {
         "type": "string",
@@ -4483,8 +4572,11 @@
         ]
       },
       "Ipv6Net": {
-        "description": "An IPv6 subnet, including prefix and subnet mask\n\n<details><summary>JSON schema</summary>\n\n```json { \"title\": \"An IPv6 subnet\", \"description\": \"An IPv6 subnet, including prefix and subnet mask\", \"examples\": [ \"fd12:3456::/64\" ], \"type\": \"string\", \"pattern\": \"^([fF][dD])[0-9a-fA-F]{2}:(([0-9a-fA-F]{1,4}:){6}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,6}:)([0-9a-fA-F]{1,4})?\\\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$\" } ``` </details>",
-        "type": "string"
+        "example": "fd12:3456::/64",
+        "title": "An IPv6 subnet",
+        "description": "An IPv6 subnet, including prefix and subnet mask",
+        "type": "string",
+        "pattern": "^([fF][dD])[0-9a-fA-F]{2}:(([0-9a-fA-F]{1,4}:){6}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,6}:)([0-9a-fA-F]{1,4})?\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$"
       },
       "Ipv6Network": {
         "type": "string",
@@ -4825,7 +4917,7 @@
         "maxLength": 63
       },
       "NetworkInterface": {
-        "description": "Information required to construct a virtual network interface\n\n<details><summary>JSON schema</summary>\n\n```json { \"description\": \"Information required to construct a virtual network interface\", \"type\": \"object\", \"required\": [ \"id\", \"ip\", \"kind\", \"mac\", \"name\", \"primary\", \"slot\", \"subnet\", \"vni\" ], \"properties\": { \"id\": { \"type\": \"string\", \"format\": \"uuid\" }, \"ip\": { \"type\": \"string\", \"format\": \"ip\" }, \"kind\": { \"$ref\": \"#/components/schemas/NetworkInterfaceKind\" }, \"mac\": { \"$ref\": \"#/components/schemas/MacAddr\" }, \"name\": { \"$ref\": \"#/components/schemas/Name\" }, \"primary\": { \"type\": \"boolean\" }, \"slot\": { \"type\": \"integer\", \"format\": \"uint8\", \"minimum\": 0.0 }, \"subnet\": { \"$ref\": \"#/components/schemas/IpNet\" }, \"vni\": { \"$ref\": \"#/components/schemas/Vni\" } } } ``` </details>",
+        "description": "Information required to construct a virtual network interface",
         "type": "object",
         "properties": {
           "id": {
@@ -4873,7 +4965,7 @@
         ]
       },
       "NetworkInterfaceKind": {
-        "description": "The type of network interface\n\n<details><summary>JSON schema</summary>\n\n```json { \"description\": \"The type of network interface\", \"oneOf\": [ { \"description\": \"A vNIC attached to a guest instance\", \"type\": \"object\", \"required\": [ \"id\", \"type\" ], \"properties\": { \"id\": { \"type\": \"string\", \"format\": \"uuid\" }, \"type\": { \"type\": \"string\", \"enum\": [ \"instance\" ] } } }, { \"description\": \"A vNIC associated with an internal service\", \"type\": \"object\", \"required\": [ \"id\", \"type\" ], \"properties\": { \"id\": { \"type\": \"string\", \"format\": \"uuid\" }, \"type\": { \"type\": \"string\", \"enum\": [ \"service\" ] } } } ] } ``` </details>",
+        "description": "The type of network interface",
         "oneOf": [
           {
             "description": "A vNIC attached to a guest instance",
@@ -4914,6 +5006,26 @@
               "id",
               "type"
             ]
+          },
+          {
+            "description": "A vNIC associated with a probe",
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "format": "uuid"
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "probe"
+                ]
+              }
+            },
+            "required": [
+              "id",
+              "type"
+            ]
           }
         ]
       },
@@ -5504,6 +5616,66 @@
           "speed400_g"
         ]
       },
+      "ProbeExternalIp": {
+        "type": "object",
+        "properties": {
+          "first_port": {
+            "type": "integer",
+            "format": "uint16",
+            "minimum": 0
+          },
+          "ip": {
+            "type": "string",
+            "format": "ip"
+          },
+          "kind": {
+            "$ref": "#/components/schemas/IpKind"
+          },
+          "last_port": {
+            "type": "integer",
+            "format": "uint16",
+            "minimum": 0
+          }
+        },
+        "required": [
+          "first_port",
+          "ip",
+          "kind",
+          "last_port"
+        ]
+      },
+      "ProbeInfo": {
+        "type": "object",
+        "properties": {
+          "external_ips": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/ProbeExternalIp"
+            }
+          },
+          "id": {
+            "type": "string",
+            "format": "uuid"
+          },
+          "interface": {
+            "$ref": "#/components/schemas/NetworkInterface"
+          },
+          "name": {
+            "$ref": "#/components/schemas/Name"
+          },
+          "sled": {
+            "type": "string",
+            "format": "uuid"
+          }
+        },
+        "required": [
+          "external_ips",
+          "id",
+          "interface",
+          "name",
+          "sled"
+        ]
+      },
       "ProducerEndpoint": {
         "description": "Information announced by a metric server, used so that clients can contact it and collect available metric data from it.",
         "type": "object",
diff --git a/openapi/nexus.json b/openapi/nexus.json
index 3d31331a90..cacd875acc 100644
--- a/openapi/nexus.json
+++ b/openapi/nexus.json
@@ -101,6 +101,206 @@
         }
       }
     },
+    "/experimental/v1/probes": {
+      "get": {
+        "tags": [
+          "hidden"
+        ],
+        "summary": "List instrumentation probes",
+        "operationId": "probe_list",
+        "parameters": [
+          {
+            "in": "query",
+            "name": "limit",
+            "description": "Maximum number of items returned by a single call",
+            "schema": {
+              "nullable": true,
+              "type": "integer",
+              "format": "uint32",
+              "minimum": 1
+            }
+          },
+          {
+            "in": "query",
+            "name": "page_token",
+            "description": "Token returned by previous call to retrieve the subsequent page",
+            "schema": {
+              "nullable": true,
+              "type": "string"
+            }
+          },
+          {
+            "in": "query",
+            "name": "project",
+            "description": "Name or ID of the project",
+            "schema": {
+              "$ref": "#/components/schemas/NameOrId"
+            }
+          },
+          {
+            "in": "query",
+            "name": "sort_by",
+            "schema": {
+              "$ref": "#/components/schemas/NameOrIdSortMode"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ProbeInfoResultsPage"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        },
+        "x-dropshot-pagination": {
+          "required": [
+            "project"
+          ]
+        }
+      },
+      "post": {
+        "tags": [
+          "hidden"
+        ],
+        "summary": "Create instrumentation probe",
+        "operationId": "probe_create",
+        "parameters": [
+          {
+            "in": "query",
+            "name": "project",
+            "description": "Name or ID of the project",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/NameOrId"
+            }
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/ProbeCreate"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "201": {
+            "description": "successful creation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/Probe"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
+    "/experimental/v1/probes/{probe}": {
+      "get": {
+        "tags": [
+          "hidden"
+        ],
+        "summary": "View instrumentation probe",
+        "operationId": "probe_view",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "probe",
+            "description": "Name or ID of the probe",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/NameOrId"
+            }
+          },
+          {
+            "in": "query",
+            "name": "project",
+            "description": "Name or ID of the project",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/NameOrId"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ProbeInfo"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      },
+      "delete": {
+        "tags": [
+          "hidden"
+        ],
+        "summary": "Delete instrumentation probe",
+        "operationId": "probe_delete",
+        "parameters": [
+          {
+            "in": "query",
+            "name": "project",
+            "description": "Name or ID of the project",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/NameOrId"
+            }
+          },
+          {
+            "in": "path",
+            "name": "probe",
+            "description": "Name or ID of the probe",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/NameOrId"
+            }
+          }
+        ],
+        "responses": {
+          "204": {
+            "description": "successful deletion"
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/login/{silo_name}/saml/{provider_name}": {
       "post": {
         "tags": [
@@ -13210,6 +13410,14 @@
           }
         ]
       },
+      "IpKind": {
+        "type": "string",
+        "enum": [
+          "snat",
+          "floating",
+          "ephemeral"
+        ]
+      },
       "IpNet": {
         "oneOf": [
           {
@@ -13914,6 +14122,119 @@
           }
         ]
       },
+      "NetworkInterface": {
+        "description": "Information required to construct a virtual network interface",
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string",
+            "format": "uuid"
+          },
+          "ip": {
+            "type": "string",
+            "format": "ip"
+          },
+          "kind": {
+            "$ref": "#/components/schemas/NetworkInterfaceKind"
+          },
+          "mac": {
+            "$ref": "#/components/schemas/MacAddr"
+          },
+          "name": {
+            "$ref": "#/components/schemas/Name"
+          },
+          "primary": {
+            "type": "boolean"
+          },
+          "slot": {
+            "type": "integer",
+            "format": "uint8",
+            "minimum": 0
+          },
+          "subnet": {
+            "$ref": "#/components/schemas/IpNet"
+          },
+          "vni": {
+            "$ref": "#/components/schemas/Vni"
+          }
+        },
+        "required": [
+          "id",
+          "ip",
+          "kind",
+          "mac",
+          "name",
+          "primary",
+          "slot",
+          "subnet",
+          "vni"
+        ]
+      },
+      "NetworkInterfaceKind": {
+        "description": "The type of network interface",
+        "oneOf": [
+          {
+            "description": "A vNIC attached to a guest instance",
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "format": "uuid"
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "instance"
+                ]
+              }
+            },
+            "required": [
+              "id",
+              "type"
+            ]
+          },
+          {
+            "description": "A vNIC associated with an internal service",
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "format": "uuid"
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "service"
+                ]
+              }
+            },
+            "required": [
+              "id",
+              "type"
+            ]
+          },
+          {
+            "description": "A vNIC associated with a probe",
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "format": "uuid"
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "probe"
+                ]
+              }
+            },
+            "required": [
+              "id",
+              "type"
+            ]
+          }
+        ]
+      },
       "Password": {
         "title": "A password used to authenticate a user",
         "description": "Passwords may be subject to additional constraints.",
@@ -14019,6 +14340,161 @@
           "ok"
         ]
       },
+      "Probe": {
+        "description": "Identity-related metadata that's included in nearly all public API objects",
+        "type": "object",
+        "properties": {
+          "description": {
+            "description": "human-readable free-form text about a resource",
+            "type": "string"
+          },
+          "id": {
+            "description": "unique, immutable, system-controlled identifier for each resource",
+            "type": "string",
+            "format": "uuid"
+          },
+          "name": {
+            "description": "unique, mutable, user-controlled identifier for each resource",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Name"
+              }
+            ]
+          },
+          "sled": {
+            "type": "string",
+            "format": "uuid"
+          },
+          "time_created": {
+            "description": "timestamp when this resource was created",
+            "type": "string",
+            "format": "date-time"
+          },
+          "time_modified": {
+            "description": "timestamp when this resource was last modified",
+            "type": "string",
+            "format": "date-time"
+          }
+        },
+        "required": [
+          "description",
+          "id",
+          "name",
+          "sled",
+          "time_created",
+          "time_modified"
+        ]
+      },
+      "ProbeCreate": {
+        "description": "Create time parameters for probes.",
+        "type": "object",
+        "properties": {
+          "description": {
+            "type": "string"
+          },
+          "ip_pool": {
+            "nullable": true,
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/NameOrId"
+              }
+            ]
+          },
+          "name": {
+            "$ref": "#/components/schemas/Name"
+          },
+          "sled": {
+            "type": "string",
+            "format": "uuid"
+          }
+        },
+        "required": [
+          "description",
+          "name",
+          "sled"
+        ]
+      },
+      "ProbeExternalIp": {
+        "type": "object",
+        "properties": {
+          "first_port": {
+            "type": "integer",
+            "format": "uint16",
+            "minimum": 0
+          },
+          "ip": {
+            "type": "string",
+            "format": "ip"
+          },
+          "kind": {
+            "$ref": "#/components/schemas/IpKind"
+          },
+          "last_port": {
+            "type": "integer",
+            "format": "uint16",
+            "minimum": 0
+          }
+        },
+        "required": [
+          "first_port",
+          "ip",
+          "kind",
+          "last_port"
+        ]
+      },
+      "ProbeInfo": {
+        "type": "object",
+        "properties": {
+          "external_ips": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/ProbeExternalIp"
+            }
+          },
+          "id": {
+            "type": "string",
+            "format": "uuid"
+          },
+          "interface": {
+            "$ref": "#/components/schemas/NetworkInterface"
+          },
+          "name": {
+            "$ref": "#/components/schemas/Name"
+          },
+          "sled": {
+            "type": "string",
+            "format": "uuid"
+          }
+        },
+        "required": [
+          "external_ips",
+          "id",
+          "interface",
+          "name",
+          "sled"
+        ]
+      },
+      "ProbeInfoResultsPage": {
+        "description": "A single page of results",
+        "type": "object",
+        "properties": {
+          "items": {
+            "description": "list of items on this page of results",
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/ProbeInfo"
+            }
+          },
+          "next_page": {
+            "nullable": true,
+            "description": "token used to fetch the next page of results (if any)",
+            "type": "string"
+          }
+        },
+        "required": [
+          "items"
+        ]
+      },
       "Project": {
         "description": "View of a Project",
         "type": "object",
@@ -16503,6 +16979,12 @@
           "storage"
         ]
       },
+      "Vni": {
+        "description": "A Geneve Virtual Network Identifier",
+        "type": "integer",
+        "format": "uint32",
+        "minimum": 0
+      },
       "Vpc": {
         "description": "View of a VPC",
         "type": "object",
@@ -17452,6 +17934,13 @@
         "url": "http://docs.oxide.computer/api/system-networking"
       }
     },
+    {
+      "name": "system/probes",
+      "description": "Probes for testing network connectivity",
+      "externalDocs": {
+        "url": "http://docs.oxide.computer/api/probes"
+      }
+    },
     {
       "name": "system/silos",
       "description": "Silos represent a logical partition of users and resources.",
diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json
index 238b5832ca..43fac710fc 100644
--- a/openapi/sled-agent.json
+++ b/openapi/sled-agent.json
@@ -5670,6 +5670,26 @@
               "id",
               "type"
             ]
+          },
+          {
+            "description": "A vNIC associated with a probe",
+            "type": "object",
+            "properties": {
+              "id": {
+                "type": "string",
+                "format": "uuid"
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "probe"
+                ]
+              }
+            },
+            "required": [
+              "id",
+              "type"
+            ]
           }
         ]
       },
diff --git a/package-manifest.toml b/package-manifest.toml
index c6f39d2ecd..b7e96935f1 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -626,6 +626,15 @@ output.type = "zone"
 output.intermediate_only = true
 setup_hint = "Run `./tools/ci_download_transceiver_control` to download the necessary binaries"
 
+[package.thundermuffin]
+service_name = "thundermuffin"
+source.type = "prebuilt"
+source.repo = "thundermuffin"
+source.commit = "a4a6108d7c9aac2464a0b6898e88132a8f701a13"
+source.sha256 = "dc55a2accd33a347df4cbdc0026cbaccea2c004940c3fec8cadcdd633d440dfa"
+output.type = "zone"
+output.intermediate_only = true
+
 # To package and install the asic variant of the switch, do:
 #
 # $ cargo run --release --bin omicron-package -- -t default target create -i standard -m gimlet -s asic
@@ -740,3 +749,11 @@ source.type = "local"
 source.rust.binary_names = ["oxlog"]
 source.rust.release = true
 output.type = "tarball"
+
+[package.probe]
+service_name = "probe"
+source.type = "composite"
+source.packages = [
+  "thundermuffin.tar.gz",
+]
+output.type = "zone"
diff --git a/schema/all-zone-requests.json b/schema/all-zone-requests.json
index 8c324a15bd..e37fbfde59 100644
--- a/schema/all-zone-requests.json
+++ b/schema/all-zone-requests.json
@@ -302,6 +302,26 @@
               ]
             }
           }
+        },
+        {
+          "description": "A vNIC associated with a probe",
+          "type": "object",
+          "required": [
+            "id",
+            "type"
+          ],
+          "properties": {
+            "id": {
+              "type": "string",
+              "format": "uuid"
+            },
+            "type": {
+              "type": "string",
+              "enum": [
+                "probe"
+              ]
+            }
+          }
         }
       ]
     },
diff --git a/schema/all-zones-requests.json b/schema/all-zones-requests.json
index 7a07e2f9ae..0ac9e760a8 100644
--- a/schema/all-zones-requests.json
+++ b/schema/all-zones-requests.json
@@ -186,6 +186,26 @@
               ]
             }
           }
+        },
+        {
+          "description": "A vNIC associated with a probe",
+          "type": "object",
+          "required": [
+            "id",
+            "type"
+          ],
+          "properties": {
+            "id": {
+              "type": "string",
+              "format": "uuid"
+            },
+            "type": {
+              "type": "string",
+              "enum": [
+                "probe"
+              ]
+            }
+          }
         }
       ]
     },
diff --git a/schema/crdb/40.0.0/up1.sql b/schema/crdb/40.0.0/up1.sql
new file mode 100644
index 0000000000..7fc8c01713
--- /dev/null
+++ b/schema/crdb/40.0.0/up1.sql
@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS omicron.public.probe (
+    id UUID NOT NULL PRIMARY KEY,
+    name STRING(63) NOT NULL,
+    description STRING(512) NOT NULL,
+    time_created TIMESTAMPTZ NOT NULL,
+    time_modified TIMESTAMPTZ NOT NULL,
+    time_deleted TIMESTAMPTZ,
+    project_id UUID NOT NULL,
+    sled UUID NOT NULL
+);
diff --git a/schema/crdb/40.0.0/up2.sql b/schema/crdb/40.0.0/up2.sql
new file mode 100644
index 0000000000..6c070463a4
--- /dev/null
+++ b/schema/crdb/40.0.0/up2.sql
@@ -0,0 +1,4 @@
+CREATE UNIQUE INDEX IF NOT EXISTS lookup_probe_by_name ON omicron.public.probe (
+    name
+) WHERE
+    time_deleted IS NULL;
diff --git a/schema/crdb/40.0.0/up3.sql b/schema/crdb/40.0.0/up3.sql
new file mode 100644
index 0000000000..3b71ba4313
--- /dev/null
+++ b/schema/crdb/40.0.0/up3.sql
@@ -0,0 +1 @@
+ALTER TABLE omicron.public.external_ip ADD COLUMN IF NOT EXISTS is_probe BOOL NOT NULL DEFAULT false;
diff --git a/schema/crdb/40.0.0/up4.sql b/schema/crdb/40.0.0/up4.sql
new file mode 100644
index 0000000000..6c989cf8c8
--- /dev/null
+++ b/schema/crdb/40.0.0/up4.sql
@@ -0,0 +1 @@
+ALTER TYPE omicron.public.network_interface_kind ADD VALUE IF NOT EXISTS 'probe';
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 79b6131d85..27faf3f79f 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -3548,6 +3548,26 @@ SELECT
   deleted
 FROM interleaved_versions;
 
+CREATE TABLE IF NOT EXISTS omicron.public.probe (
+    id UUID NOT NULL PRIMARY KEY,
+    name STRING(63) NOT NULL,
+    description STRING(512) NOT NULL,
+    time_created TIMESTAMPTZ NOT NULL,
+    time_modified TIMESTAMPTZ NOT NULL,
+    time_deleted TIMESTAMPTZ,
+    project_id UUID NOT NULL,
+    sled UUID NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS lookup_probe_by_name ON omicron.public.probe (
+    name
+) WHERE
+    time_deleted IS NULL;
+
+ALTER TABLE omicron.public.external_ip ADD COLUMN IF NOT EXISTS is_probe BOOL NOT NULL DEFAULT false;
+
+ALTER TYPE omicron.public.network_interface_kind ADD VALUE IF NOT EXISTS 'probe';
+
 INSERT INTO omicron.public.db_metadata (
     singleton,
     time_created,
@@ -3555,7 +3575,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '39.0.0', NULL)
+    ( TRUE, NOW(), NOW(), '40.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;
diff --git a/schema/rss-service-plan-v2.json b/schema/rss-service-plan-v2.json
index 10d8f8ab95..ee0b21af81 100644
--- a/schema/rss-service-plan-v2.json
+++ b/schema/rss-service-plan-v2.json
@@ -274,6 +274,26 @@
               ]
             }
           }
+        },
+        {
+          "description": "A vNIC associated with a probe",
+          "type": "object",
+          "required": [
+            "id",
+            "type"
+          ],
+          "properties": {
+            "id": {
+              "type": "string",
+              "format": "uuid"
+            },
+            "type": {
+              "type": "string",
+              "enum": [
+                "probe"
+              ]
+            }
+          }
         }
       ]
     },
diff --git a/sled-agent/src/lib.rs b/sled-agent/src/lib.rs
index bfc23b248d..a4686bdb88 100644
--- a/sled-agent/src/lib.rs
+++ b/sled-agent/src/lib.rs
@@ -30,6 +30,7 @@ mod long_running_tasks;
 mod metrics;
 mod nexus;
 pub mod params;
+mod probe_manager;
 mod profile;
 pub mod rack_setup;
 pub mod server;
diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs
index d192f745f6..f30c910efc 100644
--- a/sled-agent/src/params.rs
+++ b/sled-agent/src/params.rs
@@ -742,7 +742,7 @@ impl From<OmicronZoneType> for sled_agent_client::types::OmicronZoneType {
                 domain,
                 ntp_servers,
                 snat_cfg,
-                nic: nic.into(),
+                nic: nic,
             },
             OmicronZoneType::Clickhouse { address, dataset } => {
                 Other::Clickhouse {
@@ -778,7 +778,7 @@ impl From<OmicronZoneType> for sled_agent_client::types::OmicronZoneType {
                 dataset: dataset.into(),
                 http_address: http_address.to_string(),
                 dns_address: dns_address.to_string(),
-                nic: nic.into(),
+                nic: nic,
             },
             OmicronZoneType::InternalDns {
                 dataset,
@@ -815,7 +815,7 @@ impl From<OmicronZoneType> for sled_agent_client::types::OmicronZoneType {
                 external_ip,
                 external_tls,
                 internal_address: internal_address.to_string(),
-                nic: nic.into(),
+                nic: nic,
             },
             OmicronZoneType::Oximeter { address } => {
                 Other::Oximeter { address: address.to_string() }
diff --git a/sled-agent/src/probe_manager.rs b/sled-agent/src/probe_manager.rs
new file mode 100644
index 0000000000..8481dc4b79
--- /dev/null
+++ b/sled-agent/src/probe_manager.rs
@@ -0,0 +1,383 @@
+use crate::nexus::NexusClientWithResolver;
+use anyhow::{anyhow, Result};
+use illumos_utils::dladm::Etherstub;
+use illumos_utils::link::VnicAllocator;
+use illumos_utils::opte::params::VpcFirewallRule;
+use illumos_utils::opte::{DhcpCfg, PortManager};
+use illumos_utils::running_zone::{RunningZone, ZoneBuilderFactory};
+use illumos_utils::zone::Zones;
+use nexus_client::types::{ProbeExternalIp, ProbeInfo};
+use omicron_common::api::external::{
+    VpcFirewallRuleAction, VpcFirewallRuleDirection, VpcFirewallRulePriority,
+    VpcFirewallRuleStatus,
+};
+use omicron_common::api::internal::shared::NetworkInterface;
+use rand::prelude::SliceRandom;
+use rand::SeedableRng;
+use sled_storage::dataset::ZONE_DATASET;
+use sled_storage::manager::StorageHandle;
+use slog::{error, warn, Logger};
+use std::collections::{HashMap, HashSet};
+use std::hash::{Hash, Hasher};
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Mutex;
+use tokio::task::JoinHandle;
+use tokio::time::sleep;
+use uuid::Uuid;
+use zone::Zone;
+
+/// Prefix used for probe zone names
+const PROBE_ZONE_PREFIX: &str = "oxz_probe";
+
+/// How long to wait between check-ins with nexus
+const RECONCILIATION_INTERVAL: Duration = Duration::from_secs(1);
+
+/// The scope to use when allocating VNICs
+const VNIC_ALLOCATOR_SCOPE: &str = "probe";
+
+/// The probe manager periodically asks nexus what probes it should be running.
+/// It checks the probes it should be running versus the probes it's actually
+/// running and reconciles any differences.
+pub(crate) struct ProbeManager {
+    inner: Arc<ProbeManagerInner>,
+}
+
+pub(crate) struct ProbeManagerInner {
+    join_handle: Mutex<Option<JoinHandle<()>>>,
+    nexus_client: NexusClientWithResolver,
+    log: Logger,
+    sled_id: Uuid,
+    vnic_allocator: VnicAllocator<Etherstub>,
+    storage: StorageHandle,
+    port_manager: PortManager,
+    running_probes: Mutex<HashMap<Uuid, RunningZone>>,
+}
+
+impl ProbeManager {
+    pub(crate) fn new(
+        sled_id: Uuid,
+        nexus_client: NexusClientWithResolver,
+        etherstub: Etherstub,
+        storage: StorageHandle,
+        port_manager: PortManager,
+        log: Logger,
+    ) -> Self {
+        Self {
+            inner: Arc::new(ProbeManagerInner {
+                join_handle: Mutex::new(None),
+                vnic_allocator: VnicAllocator::new(
+                    VNIC_ALLOCATOR_SCOPE,
+                    etherstub,
+                ),
+                running_probes: Mutex::new(HashMap::new()),
+                nexus_client,
+                log,
+                sled_id,
+                storage,
+                port_manager,
+            }),
+        }
+    }
+
+    pub(crate) async fn run(&self) {
+        self.inner.run().await;
+    }
+}
+
+/// State information about a probe. This is a common representation that
+/// captures elements from both the nexus and running-zone representation of a
+/// probe.
+#[derive(Debug, Clone)]
+struct ProbeState {
+    /// Id as determined by nexus
+    id: Uuid,
+    /// Runtime state on this sled
+    status: zone::State,
+    /// The external IP addresses the probe has been assigned.
+    external_ips: Vec<ProbeExternalIp>,
+    /// The probes networking interface.
+    interface: Option<NetworkInterface>,
+}
+
+impl PartialEq for ProbeState {
+    fn eq(&self, other: &Self) -> bool {
+        self.id.eq(&other.id)
+    }
+}
+
+impl Eq for ProbeState {}
+
+impl Hash for ProbeState {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.id.hash(state)
+    }
+}
+
+/// Translate from the nexus API `ProbeInfo` into a `ProbeState`
+impl From<ProbeInfo> for ProbeState {
+    fn from(value: ProbeInfo) -> Self {
+        Self {
+            id: value.id,
+            status: zone::State::Running,
+            external_ips: value.external_ips,
+            interface: Some(value.interface),
+        }
+    }
+}
+
+/// Translate from running zone state into a `ProbeState`
+impl TryFrom<Zone> for ProbeState {
+    type Error = String;
+    fn try_from(value: Zone) -> std::result::Result<Self, Self::Error> {
+        Ok(Self {
+            id: value
+                .name()
+                .strip_prefix(&format!("{PROBE_ZONE_PREFIX}_"))
+                .ok_or(String::from("not a probe prefix"))?
+                .parse()
+                .map_err(|e| format!("invalid uuid: {e}"))?,
+            status: value.state(),
+            external_ips: Vec::new(),
+            interface: None,
+        })
+    }
+}
+
+impl ProbeManagerInner {
+    /// Run the probe manager. If it's already running this is a no-op.
+    async fn run(self: &Arc<Self>) {
+        let mut join_handle = self.join_handle.lock().await;
+        if join_handle.is_none() {
+            *join_handle = Some(self.clone().reconciler())
+        }
+    }
+
+    /// Run the reconciler loop on a background thread.
+    fn reconciler(self: Arc<Self>) -> JoinHandle<()> {
+        tokio::spawn(async move {
+            loop {
+                sleep(RECONCILIATION_INTERVAL).await;
+
+                // Collect the target and current state. Use set operations
+                // to determine what probes need to be added, removed and/or
+                // modified.
+
+                let target = match self.target_state().await {
+                    Ok(state) => state,
+                    Err(e) => {
+                        error!(self.log, "get target probe state: {e}");
+                        continue;
+                    }
+                };
+
+                let current = match self.current_state().await {
+                    Ok(state) => state,
+                    Err(e) => {
+                        error!(self.log, "get current probe state: {e}");
+                        continue;
+                    }
+                };
+
+                self.add(target.difference(&current)).await;
+                self.remove(current.difference(&target)).await;
+                self.check(current.intersection(&target)).await;
+            }
+        })
+    }
+
+    /// Add a set of probes to this sled.
+    async fn add<'a, I>(self: &Arc<Self>, probes: I)
+    where
+        I: Iterator<Item = &'a ProbeState>,
+    {
+        for probe in probes {
+            info!(self.log, "adding probe {}", probe.id);
+            if let Err(e) = self.add_probe(probe).await {
+                error!(self.log, "add probe: {e}");
+            }
+        }
+    }
+
+    /// Add a probe to this sled. This sets up resources for the probe zone
+    /// such as storage and networking. Then it configures, installs and
+    /// boots the probe zone.
+    async fn add_probe(self: &Arc<Self>, probe: &ProbeState) -> Result<()> {
+        let mut rng = rand::rngs::StdRng::from_entropy();
+        let root = self
+            .storage
+            .get_latest_resources()
+            .await
+            .all_u2_mountpoints(ZONE_DATASET)
+            .choose(&mut rng)
+            .ok_or_else(|| anyhow!("u2 not found"))?
+            .clone();
+
+        let nic = probe
+            .interface
+            .as_ref()
+            .ok_or(anyhow!("no interface specified for probe"))?;
+
+        let eip = probe
+            .external_ips
+            .get(0)
+            .ok_or(anyhow!("expected an external ip"))?;
+
+        let port = self.port_manager.create_port(
+            &nic,
+            None,
+            Some(eip.ip),
+            &[], // floating ips
+            &[VpcFirewallRule {
+                status: VpcFirewallRuleStatus::Enabled,
+                direction: VpcFirewallRuleDirection::Inbound,
+                targets: vec![nic.clone()],
+                filter_hosts: None,
+                filter_ports: None,
+                filter_protocols: None,
+                action: VpcFirewallRuleAction::Allow,
+                priority: VpcFirewallRulePriority(100),
+            }],
+            DhcpCfg::default(),
+        )?;
+
+        let installed_zone = ZoneBuilderFactory::default()
+            .builder()
+            .with_log(self.log.clone())
+            .with_underlay_vnic_allocator(&self.vnic_allocator)
+            .with_zone_root_path(&root)
+            .with_zone_image_paths(&["/opt/oxide".into()])
+            .with_zone_type("probe")
+            .with_unique_name(probe.id)
+            .with_datasets(&[])
+            .with_filesystems(&[])
+            .with_data_links(&[])
+            .with_devices(&[])
+            .with_opte_ports(vec![port])
+            .with_links(vec![])
+            .with_limit_priv(vec![])
+            .install()
+            .await?;
+
+        info!(self.log, "installed probe {}", probe.id);
+
+        //TODO SMF properties for probe services?
+
+        let rz = RunningZone::boot(installed_zone).await?;
+        rz.ensure_address_for_port("overlay", 0).await?;
+        info!(self.log, "started probe {}", probe.id);
+
+        self.running_probes.lock().await.insert(probe.id, rz);
+
+        Ok(())
+    }
+
+    /// Remove a set of probes from this sled.
+    async fn remove<'a, I>(self: &Arc<Self>, probes: I)
+    where
+        I: Iterator<Item = &'a ProbeState>,
+    {
+        for probe in probes {
+            info!(self.log, "removing probe {}", probe.id);
+            self.remove_probe(probe.id).await;
+        }
+    }
+
+    /// Remove a probe from this sled. This tears down the zone and it's
+    /// network resources.
+    async fn remove_probe(self: &Arc<Self>, id: Uuid) {
+        match self.running_probes.lock().await.remove(&id) {
+            Some(mut running_zone) => {
+                for l in running_zone.links_mut() {
+                    if let Err(e) = l.delete() {
+                        error!(self.log, "delete probe link {}: {e}", l.name());
+                    }
+                }
+                running_zone.release_opte_ports();
+                if let Err(e) = running_zone.stop().await {
+                    error!(self.log, "stop probe: {e}")
+                }
+                // TODO are there storage resources that need to be cleared
+                // out here too?
+            }
+            None => {
+                warn!(self.log, "attempt to stop non-running probe: {id}")
+            }
+        }
+    }
+
+    /// Check that probes that should be running are running, and with the
+    /// correct configuration.
+    async fn check<'a, I>(self: &Arc<Self>, probes: I)
+    where
+        I: Iterator<Item = &'a ProbeState>,
+    {
+        for probe in probes {
+            if probe.status == zone::State::Running {
+                continue;
+            }
+            warn!(
+                self.log,
+                "probe {} found in unexpected state {:?}",
+                probe.id,
+                probe.status
+            )
+            //TODO somehow handle the hooligans here?
+        }
+    }
+
+    /// Collect target probe state from the nexus internal API.
+    async fn target_state(self: &Arc<Self>) -> Result<HashSet<ProbeState>> {
+        Ok(self
+            .nexus_client
+            .client()
+            .probes_get(
+                &self.sled_id,
+                None, //limit
+                None, //page token
+                None, //sort by
+            )
+            .await?
+            .into_inner()
+            .into_iter()
+            .map(Into::into)
+            .collect())
+    }
+
+    /// Collect the current probe state from the running zones on this sled.
+    async fn current_state(self: &Arc<Self>) -> Result<HashSet<ProbeState>> {
+        Ok(Zones::get()
+            .await?
+            .into_iter()
+            .filter_map(|z| ProbeState::try_from(z).ok())
+            .collect())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use uuid::Uuid;
+
+    #[test]
+    fn probe_state_set_ops() {
+        let a = ProbeState {
+            id: Uuid::new_v4(),
+            status: zone::State::Configured,
+            external_ips: Vec::new(),
+            interface: None,
+        };
+
+        let mut b = a.clone();
+        b.status = zone::State::Running;
+
+        let target = HashSet::from([a]);
+        let current = HashSet::from([b]);
+
+        let to_add = target.difference(&current);
+        let to_remove = current.difference(&target);
+
+        assert_eq!(to_add.count(), 0);
+        assert_eq!(to_remove.count(), 0);
+    }
+}
diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs
index 8f737f879b..4a21a6fe89 100644
--- a/sled-agent/src/sled_agent.rs
+++ b/sled-agent/src/sled_agent.rs
@@ -25,6 +25,7 @@ use crate::params::{
     OmicronZonesConfig, SledRole, TimeSync, VpcFirewallRule,
     ZoneBundleMetadata, Zpool,
 };
+use crate::probe_manager::ProbeManager;
 use crate::services::{self, ServiceManager};
 use crate::storage_monitor::UnderlayAccess;
 use crate::updates::{ConfigUpdates, UpdateManager};
@@ -309,6 +310,9 @@ struct SledAgentInner {
 
     // Handle to the traffic manager for writing OS updates to our boot disks.
     boot_disk_os_writer: BootDiskOsWriter,
+
+    // Component of Sled Agent responsible for managing instrumentation probes.
+    probes: ProbeManager,
 }
 
 impl SledAgentInner {
@@ -571,6 +575,15 @@ impl SledAgent {
             nexus_notifier_task.run().await;
         });
 
+        let probes = ProbeManager::new(
+            request.body.id,
+            nexus_client.clone(),
+            etherstub.clone(),
+            storage_manager.clone(),
+            port_manager.clone(),
+            log.new(o!("component" => "ProbeManager")),
+        );
+
         let sled_agent = SledAgent {
             inner: Arc::new(SledAgentInner {
                 id: request.body.id,
@@ -578,6 +591,7 @@ impl SledAgent {
                 start_request: request,
                 storage: long_running_task_handles.storage_manager.clone(),
                 instances,
+                probes,
                 hardware: long_running_task_handles.hardware_manager.clone(),
                 updates,
                 port_manager,
@@ -593,6 +607,8 @@ impl SledAgent {
             log: log.clone(),
         };
 
+        sled_agent.inner.probes.run().await;
+
         // We immediately add a notification to the request queue about our
         // existence. If inspection of the hardware later informs us that we're
         // actually running on a scrimlet, that's fine, the updated value will
diff --git a/tools/ci_download_maghemite_mgd b/tools/ci_download_maghemite_mgd
index 9890e4505e..bf6be1d5b1 100755
--- a/tools/ci_download_maghemite_mgd
+++ b/tools/ci_download_maghemite_mgd
@@ -29,6 +29,8 @@ PACKAGE_BASE_URL="$ARTIFACT_URL/$REPO/image/$COMMIT"
 
 function main
 {
+    rm -rf $DOWNLOAD_DIR/root
+
 	#
 	# Process command-line arguments. We generally don't expect any, but
 	# we allow callers to specify a value to override OSTYPE, just for
diff --git a/tools/ci_download_maghemite_openapi b/tools/ci_download_maghemite_openapi
index 56ce640a76..7255e57cf4 100755
--- a/tools/ci_download_maghemite_openapi
+++ b/tools/ci_download_maghemite_openapi
@@ -15,10 +15,10 @@ TARGET_DIR="out"
 # Location where intermediate artifacts are downloaded / unpacked.
 DOWNLOAD_DIR="$TARGET_DIR/downloads"
 
-
-
 function main
 {
+    rm -rf $DOWNLOAD_DIR/root
+
 	if [[ $# != 0 ]]; then
 		echo "unexpected arguments" >&2
 		exit 2
diff --git a/tools/ci_download_thundermuffin b/tools/ci_download_thundermuffin
new file mode 100755
index 0000000000..014d1b30b2
--- /dev/null
+++ b/tools/ci_download_thundermuffin
@@ -0,0 +1,153 @@
+#!/bin/bash
+
+#
+# ci_download_probe_packages: fetches thundermuffin binary tarball package,
+# unpacks it, and creates a copy, all in the current directory
+#
+
+set -o pipefail
+set -o xtrace
+set -o errexit
+
+SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+ARG0="$(basename "${BASH_SOURCE[0]}")"
+
+source "$SOURCE_DIR/thundermuffin_checksums"
+source "$SOURCE_DIR/thundermuffin_version"
+
+TARGET_DIR="out"
+# Location where intermediate artifacts are downloaded / unpacked.
+DOWNLOAD_DIR="$TARGET_DIR/downloads"
+# Location where the final thundermuffin directory should end up.
+DEST_DIR="./$TARGET_DIR/thundermuffin"
+BIN_DIR="$DEST_DIR/root/opt/oxide/thundermuffin/bin"
+
+ARTIFACT_URL="https://buildomat.eng.oxide.computer/public/file"
+
+REPO='oxidecomputer/thundermuffin'
+PACKAGE_BASE_URL="$ARTIFACT_URL/$REPO/image/$COMMIT"
+
+function main
+{
+    rm -rf $DOWNLOAD_DIR/root
+
+	#
+	# Process command-line arguments. We generally don't expect any, but
+	# we allow callers to specify a value to override OSTYPE, just for
+	# testing.
+	#
+	if [[ $# != 0 ]]; then
+		CIDL_OS="$1"
+		shift
+	else
+		CIDL_OS="$OSTYPE"
+	fi
+
+	if [[ $# != 0 ]]; then
+		echo "unexpected arguments" >&2
+		exit 2
+	fi
+
+	# Configure this program
+	configure_os "$CIDL_OS"
+
+	CIDL_SHA256FUNC="do_sha256sum"
+	TARBALL_FILENAME="thundermuffin.tar.gz"
+	PACKAGE_URL="$PACKAGE_BASE_URL/$TARBALL_FILENAME"
+	TARBALL_FILE="$DOWNLOAD_DIR/$TARBALL_FILENAME"
+
+	# Download the file.
+	echo "URL: $PACKAGE_URL"
+	echo "Local file: $TARBALL_FILE"
+
+	mkdir -p "$DOWNLOAD_DIR"
+	mkdir -p "$DEST_DIR"
+
+	fetch_and_verify
+
+	do_untar "$TARBALL_FILE"
+
+	do_assemble
+
+	$SET_BINARIES
+}
+
+function fail
+{
+	echo "$ARG0: $@" >&2
+	exit 1
+}
+
+function configure_os
+{
+	echo "current directory: $PWD"
+	echo "configuring based on OS: \"$1\""
+	case "$1" in
+		solaris*)
+			SET_BINARIES=""
+			;;
+		*)
+			echo "WARNING: binaries for $1 are not published by thundermuffin"
+			SET_BINARIES="unsupported_os"
+			;;
+	esac
+}
+
+function do_download_curl
+{
+	curl --silent --show-error --fail --location --output "$2" "$1"
+}
+
+function do_sha256sum
+{
+	sha256sum < "$1" | awk '{print $1}'
+}
+
+function do_untar
+{
+	tar xzf "$1" -C "$DOWNLOAD_DIR"
+}
+
+function do_assemble
+{
+	rm -r "$DEST_DIR" || true
+	mkdir "$DEST_DIR"
+	cp -r "$DOWNLOAD_DIR/root" "$DEST_DIR/root"
+}
+
+function fetch_and_verify
+{
+	local DO_DOWNLOAD="true"
+	if [[ -f "$TARBALL_FILE" ]]; then
+		# If the file exists with a valid checksum, we can skip downloading.
+		calculated_sha256="$($CIDL_SHA256FUNC "$TARBALL_FILE")" || \
+				fail "failed to calculate sha256sum"
+		if [[ "$calculated_sha256" == "$CIDL_SHA256" ]]; then
+			DO_DOWNLOAD="false"
+		fi
+	fi
+
+	if [ "$DO_DOWNLOAD" == "true" ]; then
+		echo "Downloading..."
+		do_download_curl "$PACKAGE_URL" "$TARBALL_FILE" || \
+				fail "failed to download file"
+
+		# Verify the sha256sum.
+		calculated_sha256="$($CIDL_SHA256FUNC "$TARBALL_FILE")" || \
+				fail "failed to calculate sha256sum"
+		if [[ "$calculated_sha256" != "$CIDL_SHA256" ]]; then
+			fail "sha256sum mismatch \
+					(expected $CIDL_SHA256, found $calculated_sha256)"
+		fi
+	fi
+
+}
+
+function unsupported_os
+{
+	mkdir -p "$BIN_DIR"
+	echo "echo 'unsupported os' && exit 1" >> "$BIN_DIR/thundermuffin"
+	chmod +x "$BIN_DIR/thundermuffin"
+}
+
+main "$@"
diff --git a/tools/install_builder_prerequisites.sh b/tools/install_builder_prerequisites.sh
index 5fa8ec11ba..09b8a27677 100755
--- a/tools/install_builder_prerequisites.sh
+++ b/tools/install_builder_prerequisites.sh
@@ -217,6 +217,9 @@ retry ./tools/ci_download_maghemite_mgd
 # xcvradm binary which is bundled with the switch zone.
 retry ./tools/ci_download_transceiver_control
 
+# Download thundermuffin. This is required to launch network probes.
+retry ./tools/ci_download_thundermuffin
+
 # Validate the PATH:
 expected_in_path=(
   'pg_config'
diff --git a/tools/thundermuffin_checksums b/tools/thundermuffin_checksums
new file mode 100644
index 0000000000..5e10539bdd
--- /dev/null
+++ b/tools/thundermuffin_checksums
@@ -0,0 +1 @@
+CIDL_SHA256="dc55a2accd33a347df4cbdc0026cbaccea2c004940c3fec8cadcdd633d440dfa"
diff --git a/tools/thundermuffin_version b/tools/thundermuffin_version
new file mode 100644
index 0000000000..cbca739f5c
--- /dev/null
+++ b/tools/thundermuffin_version
@@ -0,0 +1 @@
+COMMIT="a4a6108d7c9aac2464a0b6898e88132a8f701a13"
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index e70c7c329e..5efbb6c1f1 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -14,26 +14,26 @@ publish = false
 
 ### BEGIN HAKARI SECTION
 [dependencies]
-ahash = { version = "0.8.7" }
-aho-corasick = { version = "1.0.4" }
-anyhow = { version = "1.0.75", features = ["backtrace"] }
+ahash = { version = "0.8.8" }
+aho-corasick = { version = "1.1.2" }
+anyhow = { version = "1.0.79", features = ["backtrace"] }
 base16ct = { version = "0.2.0", default-features = false, features = ["alloc"] }
 bit-set = { version = "0.5.3" }
 bit-vec = { version = "0.6.3" }
 bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" }
-bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["serde"] }
+bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.2", default-features = false, features = ["serde"] }
 bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2.17" }
-bstr-dff4ba8e3ae991db = { package = "bstr", version = "1.6.0" }
+bstr-dff4ba8e3ae991db = { package = "bstr", version = "1.9.0" }
 byteorder = { version = "1.5.0" }
 bytes = { version = "1.5.0", features = ["serde"] }
-chrono = { version = "0.4.31", features = ["alloc", "serde"] }
+chrono = { version = "0.4.34", features = ["serde"] }
 cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] }
-clap = { version = "4.5.0", features = ["cargo", "derive", "env", "wrap_help"] }
-clap_builder = { version = "4.5.0", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
+clap = { version = "4.5.1", features = ["cargo", "derive", "env", "wrap_help"] }
+clap_builder = { version = "4.5.1", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
 console = { version = "0.15.8" }
-const-oid = { version = "0.9.5", default-features = false, features = ["db", "std"] }
-crossbeam-epoch = { version = "0.9.15" }
-crossbeam-utils = { version = "0.8.16" }
+const-oid = { version = "0.9.6", default-features = false, features = ["db", "std"] }
+crossbeam-epoch = { version = "0.9.18" }
+crossbeam-utils = { version = "0.8.19" }
 crossterm = { version = "0.27.0", features = ["event-stream", "serde"] }
 crypto-common = { version = "0.1.6", default-features = false, features = ["getrandom", "std"] }
 der = { version = "0.7.8", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] }
@@ -52,12 +52,12 @@ futures-task = { version = "0.3.30", default-features = false, features = ["std"
 futures-util = { version = "0.3.30", features = ["channel", "io", "sink"] }
 gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", features = ["std"] }
 generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] }
-getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] }
+getrandom = { version = "0.2.12", default-features = false, features = ["js", "rdrand", "std"] }
 group = { version = "0.13.0", default-features = false, features = ["alloc"] }
 hashbrown = { version = "0.14.3", features = ["raw"] }
 hex = { version = "0.4.3", features = ["serde"] }
 hmac = { version = "0.12.1", default-features = false, features = ["reset"] }
-hyper = { version = "0.14.27", features = ["full"] }
+hyper = { version = "0.14.28", features = ["full"] }
 indexmap = { version = "2.2.5", features = ["serde"] }
 inout = { version = "0.1.3", default-features = false, features = ["std"] }
 ipnetwork = { version = "0.20.0", features = ["schemars"] }
@@ -67,12 +67,12 @@ lazy_static = { version = "1.4.0", default-features = false, features = ["spin_n
 libc = { version = "0.2.153", features = ["extra_traits"] }
 log = { version = "0.4.20", default-features = false, features = ["std"] }
 managed = { version = "0.8.0", default-features = false, features = ["alloc", "map"] }
-memchr = { version = "2.6.3" }
+memchr = { version = "2.7.1" }
 nom = { version = "7.1.3" }
 num-bigint = { version = "0.4.4", features = ["rand"] }
 num-integer = { version = "0.1.46", features = ["i128"] }
-num-iter = { version = "0.1.43", default-features = false, features = ["i128"] }
-num-traits = { version = "0.2.16", features = ["i128", "libm"] }
+num-iter = { version = "0.1.44", default-features = false, features = ["i128"] }
+num-traits = { version = "0.2.18", features = ["i128", "libm"] }
 openapiv3 = { version = "2.0.0", default-features = false, features = ["skip_serializing_defaults"] }
 pem-rfc7468 = { version = "0.7.0", default-features = false, features = ["std"] }
 petgraph = { version = "0.6.4", features = ["serde-1"] }
@@ -83,33 +83,32 @@ proc-macro2 = { version = "1.0.78" }
 rand = { version = "0.8.5" }
 rand_chacha = { version = "0.3.1", default-features = false, features = ["std"] }
 regex = { version = "1.10.3" }
-regex-automata = { version = "0.4.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
+regex-automata = { version = "0.4.5", default-features = false, features = ["dfa-onepass", "dfa-search", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
 regex-syntax = { version = "0.8.2" }
-reqwest = { version = "0.11.24", features = ["blocking", "json", "rustls-tls", "stream"] }
+reqwest = { version = "0.11.24", features = ["blocking", "cookies", "json", "rustls-tls", "stream"] }
 ring = { version = "0.17.8", features = ["std"] }
 schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid1"] }
 semver = { version = "1.0.22", features = ["serde"] }
 serde = { version = "1.0.197", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1.0.114", features = ["raw_value", "unbounded_depth"] }
 sha2 = { version = "0.10.8", features = ["oid"] }
-similar = { version = "2.3.0", features = ["inline", "unicode"] }
+similar = { version = "2.4.0", features = ["inline", "unicode"] }
 slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] }
-socket2 = { version = "0.5.5", default-features = false, features = ["all"] }
 spin = { version = "0.9.8" }
 string_cache = { version = "0.8.7" }
 subtle = { version = "2.5.0" }
 syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] }
 syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.51", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
-time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] }
+time = { version = "0.3.34", features = ["formatting", "local-offset", "macros", "parsing"] }
 tokio = { version = "1.36.0", features = ["full", "test-util"] }
 tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] }
 tokio-stream = { version = "0.1.14", features = ["net"] }
 tokio-util = { version = "0.7.10", features = ["codec", "io-util"] }
 toml = { version = "0.7.8" }
 toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.6", features = ["serde"] }
-tracing = { version = "0.1.37", features = ["log"] }
+tracing = { version = "0.1.40", features = ["log"] }
 trust-dns-proto = { version = "0.22.0" }
-unicode-bidi = { version = "0.3.13" }
+unicode-bidi = { version = "0.3.15" }
 unicode-normalization = { version = "0.1.22" }
 usdt = { version = "0.3.5" }
 usdt-impl = { version = "0.5.0", default-features = false, features = ["asm", "des"] }
@@ -120,26 +119,26 @@ zeroize = { version = "1.7.0", features = ["std", "zeroize_derive"] }
 zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] }
 
 [build-dependencies]
-ahash = { version = "0.8.7" }
-aho-corasick = { version = "1.0.4" }
-anyhow = { version = "1.0.75", features = ["backtrace"] }
+ahash = { version = "0.8.8" }
+aho-corasick = { version = "1.1.2" }
+anyhow = { version = "1.0.79", features = ["backtrace"] }
 base16ct = { version = "0.2.0", default-features = false, features = ["alloc"] }
 bit-set = { version = "0.5.3" }
 bit-vec = { version = "0.6.3" }
 bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" }
-bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["serde"] }
+bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.2", default-features = false, features = ["serde"] }
 bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2.17" }
-bstr-dff4ba8e3ae991db = { package = "bstr", version = "1.6.0" }
+bstr-dff4ba8e3ae991db = { package = "bstr", version = "1.9.0" }
 byteorder = { version = "1.5.0" }
 bytes = { version = "1.5.0", features = ["serde"] }
-chrono = { version = "0.4.31", features = ["alloc", "serde"] }
+chrono = { version = "0.4.34", features = ["serde"] }
 cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] }
-clap = { version = "4.5.0", features = ["cargo", "derive", "env", "wrap_help"] }
-clap_builder = { version = "4.5.0", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
+clap = { version = "4.5.1", features = ["cargo", "derive", "env", "wrap_help"] }
+clap_builder = { version = "4.5.1", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
 console = { version = "0.15.8" }
-const-oid = { version = "0.9.5", default-features = false, features = ["db", "std"] }
-crossbeam-epoch = { version = "0.9.15" }
-crossbeam-utils = { version = "0.8.16" }
+const-oid = { version = "0.9.6", default-features = false, features = ["db", "std"] }
+crossbeam-epoch = { version = "0.9.18" }
+crossbeam-utils = { version = "0.8.19" }
 crossterm = { version = "0.27.0", features = ["event-stream", "serde"] }
 crypto-common = { version = "0.1.6", default-features = false, features = ["getrandom", "std"] }
 der = { version = "0.7.8", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] }
@@ -158,12 +157,12 @@ futures-task = { version = "0.3.30", default-features = false, features = ["std"
 futures-util = { version = "0.3.30", features = ["channel", "io", "sink"] }
 gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", features = ["std"] }
 generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] }
-getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] }
+getrandom = { version = "0.2.12", default-features = false, features = ["js", "rdrand", "std"] }
 group = { version = "0.13.0", default-features = false, features = ["alloc"] }
 hashbrown = { version = "0.14.3", features = ["raw"] }
 hex = { version = "0.4.3", features = ["serde"] }
 hmac = { version = "0.12.1", default-features = false, features = ["reset"] }
-hyper = { version = "0.14.27", features = ["full"] }
+hyper = { version = "0.14.28", features = ["full"] }
 indexmap = { version = "2.2.5", features = ["serde"] }
 inout = { version = "0.1.3", default-features = false, features = ["std"] }
 ipnetwork = { version = "0.20.0", features = ["schemars"] }
@@ -173,12 +172,12 @@ lazy_static = { version = "1.4.0", default-features = false, features = ["spin_n
 libc = { version = "0.2.153", features = ["extra_traits"] }
 log = { version = "0.4.20", default-features = false, features = ["std"] }
 managed = { version = "0.8.0", default-features = false, features = ["alloc", "map"] }
-memchr = { version = "2.6.3" }
+memchr = { version = "2.7.1" }
 nom = { version = "7.1.3" }
 num-bigint = { version = "0.4.4", features = ["rand"] }
 num-integer = { version = "0.1.46", features = ["i128"] }
-num-iter = { version = "0.1.43", default-features = false, features = ["i128"] }
-num-traits = { version = "0.2.16", features = ["i128", "libm"] }
+num-iter = { version = "0.1.44", default-features = false, features = ["i128"] }
+num-traits = { version = "0.2.18", features = ["i128", "libm"] }
 openapiv3 = { version = "2.0.0", default-features = false, features = ["skip_serializing_defaults"] }
 pem-rfc7468 = { version = "0.7.0", default-features = false, features = ["std"] }
 petgraph = { version = "0.6.4", features = ["serde-1"] }
@@ -189,34 +188,33 @@ proc-macro2 = { version = "1.0.78" }
 rand = { version = "0.8.5" }
 rand_chacha = { version = "0.3.1", default-features = false, features = ["std"] }
 regex = { version = "1.10.3" }
-regex-automata = { version = "0.4.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
+regex-automata = { version = "0.4.5", default-features = false, features = ["dfa-onepass", "dfa-search", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
 regex-syntax = { version = "0.8.2" }
-reqwest = { version = "0.11.24", features = ["blocking", "json", "rustls-tls", "stream"] }
+reqwest = { version = "0.11.24", features = ["blocking", "cookies", "json", "rustls-tls", "stream"] }
 ring = { version = "0.17.8", features = ["std"] }
 schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid1"] }
 semver = { version = "1.0.22", features = ["serde"] }
 serde = { version = "1.0.197", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1.0.114", features = ["raw_value", "unbounded_depth"] }
 sha2 = { version = "0.10.8", features = ["oid"] }
-similar = { version = "2.3.0", features = ["inline", "unicode"] }
+similar = { version = "2.4.0", features = ["inline", "unicode"] }
 slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] }
-socket2 = { version = "0.5.5", default-features = false, features = ["all"] }
 spin = { version = "0.9.8" }
 string_cache = { version = "0.8.7" }
 subtle = { version = "2.5.0" }
 syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] }
 syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.51", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
-time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] }
-time-macros = { version = "0.2.13", default-features = false, features = ["formatting", "parsing"] }
+time = { version = "0.3.34", features = ["formatting", "local-offset", "macros", "parsing"] }
+time-macros = { version = "0.2.17", default-features = false, features = ["formatting", "parsing"] }
 tokio = { version = "1.36.0", features = ["full", "test-util"] }
 tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] }
 tokio-stream = { version = "0.1.14", features = ["net"] }
 tokio-util = { version = "0.7.10", features = ["codec", "io-util"] }
 toml = { version = "0.7.8" }
 toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.6", features = ["serde"] }
-tracing = { version = "0.1.37", features = ["log"] }
+tracing = { version = "0.1.40", features = ["log"] }
 trust-dns-proto = { version = "0.22.0" }
-unicode-bidi = { version = "0.3.13" }
+unicode-bidi = { version = "0.3.15" }
 unicode-normalization = { version = "0.1.22" }
 usdt = { version = "0.3.5" }
 usdt-impl = { version = "0.5.0", default-features = false, features = ["asm", "des"] }
@@ -227,45 +225,45 @@ zeroize = { version = "1.7.0", features = ["std", "zeroize_derive"] }
 zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] }
 
 [target.x86_64-unknown-linux-gnu.dependencies]
-bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
+bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.2", default-features = false, features = ["std"] }
 dof = { version = "0.3.0", default-features = false, features = ["des"] }
 mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.x86_64-unknown-linux-gnu.build-dependencies]
-bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
+bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.2", default-features = false, features = ["std"] }
 dof = { version = "0.3.0", default-features = false, features = ["des"] }
 mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.x86_64-apple-darwin.dependencies]
-bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
+bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.2", default-features = false, features = ["std"] }
 mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.x86_64-apple-darwin.build-dependencies]
-bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
+bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.2", default-features = false, features = ["std"] }
 mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.aarch64-apple-darwin.dependencies]
-bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
+bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.2", default-features = false, features = ["std"] }
 mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.aarch64-apple-darwin.build-dependencies]
-bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
+bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.2", default-features = false, features = ["std"] }
 mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
 
 [target.x86_64-unknown-illumos.dependencies]
-bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
+bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.2", default-features = false, features = ["std"] }
 dof = { version = "0.3.0", default-features = false, features = ["des"] }
 mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
@@ -274,7 +272,7 @@ toml_datetime = { version = "0.6.5", default-features = false, features = ["serd
 toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] }
 
 [target.x86_64-unknown-illumos.build-dependencies]
-bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] }
+bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.2", default-features = false, features = ["std"] }
 dof = { version = "0.3.0", default-features = false, features = ["des"] }
 mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }

From 1eaad083ca8a4363f783f550266084d84ba8c866 Mon Sep 17 00:00:00 2001
From: David Pacheco <dap@oxidecomputer.com>
Date: Sat, 9 Mar 2024 19:20:01 -0800
Subject: [PATCH 095/157] test environment could be more realistic (#5239)

---
 Cargo.lock                                    |   2 +
 dev-tools/omdb/tests/env.out                  |   3 +
 dev-tools/omdb/tests/successes.out            |  13 +-
 dev-tools/omicron-dev/src/bin/omicron-dev.rs  |  10 +-
 .../tests/output/collector_basic.txt          |   4 +-
 .../output/collector_sled_agent_errors.txt    |   2 +-
 .../app/background/inventory_collection.rs    |   4 +-
 nexus/src/lib.rs                              |  22 +-
 nexus/test-interface/src/lib.rs               |   1 +
 nexus/test-utils/Cargo.toml                   |   2 +
 nexus/test-utils/src/lib.rs                   | 376 +++++++++++++++---
 .../tests/integration_tests/initialization.rs |  29 +-
 nexus/tests/integration_tests/sleds.rs        |  14 +-
 sled-agent/src/sim/sled_agent.rs              |   2 +-
 14 files changed, 383 insertions(+), 101 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 0cfc7b4500..db48902f75 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4648,6 +4648,7 @@ dependencies = [
  "headers",
  "http 0.2.12",
  "hyper 0.14.28",
+ "illumos-utils",
  "internal-dns",
  "nexus-config",
  "nexus-db-queries",
@@ -4664,6 +4665,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_urlencoded",
+ "sled-agent-client",
  "slog",
  "tokio",
  "tokio-util",
diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out
index 3e6e89d508..ef8cf1631e 100644
--- a/dev-tools/omdb/tests/env.out
+++ b/dev-tools/omdb/tests/env.out
@@ -3,6 +3,7 @@ termination: Exited(0)
 ---------------------------------------------
 stdout:
 SERIAL       IP          ROLE     ID                                   
+sim-039be560 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED 
 sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED 
 ---------------------------------------------
 stderr:
@@ -268,6 +269,7 @@ termination: Exited(0)
 ---------------------------------------------
 stdout:
 SERIAL       IP          ROLE     ID                                   
+sim-039be560 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED 
 sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED 
 ---------------------------------------------
 stderr:
@@ -281,6 +283,7 @@ termination: Exited(0)
 ---------------------------------------------
 stdout:
 SERIAL       IP          ROLE     ID                                   
+sim-039be560 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED 
 sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED 
 ---------------------------------------------
 stderr:
diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out
index fe590acf55..2da6e4dceb 100644
--- a/dev-tools/omdb/tests/successes.out
+++ b/dev-tools/omdb/tests/successes.out
@@ -77,12 +77,10 @@ termination: Exited(0)
 stdout:
 SERVICE        INSTANCE_ID                          ADDR                     SLED_SERIAL  
 CruciblePantry REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              sim-b6d65341 
-Dendrite       REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              sim-b6d65341 
-Dendrite       REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              sim-b6d65341 
 ExternalDns    REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              sim-b6d65341 
 InternalDns    REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              sim-b6d65341 
 Nexus          REDACTED_UUID_REDACTED_UUID_REDACTED [::ffff:127.0.0.1]:REDACTED_PORT sim-b6d65341 
-Mgd            REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              sim-b6d65341 
+Mgd            REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              sim-039be560 
 Mgd            REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              sim-b6d65341 
 ---------------------------------------------
 stderr:
@@ -93,17 +91,19 @@ EXECUTING COMMAND: omdb ["db", "services", "list-by-sled"]
 termination: Exited(0)
 ---------------------------------------------
 stdout:
+sled: sim-039be560 (id REDACTED_UUID_REDACTED_UUID_REDACTED)
+
+  SERVICE INSTANCE_ID                          ADDR        
+  Mgd     REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT 
+
 sled: sim-b6d65341 (id REDACTED_UUID_REDACTED_UUID_REDACTED)
 
   SERVICE        INSTANCE_ID                          ADDR                     
   CruciblePantry REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              
-  Dendrite       REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              
-  Dendrite       REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              
   ExternalDns    REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              
   InternalDns    REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              
   Nexus          REDACTED_UUID_REDACTED_UUID_REDACTED [::ffff:127.0.0.1]:REDACTED_PORT 
   Mgd            REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              
-  Mgd            REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT              
 
 ---------------------------------------------
 stderr:
@@ -115,6 +115,7 @@ termination: Exited(0)
 ---------------------------------------------
 stdout:
 SERIAL       IP          ROLE     ID                                   
+sim-039be560 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED 
 sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED 
 ---------------------------------------------
 stderr:
diff --git a/dev-tools/omicron-dev/src/bin/omicron-dev.rs b/dev-tools/omicron-dev/src/bin/omicron-dev.rs
index 5e0c6486d6..705049bdb1 100644
--- a/dev-tools/omicron-dev/src/bin/omicron-dev.rs
+++ b/dev-tools/omicron-dev/src/bin/omicron-dev.rs
@@ -543,10 +543,12 @@ async fn cmd_run_all(args: &RunAllArgs) -> Result<(), anyhow::Error> {
         cptestctx.silo_name,
         cptestctx.external_dns_zone_name,
     );
-    println!(
-        "omicron-dev: management gateway:    http://{}",
-        cptestctx.gateway.client.bind_address,
-    );
+    for (location, gateway) in &cptestctx.gateway {
+        println!(
+            "omicron-dev: management gateway:    http://{} ({})",
+            gateway.client.bind_address, location,
+        );
+    }
     println!("omicron-dev: silo name:             {}", cptestctx.silo_name,);
     println!(
         "omicron-dev: privileged user name:  {}",
diff --git a/nexus/inventory/tests/output/collector_basic.txt b/nexus/inventory/tests/output/collector_basic.txt
index 4a05f09e1c..0fc1c552ab 100644
--- a/nexus/inventory/tests/output/collector_basic.txt
+++ b/nexus/inventory/tests/output/collector_basic.txt
@@ -71,12 +71,12 @@ rot pages found:
     CfpaScratch baseboard part "i86pc" serial "SimGimlet01": data_base64 "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
 
 sled agents found:
-  sled 03265caf-da7d-46c7-b1c2-39fa90ce5c65 (Gimlet)
+  sled 03265caf-da7d-46c7-b1c2-39fa90ce5c65 (Scrimlet)
     baseboard Some(BaseboardId { part_number: "sim-gimlet", serial_number: "sim-03265caf-da7d-46c7-b1c2-39fa90ce5c65" })
     zone generation: Generation(3)
     zones found:
         zone 8b88a56f-3eb6-4d80-ba42-75d867bc427d type oximeter
-  sled 9cb9b78f-5614-440c-b66d-e8e81fab69b0 (Gimlet)
+  sled 9cb9b78f-5614-440c-b66d-e8e81fab69b0 (Scrimlet)
     baseboard Some(BaseboardId { part_number: "sim-gimlet", serial_number: "sim-9cb9b78f-5614-440c-b66d-e8e81fab69b0" })
     zone generation: Generation(3)
     zones found:
diff --git a/nexus/inventory/tests/output/collector_sled_agent_errors.txt b/nexus/inventory/tests/output/collector_sled_agent_errors.txt
index aaa31fd1bb..7b9bbce84e 100644
--- a/nexus/inventory/tests/output/collector_sled_agent_errors.txt
+++ b/nexus/inventory/tests/output/collector_sled_agent_errors.txt
@@ -70,7 +70,7 @@ rot pages found:
     CfpaScratch baseboard part "i86pc" serial "SimGimlet01": data_base64 "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
 
 sled agents found:
-  sled 9cb9b78f-5614-440c-b66d-e8e81fab69b0 (Gimlet)
+  sled 9cb9b78f-5614-440c-b66d-e8e81fab69b0 (Scrimlet)
     baseboard Some(BaseboardId { part_number: "sim-gimlet", serial_number: "sim-9cb9b78f-5614-440c-b66d-e8e81fab69b0" })
     zone generation: Generation(3)
     zones found:
diff --git a/nexus/src/app/background/inventory_collection.rs b/nexus/src/app/background/inventory_collection.rs
index 27f08ec738..0666c136fc 100644
--- a/nexus/src/app/background/inventory_collection.rs
+++ b/nexus/src/app/background/inventory_collection.rs
@@ -314,9 +314,9 @@ mod test {
             page_size: NonZeroU32::new(3).unwrap(),
         };
 
-        // There will be one sled agent set up as part of the test context.
+        // There will be two sled agents set up as part of the test context.
         let found_urls = db_enum.list_sled_agents().await.unwrap();
-        assert_eq!(found_urls.len(), 1);
+        assert_eq!(found_urls.len(), 2);
 
         // Insert some sleds.
         let rack_id = Uuid::new_v4();
diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs
index 771a78f0b1..c0fba31afb 100644
--- a/nexus/src/lib.rs
+++ b/nexus/src/lib.rs
@@ -27,6 +27,7 @@ use dropshot::ConfigDropshot;
 use external_api::http_entrypoints::external_api;
 use internal_api::http_entrypoints::internal_api;
 use nexus_config::NexusConfig;
+use nexus_types::external_api::views::SledProvisionPolicy;
 use nexus_types::internal_api::params::ServiceKind;
 use omicron_common::address::IpRange;
 use omicron_common::api::internal::shared::{
@@ -237,6 +238,7 @@ impl nexus_test_interface::NexusServer for Server {
         external_dns_zone_name: &str,
         recovery_silo: nexus_types::internal_api::params::RecoverySiloConfig,
         certs: Vec<nexus_types::internal_api::params::Certificate>,
+        disable_sled_id: Uuid,
     ) -> Self {
         // Perform the "handoff from RSS".
         //
@@ -302,7 +304,25 @@ impl nexus_test_interface::NexusServer for Server {
             .expect("Could not initialize rack");
 
         // Start the Nexus external API.
-        Server::start(internal_server).await.unwrap()
+        let rv = Server::start(internal_server).await.unwrap();
+
+        // Historically, tests have assumed that there's only one provisionable
+        // sled, and that's convenient for a lot of purposes.  Mark our second
+        // sled non-provisionable.
+        let nexus = &rv.apictx().nexus;
+        nexus
+            .sled_set_provision_policy(
+                &opctx,
+                &nexus_db_queries::db::lookup::LookupPath::new(
+                    &opctx,
+                    nexus.datastore(),
+                )
+                .sled_id(disable_sled_id),
+                SledProvisionPolicy::NonProvisionable,
+            )
+            .await
+            .unwrap();
+        rv
     }
 
     async fn get_http_server_external_address(&self) -> SocketAddr {
diff --git a/nexus/test-interface/src/lib.rs b/nexus/test-interface/src/lib.rs
index 0f53ac6445..10bc9e63f0 100644
--- a/nexus/test-interface/src/lib.rs
+++ b/nexus/test-interface/src/lib.rs
@@ -56,6 +56,7 @@ pub trait NexusServer: Send + Sync + 'static {
         external_dns_zone_name: &str,
         recovery_silo: nexus_types::internal_api::params::RecoverySiloConfig,
         tls_certificates: Vec<nexus_types::internal_api::params::Certificate>,
+        disable_sled_id: Uuid,
     ) -> Self;
 
     async fn get_http_server_external_address(&self) -> SocketAddr;
diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml
index e612547fa8..861527108b 100644
--- a/nexus/test-utils/Cargo.toml
+++ b/nexus/test-utils/Cargo.toml
@@ -20,6 +20,7 @@ gateway-test-utils.workspace = true
 headers.workspace = true
 http.workspace = true
 hyper.workspace = true
+illumos-utils.workspace = true
 internal-dns.workspace = true
 nexus-config.workspace = true
 nexus-db-queries.workspace = true
@@ -35,6 +36,7 @@ oximeter-producer.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 serde_urlencoded.workspace = true
+sled-agent-client.workspace = true
 slog.workspace = true
 tokio.workspace = true
 tokio-util.workspace = true
diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs
index 4ef77b3352..9681d9ff97 100644
--- a/nexus/test-utils/src/lib.rs
+++ b/nexus/test-utils/src/lib.rs
@@ -33,12 +33,20 @@ use nexus_types::internal_api::params::RecoverySiloConfig;
 use nexus_types::internal_api::params::ServiceKind;
 use nexus_types::internal_api::params::ServiceNic;
 use nexus_types::internal_api::params::ServicePutRequest;
+use nexus_types::inventory::OmicronZoneConfig;
+use nexus_types::inventory::OmicronZoneDataset;
+use nexus_types::inventory::OmicronZoneType;
+use nexus_types::inventory::OmicronZonesConfig;
 use omicron_common::address::DNS_OPTE_IPV4_SUBNET;
 use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET;
+use omicron_common::api::external::Generation;
 use omicron_common::api::external::MacAddr;
+use omicron_common::api::external::Vni;
 use omicron_common::api::external::{IdentityMetadata, Name};
 use omicron_common::api::internal::nexus::ProducerEndpoint;
 use omicron_common::api::internal::nexus::ProducerKind;
+use omicron_common::api::internal::shared::NetworkInterface;
+use omicron_common::api::internal::shared::NetworkInterfaceKind;
 use omicron_common::api::internal::shared::SwitchLocation;
 use omicron_sled_agent::sim;
 use omicron_test_utils::dev;
@@ -65,6 +73,7 @@ pub mod http_testing;
 pub mod resource_helpers;
 
 pub const SLED_AGENT_UUID: &str = "b6d65341-167c-41df-9b5c-41cded99c229";
+pub const SLED_AGENT2_UUID: &str = "039be560-54cc-49e3-88df-1a29dadbf913";
 pub const RACK_UUID: &str = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc";
 pub const SWITCH_UUID: &str = "dae4e1f1-410e-4314-bff1-fec0504be07e";
 pub const OXIMETER_UUID: &str = "39e6175b-4df2-4730-b11d-cbc1e60a2e78";
@@ -88,9 +97,11 @@ pub struct ControlPlaneTestContext<N> {
     pub logctx: LogContext,
     pub sled_agent_storage: camino_tempfile::Utf8TempDir,
     pub sled_agent: sim::Server,
+    pub sled_agent2_storage: camino_tempfile::Utf8TempDir,
+    pub sled_agent2: sim::Server,
     pub oximeter: Oximeter,
     pub producer: ProducerServer,
-    pub gateway: GatewayTestContext,
+    pub gateway: HashMap<SwitchLocation, GatewayTestContext>,
     pub dendrite: HashMap<SwitchLocation, dev::dendrite::DendriteInstance>,
     pub mgd: HashMap<SwitchLocation, dev::maghemite::MgdInstance>,
     pub external_dns_zone_name: String,
@@ -110,9 +121,12 @@ impl<N: NexusServer> ControlPlaneTestContext<N> {
         self.database.cleanup().await.unwrap();
         self.clickhouse.cleanup().await.unwrap();
         self.sled_agent.http_server.close().await.unwrap();
+        self.sled_agent2.http_server.close().await.unwrap();
         self.oximeter.close().await.unwrap();
         self.producer.close().await.unwrap();
-        self.gateway.teardown().await;
+        for (_, gateway) in self.gateway {
+            gateway.teardown().await;
+        }
         for (_, mut dendrite) in self.dendrite {
             dendrite.cleanup().await.unwrap();
         }
@@ -179,18 +193,18 @@ impl RackInitRequestBuilder {
     // Keeps track of:
     // - The "ServicePutRequest" (for handoff to Nexus)
     // - The internal DNS configuration for this service
-    fn add_service(
+    fn add_service_with_id(
         &mut self,
+        zone_id: Uuid,
         address: SocketAddrV6,
         kind: ServiceKind,
         service_name: internal_dns::ServiceName,
         sled_id: Uuid,
     ) {
-        let zone_id = Uuid::new_v4();
         self.services.push(ServicePutRequest {
             address,
             kind,
-            service_id: Uuid::new_v4(),
+            service_id: zone_id,
             sled_id,
             zone_id: Some(zone_id),
         });
@@ -203,6 +217,22 @@ impl RackInitRequestBuilder {
             .expect("Failed to set up DNS for {kind}");
     }
 
+    fn add_service_without_dns(
+        &mut self,
+        zone_id: Uuid,
+        address: SocketAddrV6,
+        kind: ServiceKind,
+        sled_id: Uuid,
+    ) {
+        self.services.push(ServicePutRequest {
+            address,
+            kind,
+            service_id: zone_id,
+            sled_id,
+            zone_id: Some(zone_id),
+        });
+    }
+
     // Keeps track of:
     // - The "DatasetPutRequest" (for handoff to Nexus)
     // - The internal DNS configuration for this service
@@ -245,9 +275,11 @@ pub struct ControlPlaneTestContextBuilder<'a, N: NexusServer> {
     pub clickhouse: Option<dev::clickhouse::ClickHouseInstance>,
     pub sled_agent_storage: Option<camino_tempfile::Utf8TempDir>,
     pub sled_agent: Option<sim::Server>,
+    pub sled_agent2_storage: Option<camino_tempfile::Utf8TempDir>,
+    pub sled_agent2: Option<sim::Server>,
     pub oximeter: Option<Oximeter>,
     pub producer: Option<ProducerServer>,
-    pub gateway: Option<GatewayTestContext>,
+    pub gateway: HashMap<SwitchLocation, GatewayTestContext>,
     pub dendrite: HashMap<SwitchLocation, dev::dendrite::DendriteInstance>,
     pub mgd: HashMap<SwitchLocation, dev::maghemite::MgdInstance>,
 
@@ -260,6 +292,8 @@ pub struct ControlPlaneTestContextBuilder<'a, N: NexusServer> {
     pub external_dns: Option<dns_server::TransientServer>,
     pub internal_dns: Option<dns_server::TransientServer>,
     dns_config: Option<DnsConfigParams>,
+    omicron_zones: Vec<OmicronZoneConfig>,
+    omicron_zones2: Vec<OmicronZoneConfig>,
 
     pub silo_name: Option<Name>,
     pub user_name: Option<UserId>,
@@ -289,9 +323,11 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             clickhouse: None,
             sled_agent_storage: None,
             sled_agent: None,
+            sled_agent2_storage: None,
+            sled_agent2: None,
             oximeter: None,
             producer: None,
-            gateway: None,
+            gateway: HashMap::new(),
             dendrite: HashMap::new(),
             mgd: HashMap::new(),
             nexus_internal: None,
@@ -300,6 +336,8 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             external_dns: None,
             internal_dns: None,
             dns_config: None,
+            omicron_zones: Vec::new(),
+            omicron_zones2: Vec::new(),
             silo_name: None,
             user_name: None,
         }
@@ -380,6 +418,18 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             DatasetKind::Cockroach,
             internal_dns::ServiceName::Cockroach,
         );
+        let pool_name = illumos_utils::zpool::ZpoolName::new_external(zpool_id)
+            .to_string()
+            .parse()
+            .unwrap();
+        self.omicron_zones.push(OmicronZoneConfig {
+            id: dataset_id,
+            underlay_address: *address.ip(),
+            zone_type: OmicronZoneType::CockroachDb {
+                address: address.to_string(),
+                dataset: OmicronZoneDataset { pool_name },
+            },
+        });
         self.database = Some(database);
     }
 
@@ -416,37 +466,40 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             .as_mut()
             .expect("Tests expect to set a port of Clickhouse")
             .set_port(port);
+
+        let pool_name = illumos_utils::zpool::ZpoolName::new_external(zpool_id)
+            .to_string()
+            .parse()
+            .unwrap();
+        self.omicron_zones.push(OmicronZoneConfig {
+            id: dataset_id,
+            underlay_address: *address.ip(),
+            zone_type: OmicronZoneType::Clickhouse {
+                address: address.to_string(),
+                dataset: OmicronZoneDataset { pool_name },
+            },
+        });
     }
 
-    pub async fn start_gateway(&mut self) {
-        // For now, this MGS is not configured to match up in any way with
-        // either the simulated sled agent or the Dendrite instances.  It's
-        // useful for testing stuff unrelated to that.  But at some point we
-        // will probably want the reported data to match up better.
+    pub async fn start_gateway(
+        &mut self,
+        switch_location: SwitchLocation,
+        port: Option<u16>,
+    ) {
         debug!(&self.logctx.log, "Starting Management Gateway");
-        let gateway = gateway_test_utils::setup::test_setup(
+        let (mgs_config, sp_sim_config) =
+            gateway_test_utils::setup::load_test_config();
+        let mgs_addr =
+            port.map(|port| SocketAddrV6::new(Ipv6Addr::LOCALHOST, port, 0, 0));
+        let gateway = gateway_test_utils::setup::test_setup_with_config(
             self.test_name,
             gateway_messages::SpPort::One,
+            mgs_config,
+            &sp_sim_config,
+            mgs_addr,
         )
         .await;
-        let fake_mgs_zone_id = Uuid::new_v4();
-        let SocketAddr::V6(v6addr) = gateway.client.bind_address else {
-            panic!("MGS unexpectedly listening on IPv4?");
-        };
-        let zone = self
-            .rack_init_builder
-            .internal_dns_config
-            .host_zone(fake_mgs_zone_id, *v6addr.ip())
-            .expect("Failed to add DNS for MGS zone");
-        self.rack_init_builder
-            .internal_dns_config
-            .service_backend_zone(
-                internal_dns::ServiceName::ManagementGatewayService,
-                &zone,
-                v6addr.port(),
-            )
-            .expect("Failed to add DNS for MGS service");
-        self.gateway = Some(gateway);
+        self.gateway.insert(switch_location, gateway);
     }
 
     pub async fn start_dendrite(&mut self, switch_location: SwitchLocation) {
@@ -466,11 +519,16 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
         let config = DpdConfig { address: std::net::SocketAddr::V6(address) };
         self.config.pkg.dendrite.insert(switch_location, config);
 
-        let sled_id = Uuid::parse_str(SLED_AGENT_UUID).unwrap();
-        self.rack_init_builder.add_service(
+        let sled_id = Uuid::parse_str(match switch_location {
+            SwitchLocation::Switch0 => SLED_AGENT_UUID,
+            SwitchLocation::Switch1 => SLED_AGENT2_UUID,
+        })
+        .unwrap();
+
+        self.rack_init_builder.add_service_without_dns(
+            sled_id,
             address,
             ServiceKind::Dendrite,
-            internal_dns::ServiceName::Dendrite,
             sled_id,
         );
     }
@@ -490,15 +548,46 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
         let config = MgdConfig { address: std::net::SocketAddr::V6(address) };
         self.config.pkg.mgd.insert(switch_location, config);
 
-        let sled_id = Uuid::parse_str(SLED_AGENT_UUID).unwrap();
-        self.rack_init_builder.add_service(
+        let sled_id = Uuid::parse_str(match switch_location {
+            SwitchLocation::Switch0 => SLED_AGENT_UUID,
+            SwitchLocation::Switch1 => SLED_AGENT2_UUID,
+        })
+        .unwrap();
+
+        self.rack_init_builder.add_service_without_dns(
+            sled_id,
             address,
             ServiceKind::Mgd,
-            internal_dns::ServiceName::Mgd,
             sled_id,
         );
     }
 
+    pub async fn record_switch_dns(&mut self) {
+        let log = &self.logctx.log;
+        debug!(log, "Recording DNS for the switch zones");
+        for (sled_id, switch_location) in &[
+            (SLED_AGENT_UUID, SwitchLocation::Switch0),
+            (SLED_AGENT2_UUID, SwitchLocation::Switch1),
+        ] {
+            let id = sled_id.parse().unwrap();
+            self.rack_init_builder
+                .internal_dns_config
+                .host_zone_switch(
+                    id,
+                    Ipv6Addr::LOCALHOST,
+                    self.dendrite.get(switch_location).unwrap().port,
+                    self.gateway
+                        .get(switch_location)
+                        .unwrap()
+                        .client
+                        .bind_address
+                        .port(),
+                    self.mgd.get(switch_location).unwrap().port,
+                )
+                .unwrap();
+        }
+    }
+
     pub async fn start_oximeter(&mut self) {
         let log = &self.logctx.log;
         debug!(log, "Starting Oximeter");
@@ -585,16 +674,14 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             .mac_addrs
             .next()
             .expect("ran out of MAC addresses");
-        self.rack_init_builder.add_service(
+        let external_address =
+            self.config.deployment.dropshot_external.dropshot.bind_address.ip();
+        let nexus_id = self.config.deployment.id;
+        self.rack_init_builder.add_service_with_id(
+            nexus_id,
             address,
             ServiceKind::Nexus {
-                external_address: self
-                    .config
-                    .deployment
-                    .dropshot_external
-                    .dropshot
-                    .bind_address
-                    .ip(),
+                external_address,
                 nic: ServiceNic {
                     id: Uuid::new_v4(),
                     name: "nexus".parse().unwrap(),
@@ -610,6 +697,32 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             sled_id,
         );
 
+        self.omicron_zones.push(OmicronZoneConfig {
+            id: nexus_id,
+            underlay_address: *address.ip(),
+            zone_type: OmicronZoneType::Nexus {
+                external_dns_servers: self
+                    .config
+                    .deployment
+                    .external_dns_servers
+                    .clone(),
+                external_ip: external_address,
+                external_tls: self.config.deployment.dropshot_external.tls,
+                internal_address: address.to_string(),
+                nic: NetworkInterface {
+                    id: Uuid::new_v4(),
+                    ip: external_address,
+                    kind: NetworkInterfaceKind::Service { id: nexus_id },
+                    mac,
+                    name: format!("nexus-{}", nexus_id).parse().unwrap(),
+                    primary: true,
+                    slot: 0,
+                    subnet: (*NEXUS_OPTE_IPV4_SUBNET).into(),
+                    vni: Vni::SERVICES_VNI,
+                },
+            },
+        });
+
         self.nexus_internal = Some(nexus_internal);
         self.nexus_internal_addr = Some(nexus_internal_addr);
     }
@@ -701,6 +814,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             &external_dns_zone_name,
             recovery_silo,
             tls_certificates,
+            SLED_AGENT2_UUID.parse().unwrap(),
         )
         .await;
 
@@ -729,12 +843,22 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
         self.server = Some(server);
     }
 
-    pub async fn start_sled(&mut self, sim_mode: sim::SimMode) {
+    pub async fn start_sled(
+        &mut self,
+        switch_location: SwitchLocation,
+        sim_mode: sim::SimMode,
+    ) {
         let nexus_address =
             self.nexus_internal_addr.expect("Must launch Nexus first");
 
         // Set up a single sled agent.
-        let sa_id = Uuid::parse_str(SLED_AGENT_UUID).unwrap();
+        let sa_id: Uuid = if switch_location == SwitchLocation::Switch0 {
+            SLED_AGENT_UUID
+        } else {
+            SLED_AGENT2_UUID
+        }
+        .parse()
+        .unwrap();
         let tempdir = camino_tempfile::tempdir().unwrap();
         let sled_agent = start_sled_agent(
             self.logctx.log.new(o!(
@@ -749,8 +873,40 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
         .await
         .expect("Failed to start sled agent");
 
-        self.sled_agent = Some(sled_agent);
-        self.sled_agent_storage = Some(tempdir);
+        if switch_location == SwitchLocation::Switch0 {
+            self.sled_agent = Some(sled_agent);
+            self.sled_agent_storage = Some(tempdir);
+        } else {
+            self.sled_agent2 = Some(sled_agent);
+            self.sled_agent2_storage = Some(tempdir);
+        }
+    }
+
+    pub async fn configure_sled_agent(
+        &mut self,
+        switch_location: SwitchLocation,
+    ) {
+        let (field, zones) = if switch_location == SwitchLocation::Switch0 {
+            (&self.sled_agent, &self.omicron_zones)
+        } else {
+            (&self.sled_agent2, &self.omicron_zones2)
+        };
+
+        // Tell our Sled Agent to report the zones that we configured.
+        let Some(sled_agent) = field else {
+            panic!("expected sled agent has not been created");
+        };
+        let client = sled_agent_client::Client::new(
+            &format!("http://{}", sled_agent.http_server.local_addr()),
+            self.logctx.log.clone(),
+        );
+        client
+            .omicron_zones_put(&OmicronZonesConfig {
+                zones: zones.clone(),
+                generation: Generation::new().next(),
+            })
+            .await
+            .expect("Failed to configure sled agent with our zones");
     }
 
     // Set up the Crucible Pantry on an existing Sled Agent.
@@ -768,12 +924,21 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
         };
 
         let sled_id = Uuid::parse_str(SLED_AGENT_UUID).unwrap();
-        self.rack_init_builder.add_service(
+        let zone_id = Uuid::new_v4();
+        self.rack_init_builder.add_service_with_id(
+            zone_id,
             address,
             ServiceKind::CruciblePantry,
             internal_dns::ServiceName::CruciblePantry,
             sled_id,
         );
+        self.omicron_zones.push(OmicronZoneConfig {
+            id: zone_id,
+            underlay_address: *address.ip(),
+            zone_type: OmicronZoneType::CruciblePantry {
+                address: address.to_string(),
+            },
+        });
     }
 
     // Set up an external DNS server.
@@ -796,7 +961,9 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             .mac_addrs
             .next()
             .expect("ran out of MAC addresses");
-        self.rack_init_builder.add_service(
+        let zone_id = Uuid::new_v4();
+        self.rack_init_builder.add_service_with_id(
+            zone_id,
             dropshot_address,
             ServiceKind::ExternalDns {
                 external_address: (*dns_address.ip()).into(),
@@ -814,6 +981,33 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             internal_dns::ServiceName::ExternalDns,
             sled_id,
         );
+
+        let zpool_id = Uuid::new_v4();
+        let pool_name = illumos_utils::zpool::ZpoolName::new_external(zpool_id)
+            .to_string()
+            .parse()
+            .unwrap();
+        self.omicron_zones.push(OmicronZoneConfig {
+            id: zone_id,
+            underlay_address: *dropshot_address.ip(),
+            zone_type: OmicronZoneType::ExternalDns {
+                dataset: OmicronZoneDataset { pool_name },
+                dns_address: dns_address.to_string(),
+                http_address: dropshot_address.to_string(),
+                nic: NetworkInterface {
+                    id: Uuid::new_v4(),
+                    ip: (*dns_address.ip()).into(),
+                    kind: NetworkInterfaceKind::Service { id: zone_id },
+                    mac,
+                    name: format!("external-dns-{}", zone_id).parse().unwrap(),
+                    primary: true,
+                    slot: 0,
+                    subnet: (*DNS_OPTE_IPV4_SUBNET).into(),
+                    vni: Vni::SERVICES_VNI,
+                },
+            },
+        });
+
         self.external_dns = Some(dns);
     }
 
@@ -826,13 +1020,32 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
         let SocketAddr::V6(address) = dns.dropshot_server.local_addr() else {
             panic!("Unsupported IPv4 DNS address");
         };
-        self.rack_init_builder.add_service(
+        let zone_id = Uuid::new_v4();
+        self.rack_init_builder.add_service_with_id(
+            zone_id,
             address,
             ServiceKind::InternalDns,
             internal_dns::ServiceName::InternalDns,
             sled_id,
         );
 
+        let zpool_id = Uuid::new_v4();
+        let pool_name = illumos_utils::zpool::ZpoolName::new_external(zpool_id)
+            .to_string()
+            .parse()
+            .unwrap();
+        self.omicron_zones.push(OmicronZoneConfig {
+            id: zone_id,
+            underlay_address: *address.ip(),
+            zone_type: OmicronZoneType::InternalDns {
+                dataset: OmicronZoneDataset { pool_name },
+                dns_address: dns.dns_server.local_address().to_string(),
+                http_address: address.to_string(),
+                gz_address: Ipv6Addr::LOCALHOST,
+                gz_address_index: 0,
+            },
+        });
+
         self.internal_dns = Some(dns);
     }
 
@@ -846,10 +1059,12 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             clickhouse: self.clickhouse.unwrap(),
             sled_agent_storage: self.sled_agent_storage.unwrap(),
             sled_agent: self.sled_agent.unwrap(),
+            sled_agent2_storage: self.sled_agent2_storage.unwrap(),
+            sled_agent2: self.sled_agent2.unwrap(),
             oximeter: self.oximeter.unwrap(),
             producer: self.producer.unwrap(),
             logctx: self.logctx,
-            gateway: self.gateway.unwrap(),
+            gateway: self.gateway,
             dendrite: self.dendrite,
             mgd: self.mgd,
             external_dns_zone_name: self.external_dns_zone_name.unwrap(),
@@ -873,13 +1088,16 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
         if let Some(sled_agent) = self.sled_agent {
             sled_agent.http_server.close().await.unwrap();
         }
+        if let Some(sled_agent2) = self.sled_agent2 {
+            sled_agent2.http_server.close().await.unwrap();
+        }
         if let Some(oximeter) = self.oximeter {
             oximeter.close().await.unwrap();
         }
         if let Some(producer) = self.producer {
             producer.close().await.unwrap();
         }
-        if let Some(gateway) = self.gateway {
+        for (_, gateway) in self.gateway {
             gateway.teardown().await;
         }
         for (_, mut dendrite) in self.dendrite {
@@ -990,8 +1208,20 @@ async fn setup_with_config_impl<N: NexusServer>(
                     Box::new(|builder| builder.start_clickhouse().boxed()),
                 ),
                 (
-                    "start_gateway",
-                    Box::new(|builder| builder.start_gateway().boxed()),
+                    "start_gateway_switch0",
+                    Box::new(|builder| {
+                        builder
+                            .start_gateway(SwitchLocation::Switch0, None)
+                            .boxed()
+                    }),
+                ),
+                (
+                    "start_gateway_switch1",
+                    Box::new(|builder| {
+                        builder
+                            .start_gateway(SwitchLocation::Switch1, None)
+                            .boxed()
+                    }),
                 ),
                 (
                     "start_dendrite_switch0",
@@ -1017,6 +1247,10 @@ async fn setup_with_config_impl<N: NexusServer>(
                         builder.start_mgd(SwitchLocation::Switch1).boxed()
                     }),
                 ),
+                (
+                    "record_switch_dns",
+                    Box::new(|builder| builder.record_switch_dns().boxed()),
+                ),
                 (
                     "start_internal_dns",
                     Box::new(|builder| builder.start_internal_dns().boxed()),
@@ -1030,9 +1264,19 @@ async fn setup_with_config_impl<N: NexusServer>(
                     Box::new(|builder| builder.start_nexus_internal().boxed()),
                 ),
                 (
-                    "start_sled",
+                    "start_sled1",
                     Box::new(move |builder| {
-                        builder.start_sled(sim_mode).boxed()
+                        builder
+                            .start_sled(SwitchLocation::Switch0, sim_mode)
+                            .boxed()
+                    }),
+                ),
+                (
+                    "start_sled2",
+                    Box::new(move |builder| {
+                        builder
+                            .start_sled(SwitchLocation::Switch1, sim_mode)
+                            .boxed()
                     }),
                 ),
                 (
@@ -1043,6 +1287,22 @@ async fn setup_with_config_impl<N: NexusServer>(
                     "populate_internal_dns",
                     Box::new(|builder| builder.populate_internal_dns().boxed()),
                 ),
+                (
+                    "configure_sled_agent1",
+                    Box::new(|builder| {
+                        builder
+                            .configure_sled_agent(SwitchLocation::Switch0)
+                            .boxed()
+                    }),
+                ),
+                (
+                    "configure_sled_agent2",
+                    Box::new(|builder| {
+                        builder
+                            .configure_sled_agent(SwitchLocation::Switch1)
+                            .boxed()
+                    }),
+                ),
                 (
                     "start_nexus_external",
                     Box::new(|builder| {
diff --git a/nexus/tests/integration_tests/initialization.rs b/nexus/tests/integration_tests/initialization.rs
index b77a121080..a76aef832e 100644
--- a/nexus/tests/integration_tests/initialization.rs
+++ b/nexus/tests/integration_tests/initialization.rs
@@ -2,17 +2,13 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use std::collections::HashMap;
-use std::net::{Ipv6Addr, SocketAddrV6};
-
-use gateway_messages::SpPort;
-use gateway_test_utils::setup as mgs_setup;
 use nexus_config::Database;
 use nexus_config::InternalDns;
 use nexus_test_interface::NexusServer;
 use nexus_test_utils::{load_test_config, ControlPlaneTestContextBuilder};
 use omicron_common::address::MGS_PORT;
 use omicron_common::api::internal::shared::SwitchLocation;
+use std::collections::HashMap;
 use tokio::time::sleep;
 use tokio::time::timeout;
 use tokio::time::Duration;
@@ -78,19 +74,6 @@ async fn test_nexus_boots_before_cockroach() {
 
 #[tokio::test]
 async fn test_nexus_boots_before_dendrite() {
-    // Start MGS + Sim SP. This is needed for the Dendrite client initialization
-    // inside of Nexus initialization
-    let (mgs_config, sp_sim_config) = mgs_setup::load_test_config();
-    let mgs_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, MGS_PORT, 0, 0);
-    let mgs = mgs_setup::test_setup_with_config(
-        "test_nexus_boots_before_dendrite",
-        SpPort::One,
-        mgs_config,
-        &sp_sim_config,
-        Some(mgs_addr),
-    )
-    .await;
-
     let mut config = load_test_config();
 
     let mut builder =
@@ -101,6 +84,14 @@ async fn test_nexus_boots_before_dendrite() {
 
     let log = builder.logctx.log.new(o!("component" => "test"));
 
+    // Start MGS + Sim SP. This is needed for the Dendrite client initialization
+    // inside of Nexus initialization.  We must use MGS_PORT here because Nexus
+    // hardcodes it.
+    info!(&log, "Starting MGS");
+    builder.start_gateway(SwitchLocation::Switch0, Some(MGS_PORT)).await;
+    builder.start_gateway(SwitchLocation::Switch1, None).await;
+    info!(&log, "Started MGS");
+
     let populate = true;
     builder.start_crdb(populate).await;
     builder.start_internal_dns().await;
@@ -150,6 +141,7 @@ async fn test_nexus_boots_before_dendrite() {
     info!(log, "Started mgd");
 
     info!(log, "Populating internal DNS records");
+    builder.record_switch_dns().await;
     builder.populate_internal_dns().await;
     info!(log, "Populated internal DNS records");
 
@@ -157,7 +149,6 @@ async fn test_nexus_boots_before_dendrite() {
     nexus_handle.await.expect("Test: Task starting Nexus has failed");
 
     builder.teardown().await;
-    mgs.teardown().await;
 }
 
 // Helper to ensure we perform the same setup for the positive and negative test
diff --git a/nexus/tests/integration_tests/sleds.rs b/nexus/tests/integration_tests/sleds.rs
index b551cf51b5..743a76be17 100644
--- a/nexus/tests/integration_tests/sleds.rs
+++ b/nexus/tests/integration_tests/sleds.rs
@@ -48,9 +48,9 @@ async fn sled_instance_list(
 async fn test_sleds_list(cptestctx: &ControlPlaneTestContext) {
     let client = &cptestctx.external_client;
 
-    // Verify that there is one sled to begin with.
+    // Verify that there are two sleds to begin with.
     let sleds_url = "/v1/system/hardware/sleds";
-    assert_eq!(sleds_list(&client, &sleds_url).await.len(), 1);
+    assert_eq!(sleds_list(&client, &sleds_url).await.len(), 2);
 
     // Now start a few more sled agents.
     let nsleds = 3;
@@ -76,7 +76,7 @@ async fn test_sleds_list(cptestctx: &ControlPlaneTestContext) {
 
     // List sleds again.
     let sleds_found = sleds_list(&client, &sleds_url).await;
-    assert_eq!(sleds_found.len(), nsleds + 1);
+    assert_eq!(sleds_found.len(), nsleds + 2);
 
     let sledids_found =
         sleds_found.iter().map(|sv| sv.identity.id).collect::<Vec<Uuid>>();
@@ -97,9 +97,9 @@ async fn test_physical_disk_create_list_delete(
     let external_client = &cptestctx.external_client;
     let internal_client = &cptestctx.internal_client;
 
-    // Verify that there is one sled to begin with.
+    // Verify that there are two sleds to begin with.
     let sleds_url = "/v1/system/hardware/sleds";
-    assert_eq!(sleds_list(&external_client, &sleds_url).await.len(), 1);
+    assert_eq!(sleds_list(&external_client, &sleds_url).await.len(), 2);
 
     // The test framework may set up some disks initially.
     let disks_url =
@@ -140,9 +140,9 @@ async fn test_physical_disk_create_list_delete(
 async fn test_sled_instance_list(cptestctx: &ControlPlaneTestContext) {
     let external_client = &cptestctx.external_client;
 
-    // Verify that there is one sled to begin with.
+    // Verify that there are two sleds to begin with.
     let sleds_url = "/v1/system/hardware/sleds";
-    assert_eq!(sleds_list(&external_client, &sleds_url).await.len(), 1);
+    assert_eq!(sleds_list(&external_client, &sleds_url).await.len(), 2);
 
     // Verify that there are no instances.
     let instances_url =
diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs
index 483b2d6aa8..0b90bef590 100644
--- a/sled-agent/src/sim/sled_agent.rs
+++ b/sled-agent/src/sim/sled_agent.rs
@@ -740,7 +740,7 @@ impl SledAgent {
         Ok(Inventory {
             sled_id: self.id,
             sled_agent_address,
-            sled_role: SledRole::Gimlet,
+            sled_role: SledRole::Scrimlet,
             baseboard: self.config.hardware.baseboard.clone(),
             usable_hardware_threads: self.config.hardware.hardware_threads,
             usable_physical_ram: ByteCount::try_from(

From 9dc85d649f88f2cac9e1873ce9c019a5d3664e69 Mon Sep 17 00:00:00 2001
From: Andy Fiddaman <andy@oxidecomputer.com>
Date: Sun, 10 Mar 2024 22:18:51 +0000
Subject: [PATCH 096/157] oxide/opte-interface-setup should default to disabled
 (#5230)

The `oxide/opte-interface-setup` service is present in all NTP zones,
but only the boundary zones require an OPTE interface. sled-agent
explicitly enables the service for just the boundary zones, but because
the service is enabled by default it ends up attempting to start in the
non-boundaries too, where it fails.

```
[ Mar  8 12:56:26 Executing start method ("/opt/oxide/zone-network-cli/bin/zone-networking opte-interface-set-up -i unknown -g unknown -p unknown"). ]
note: configured to log to "/dev/stderr"
error: invalid value 'unknown' for '--opte_interface <STRING>': ERROR: Missing OPTE interface

For more information, try '--help'.
[ Mar  8 12:56:26 Method "start" exited with status 2. ]
```
---
 smf/opte-interface-setup/manifest.xml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/smf/opte-interface-setup/manifest.xml b/smf/opte-interface-setup/manifest.xml
index 5b886c8a71..68641f56fc 100644
--- a/smf/opte-interface-setup/manifest.xml
+++ b/smf/opte-interface-setup/manifest.xml
@@ -4,7 +4,7 @@
 <service_bundle type='manifest' name='opte-interface-setup'>
 
 <service name='oxide/opte-interface-setup' type='service' version='1'>
-  <create_default_instance enabled='true' />
+  <create_default_instance enabled='false' />
 
   <!-- Run after the operating system's svc:/network/physical service is done. -->
   <dependency name='physical' grouping='require_all' restart_on='none'
@@ -20,7 +20,7 @@
   <exec_method type='method' name='start'
     exec='/opt/oxide/zone-network-cli/bin/zone-networking opte-interface-set-up -i %{config/interface} -g %{config/gateway} -p %{config/ip}'
     timeout_seconds='0' />
-  
+
   <property_group name='startd' type='framework'>
     <propval name='duration' type='astring' value='transient' />
   </property_group>
@@ -35,7 +35,7 @@
 
   <template>
     <common_name>
-      <loctext xml:lang='C'>Oxide OPTE interface set up</loctext>
+      <loctext xml:lang='C'>Oxide OPTE interface setup</loctext>
     </common_name>
     <description>
       <loctext xml:lang='C'>Sets up an OPTE interface for control plane zones that require it</loctext>

From 72cdbd71241d8909d100a81918e9aad3ea0126db Mon Sep 17 00:00:00 2001
From: Michael Zeller <mike@mikezeller.net>
Date: Mon, 11 Mar 2024 16:07:27 -0400
Subject: [PATCH 097/157] sled-agent needs to deal with SSD overprovisioning
 (#5158)

This introduces the ability for omicron's storage manager to resize and
format NVMe disks based upon a lookup table keyed off of model number to
preferred settings.

Fixes #4619
---
 Cargo.lock                              |  15 ++
 Cargo.toml                              |   1 +
 sled-hardware/Cargo.toml                |   1 +
 sled-hardware/src/disk.rs               |   6 +-
 sled-hardware/src/illumos/mod.rs        |   2 +-
 sled-hardware/src/illumos/partitions.rs | 199 +++++++++++++++++++++++-
 sled-hardware/src/non_illumos/mod.rs    |   8 +
 test-utils/src/dev/mod.rs               |  11 ++
 8 files changed, 239 insertions(+), 4 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index db48902f75..5900cf3dec 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3944,6 +3944,20 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "libnvme"
+version = "0.1.0"
+source = "git+https://github.com/oxidecomputer/libnvme?rev=6fffcc81d2c423ed2d2e6c5c2827485554c4ecbe#6fffcc81d2c423ed2d2e6c5c2827485554c4ecbe"
+dependencies = [
+ "libnvme-sys",
+ "thiserror",
+]
+
+[[package]]
+name = "libnvme-sys"
+version = "0.0.0"
+source = "git+https://github.com/oxidecomputer/libnvme?rev=6fffcc81d2c423ed2d2e6c5c2827485554c4ecbe#6fffcc81d2c423ed2d2e6c5c2827485554c4ecbe"
+
 [[package]]
 name = "libsw"
 version = "3.3.1"
@@ -8307,6 +8321,7 @@ dependencies = [
  "illumos-utils",
  "libc",
  "libefi-illumos",
+ "libnvme",
  "macaddr",
  "omicron-common",
  "omicron-test-utils",
diff --git a/Cargo.toml b/Cargo.toml
index 96f47be86f..f291d9642a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -256,6 +256,7 @@ ispf = { git = "https://github.com/oxidecomputer/ispf" }
 key-manager = { path = "key-manager" }
 kstat-rs = "0.2.3"
 libc = "0.2.153"
+libnvme = { git = "https://github.com/oxidecomputer/libnvme", rev = "6fffcc81d2c423ed2d2e6c5c2827485554c4ecbe" }
 linear-map = "1.2.0"
 macaddr = { version = "1.0.1", features = ["serde_std"] }
 maplit = "1.0.2"
diff --git a/sled-hardware/Cargo.toml b/sled-hardware/Cargo.toml
index 50e3cce072..28cec20041 100644
--- a/sled-hardware/Cargo.toml
+++ b/sled-hardware/Cargo.toml
@@ -29,6 +29,7 @@ omicron-workspace-hack.workspace = true
 [target.'cfg(target_os = "illumos")'.dependencies]
 illumos-devinfo = { git = "https://github.com/oxidecomputer/illumos-devinfo", branch = "main" }
 libefi-illumos = { git = "https://github.com/oxidecomputer/libefi-illumos", branch = "master" }
+libnvme.workspace = true
 
 [dev-dependencies]
 illumos-utils = { workspace = true, features = ["testing"] }
diff --git a/sled-hardware/src/disk.rs b/sled-hardware/src/disk.rs
index 44658658be..6a4c968f14 100644
--- a/sled-hardware/src/disk.rs
+++ b/sled-hardware/src/disk.rs
@@ -38,6 +38,8 @@ pub enum PooledDiskError {
     CannotFormatMissingDevPath { path: Utf8PathBuf },
     #[error("Formatting M.2 devices is not yet implemented")]
     CannotFormatM2NotImplemented,
+    #[error(transparent)]
+    NvmeFormatAndResize(#[from] NvmeFormattingError),
 }
 
 /// A partition (or 'slice') of a disk.
@@ -196,9 +198,11 @@ impl PooledDisk {
     ) -> Result<Self, PooledDiskError> {
         let paths = &unparsed_disk.paths;
         let variant = unparsed_disk.variant;
+        let identity = unparsed_disk.identity();
         // Ensure the GPT has the right format. This does not necessarily
         // mean that the partitions are populated with the data we need.
-        let partitions = ensure_partition_layout(&log, &paths, variant)?;
+        let partitions =
+            ensure_partition_layout(&log, &paths, variant, identity)?;
 
         // Find the path to the zpool which exists on this disk.
         //
diff --git a/sled-hardware/src/illumos/mod.rs b/sled-hardware/src/illumos/mod.rs
index ebc9a9c2b0..f2db424bda 100644
--- a/sled-hardware/src/illumos/mod.rs
+++ b/sled-hardware/src/illumos/mod.rs
@@ -26,7 +26,7 @@ mod gpt;
 mod partitions;
 mod sysconf;
 
-pub use partitions::ensure_partition_layout;
+pub use partitions::{ensure_partition_layout, NvmeFormattingError};
 
 #[derive(thiserror::Error, Debug)]
 enum Error {
diff --git a/sled-hardware/src/illumos/partitions.rs b/sled-hardware/src/illumos/partitions.rs
index 29b2466ad9..3b8e0af2ee 100644
--- a/sled-hardware/src/illumos/partitions.rs
+++ b/sled-hardware/src/illumos/partitions.rs
@@ -4,10 +4,14 @@
 
 //! illumos-specific mechanisms for parsing disk info.
 
+use std::collections::HashMap;
+use std::sync::OnceLock;
+
 use crate::illumos::gpt;
 use crate::{DiskPaths, DiskVariant, Partition, PooledDiskError};
 use camino::Utf8Path;
 use illumos_utils::zpool::ZpoolName;
+use omicron_common::disk::DiskIdentity;
 use slog::info;
 use slog::Logger;
 use uuid::Uuid;
@@ -17,6 +21,70 @@ use illumos_utils::zpool::MockZpool as Zpool;
 #[cfg(not(test))]
 use illumos_utils::zpool::Zpool;
 
+/// NVMe devices use a meta size of 0 as we don't support writing addditional
+/// metadata
+static NVME_LBA_META_SIZE: u32 = 0;
+/// NVMe devices default to using 4k logical block addressing unless overriden.
+static DEFAULT_NVME_LBA_DATA_SIZE: u64 = 4096;
+
+/// NVMe device settings for a particular NVMe model.
+struct NvmeDeviceSettings {
+    /// The desired disk size for dealing with overprovisioning.
+    size: u32,
+    /// An override for the default 4k LBA formatting.
+    lba_data_size_override: Option<u64>,
+}
+
+/// A mapping from model to desired settings.
+/// A device not found in this lookup table will not be modified by sled-agent.
+static PREFERRED_NVME_DEVICE_SETTINGS: OnceLock<
+    HashMap<&'static str, NvmeDeviceSettings>,
+> = OnceLock::new();
+
+fn preferred_nvme_device_settings(
+) -> &'static HashMap<&'static str, NvmeDeviceSettings> {
+    PREFERRED_NVME_DEVICE_SETTINGS.get_or_init(|| {
+        HashMap::from([
+            (
+                "WUS4C6432DSP3X3",
+                NvmeDeviceSettings { size: 3200, lba_data_size_override: None },
+            ),
+            (
+                "WUS5EA138ESP7E1",
+                NvmeDeviceSettings { size: 3200, lba_data_size_override: None },
+            ),
+            (
+                "WUS5EA138ESP7E3",
+                NvmeDeviceSettings { size: 3200, lba_data_size_override: None },
+            ),
+            (
+                "WUS5EA176ESP7E1",
+                NvmeDeviceSettings { size: 6400, lba_data_size_override: None },
+            ),
+            (
+                "WUS5EA176ESP7E3",
+                NvmeDeviceSettings { size: 6400, lba_data_size_override: None },
+            ),
+        ])
+    })
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum NvmeFormattingError {
+    #[error(transparent)]
+    NvmeInit(#[from] libnvme::NvmeInitError),
+    #[error(transparent)]
+    Nvme(#[from] libnvme::NvmeError),
+    #[error("Device is missing expected LBA format")]
+    LbaFormatMissing,
+    #[error("Device has {0} active namespaces but we expected 1")]
+    UnexpectedNamespaces(usize),
+    #[error(transparent)]
+    InfoError(#[from] libnvme::controller_info::NvmeInfoError),
+    #[error("Could not find NVMe controller for disk with serial {0}")]
+    NoController(String),
+}
+
 // The expected layout of an M.2 device within the Oxide rack.
 //
 // Partitions beyond this "expected partition" array are ignored.
@@ -79,8 +147,11 @@ pub fn ensure_partition_layout(
     log: &Logger,
     paths: &DiskPaths,
     variant: DiskVariant,
+    identity: &DiskIdentity,
 ) -> Result<Vec<Partition>, PooledDiskError> {
-    internal_ensure_partition_layout::<libefi_illumos::Gpt>(log, paths, variant)
+    internal_ensure_partition_layout::<libefi_illumos::Gpt>(
+        log, paths, variant, identity,
+    )
 }
 
 // Same as the [ensure_partition_layout], but with generic parameters
@@ -89,6 +160,7 @@ fn internal_ensure_partition_layout<GPT: gpt::LibEfiGpt>(
     log: &Logger,
     paths: &DiskPaths,
     variant: DiskVariant,
+    identity: &DiskIdentity,
 ) -> Result<Vec<Partition>, PooledDiskError> {
     // Open the "Whole Disk" as a raw device to be parsed by the
     // libefi-illumos library. This lets us peek at the GPT before
@@ -119,6 +191,11 @@ fn internal_ensure_partition_layout<GPT: gpt::LibEfiGpt>(
             };
             match variant {
                 DiskVariant::U2 => {
+                    // First we need to check that this disk is of the proper
+                    // size and correct logical block address formatting.
+                    ensure_size_and_formatting(log, identity)?;
+
+                    // If we were successful we can create a zpool on this disk.
                     info!(log, "Formatting zpool on disk {}", paths.devfs_path);
                     // If a zpool does not already exist, create one.
                     let zpool_name = ZpoolName::new_external(Uuid::new_v4());
@@ -154,13 +231,124 @@ fn internal_ensure_partition_layout<GPT: gpt::LibEfiGpt>(
     }
 }
 
+fn ensure_size_and_formatting(
+    log: &Logger,
+    identity: &DiskIdentity,
+) -> Result<(), NvmeFormattingError> {
+    use libnvme::namespace::NamespaceDiscoveryLevel;
+    use libnvme::Nvme;
+
+    let mut controller_found = false;
+
+    if let Some(nvme_settings) =
+        preferred_nvme_device_settings().get(identity.model.as_str())
+    {
+        let nvme = Nvme::new()?;
+        for controller in nvme.controller_discovery()? {
+            let controller = controller?.write_lock().map_err(|(_, e)| e)?;
+            let controller_info = controller.get_info()?;
+            // Make sure we are operating on the correct NVMe device.
+            if controller_info.serial() != identity.serial {
+                continue;
+            };
+            controller_found = true;
+            let nsdisc = controller
+                .namespace_discovery(NamespaceDiscoveryLevel::Active)?;
+            let namespaces =
+                nsdisc.into_iter().collect::<Result<Vec<_>, _>>()?;
+            if namespaces.len() != 1 {
+                return Err(NvmeFormattingError::UnexpectedNamespaces(
+                    namespaces.len(),
+                ));
+            }
+            // Safe because verified there is exactly one namespace.
+            let namespace = namespaces.into_iter().next().unwrap();
+
+            // NB: Only some vendors such as WDC support adjusting the size
+            // of the disk to deal with overprovisioning. This will need to be
+            // abstracted away if/when we ever start using another vendor with
+            // this capability.
+            let size = controller.wdc_resize_get()?;
+
+            // First we need to detach blkdev from the namespace.
+            namespace.blkdev_detach()?;
+
+            // Resize the device if needed to ensure we get the expected
+            // durability level in terms of drive writes per day.
+            if size != nvme_settings.size {
+                controller.wdc_resize_set(nvme_settings.size)?;
+                info!(
+                    log,
+                    "Resized {} from {size} to {}",
+                    identity.serial,
+                    nvme_settings.size
+                )
+            }
+
+            // Find the LBA format we want to use for the device.
+            let wanted_data_size = nvme_settings
+                .lba_data_size_override
+                .unwrap_or(DEFAULT_NVME_LBA_DATA_SIZE);
+            let desired_lba = controller_info
+                .lba_formats()
+                .collect::<Result<Vec<_>, _>>()?
+                .into_iter()
+                .find(|lba| {
+                    lba.meta_size() == NVME_LBA_META_SIZE
+                        && lba.data_size() == wanted_data_size
+                })
+                .ok_or_else(|| NvmeFormattingError::LbaFormatMissing)?;
+
+            // If the controller isn't formatted to our desired LBA we need to
+            // issue a format request.
+            let ns_info = namespace.get_info()?;
+            let current_lba = ns_info.current_format()?;
+            if current_lba.id() != desired_lba.id() {
+                controller
+                    .format_request()?
+                    .set_lbaf(desired_lba.id())?
+                    // TODO map this to libnvme::BROADCAST_NAMESPACE once added
+                    .set_nsid(u32::MAX)?
+                    // No secure erase
+                    .set_ses(0)?
+                    .execute()?;
+
+                info!(
+                    log,
+                    "Formatted disk with serial {} to an LBA with data size \
+                    {wanted_data_size}",
+                    identity.serial,
+                );
+            }
+
+            // Attach blkdev to the namespace again
+            namespace.blkdev_attach()?;
+        }
+    } else {
+        info!(
+            log,
+            "There are no preferred NVMe settings for disk model {}; nothing to\
+             do for disk with serial {}",
+            identity.model,
+            identity.serial
+        );
+        return Ok(());
+    }
+
+    if !controller_found {
+        return Err(NvmeFormattingError::NoController(identity.serial.clone()));
+    }
+
+    Ok(())
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
     use crate::DiskPaths;
     use camino::Utf8PathBuf;
     use illumos_utils::zpool::MockZpool;
-    use omicron_test_utils::dev::test_setup_log;
+    use omicron_test_utils::dev::{mock_disk_identity, test_setup_log};
     use std::path::Path;
 
     struct FakePartition {
@@ -196,6 +384,7 @@ mod test {
             &log,
             &DiskPaths { devfs_path, dev_path: None },
             DiskVariant::U2,
+            &mock_disk_identity(),
         );
         match result {
             Err(PooledDiskError::CannotFormatMissingDevPath { .. }) => {}
@@ -229,6 +418,7 @@ mod test {
                 dev_path: Some(Utf8PathBuf::from(DEV_PATH)),
             },
             DiskVariant::U2,
+            &mock_disk_identity(),
         )
         .expect("Should have succeeded partitioning disk");
 
@@ -253,6 +443,7 @@ mod test {
                 dev_path: Some(Utf8PathBuf::from(DEV_PATH))
             },
             DiskVariant::M2,
+            &mock_disk_identity(),
         )
         .is_err());
 
@@ -290,6 +481,7 @@ mod test {
                 dev_path: Some(Utf8PathBuf::from(DEV_PATH)),
             },
             DiskVariant::U2,
+            &mock_disk_identity(),
         )
         .expect("Should be able to parse disk");
 
@@ -332,6 +524,7 @@ mod test {
                 dev_path: Some(Utf8PathBuf::from(DEV_PATH)),
             },
             DiskVariant::M2,
+            &mock_disk_identity(),
         )
         .expect("Should be able to parse disk");
 
@@ -371,6 +564,7 @@ mod test {
                     dev_path: Some(Utf8PathBuf::from(DEV_PATH)),
                 },
                 DiskVariant::M2,
+                &mock_disk_identity(),
             )
             .expect_err("Should have failed parsing empty GPT"),
             PooledDiskError::BadPartitionLayout { .. }
@@ -396,6 +590,7 @@ mod test {
                     dev_path: Some(Utf8PathBuf::from(DEV_PATH)),
                 },
                 DiskVariant::U2,
+                &mock_disk_identity(),
             )
             .expect_err("Should have failed parsing empty GPT"),
             PooledDiskError::BadPartitionLayout { .. }
diff --git a/sled-hardware/src/non_illumos/mod.rs b/sled-hardware/src/non_illumos/mod.rs
index d8372dd8aa..8518aae495 100644
--- a/sled-hardware/src/non_illumos/mod.rs
+++ b/sled-hardware/src/non_illumos/mod.rs
@@ -6,10 +6,17 @@ use crate::disk::{
     DiskPaths, DiskVariant, Partition, PooledDiskError, UnparsedDisk,
 };
 use crate::{Baseboard, SledMode};
+use omicron_common::disk::DiskIdentity;
 use slog::Logger;
 use std::collections::HashSet;
 use tokio::sync::broadcast;
 
+#[derive(Debug, thiserror::Error)]
+pub enum NvmeFormattingError {
+    #[error("NVMe formatting is unsupported on this platform")]
+    UnsupportedPlatform,
+}
+
 /// An unimplemented, stub representation of the underlying hardware.
 ///
 /// This is intended for non-illumos systems to have roughly the same interface
@@ -59,6 +66,7 @@ pub fn ensure_partition_layout(
     _log: &Logger,
     _paths: &DiskPaths,
     _variant: DiskVariant,
+    _identity: &DiskIdentity,
 ) -> Result<Vec<Partition>, PooledDiskError> {
     unimplemented!("Accessing hardware unsupported on non-illumos");
 }
diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs
index e29da9c51e..705d6ff08c 100644
--- a/test-utils/src/dev/mod.rs
+++ b/test-utils/src/dev/mod.rs
@@ -20,6 +20,7 @@ pub use dropshot::test_util::LogContext;
 use dropshot::ConfigLogging;
 use dropshot::ConfigLoggingIfExists;
 use dropshot::ConfigLoggingLevel;
+use omicron_common::disk::DiskIdentity;
 use slog::Logger;
 use std::io::BufReader;
 
@@ -145,3 +146,13 @@ pub fn process_running(pid: u32) -> bool {
     // only checks whether the process is running.
     0 == (unsafe { libc::kill(pid as libc::pid_t, 0) })
 }
+
+/// Returns a DiskIdentity that can be passed to ensure_partition_layout when
+/// not operating on a real disk.
+pub fn mock_disk_identity() -> DiskIdentity {
+    DiskIdentity {
+        vendor: "MockVendor".to_string(),
+        serial: "MOCKSERIAL".to_string(),
+        model: "MOCKMODEL".to_string(),
+    }
+}

From 145f25cd9b8665d1bc4906a06369dbcac7dea268 Mon Sep 17 00:00:00 2001
From: Nils Nieuwejaar <nils@oxidecomputer.com>
Date: Mon, 11 Mar 2024 17:42:37 -0400
Subject: [PATCH 098/157] Add LLDP (#5196)

This adds the `lldp` daemon and CLI to the switch zone and updates
`sled-agent` to do the basic configuration and enablement of the
service.

On its own, this change will not cause the switch zone to advertise or
receive `lldp` packets, as no interfaces are configured. That can be
done using the CLI in the switch zone for testing on `dogfood` and/or
the colo. Adding runtime configuration by `nexus` will be follow-on
work.
---
 common/src/address.rs      |  1 +
 package-manifest.toml      | 12 +++++
 sled-agent/src/services.rs | 90 ++++++++++++++++++++++++++++++--------
 tools/update_lldp.sh       | 52 ++++++++++++++++++++++
 4 files changed, 137 insertions(+), 18 deletions(-)
 create mode 100755 tools/update_lldp.sh

diff --git a/common/src/address.rs b/common/src/address.rs
index 3a470c189b..48ec7c46ed 100644
--- a/common/src/address.rs
+++ b/common/src/address.rs
@@ -50,6 +50,7 @@ pub const CLICKHOUSE_PORT: u16 = 8123;
 pub const CLICKHOUSE_KEEPER_PORT: u16 = 9181;
 pub const OXIMETER_PORT: u16 = 12223;
 pub const DENDRITE_PORT: u16 = 12224;
+pub const LLDP_PORT: u16 = 12230;
 pub const MGD_PORT: u16 = 4676;
 pub const DDMD_PORT: u16 = 8000;
 pub const MGS_PORT: u16 = 12225;
diff --git a/package-manifest.toml b/package-manifest.toml
index b7e96935f1..a3e1fa133d 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -553,6 +553,15 @@ source.sha256 = "151aeb26414989cad571b3886786efbeeafd91c41a93a747c784cdc654d5876
 output.type = "zone"
 output.intermediate_only = true
 
+[package.lldp]
+service_name = "lldp"
+source.type = "prebuilt"
+source.repo = "lldp"
+source.commit = "e165090e393df90874329265799480e221884236"
+source.sha256 = "004a6488a948d894fa203222aa42e159e1ae12c13751607852302d111e2c58e6"
+output.type = "zone"
+output.intermediate_only = true
+
 [package.dendrite-stub]
 service_name = "dendrite"
 only_for_targets.switch = "stub"
@@ -648,6 +657,7 @@ source.type = "composite"
 source.packages = [
   "omicron-gateway-asic.tar.gz",
   "dendrite-asic.tar.gz",
+  "lldp.tar.gz",
   "wicketd.tar.gz",
   "wicket.tar.gz",
   "mg-ddm.tar.gz",
@@ -671,6 +681,7 @@ source.type = "composite"
 source.packages = [
   "omicron-gateway-stub.tar.gz",
   "dendrite-stub.tar.gz",
+  "lldp.tar.gz",
   "wicketd.tar.gz",
   "wicket.tar.gz",
   "mg-ddm.tar.gz",
@@ -694,6 +705,7 @@ source.type = "composite"
 source.packages = [
   "omicron-gateway-softnpu.tar.gz",
   "dendrite-softnpu.tar.gz",
+  "lldp.tar.gz",
   "wicketd.tar.gz",
   "wicket.tar.gz",
   "mg-ddm.tar.gz",
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
index 9273e0ef3b..2be4b8ec45 100644
--- a/sled-agent/src/services.rs
+++ b/sled-agent/src/services.rs
@@ -71,6 +71,7 @@ use omicron_common::address::CRUCIBLE_PORT;
 use omicron_common::address::DENDRITE_PORT;
 use omicron_common::address::DNS_HTTP_PORT;
 use omicron_common::address::DNS_PORT;
+use omicron_common::address::LLDP_PORT;
 use omicron_common::address::MGS_PORT;
 use omicron_common::address::RACK_PREFIX;
 use omicron_common::address::SLED_PREFIX;
@@ -470,6 +471,7 @@ enum SwitchService {
     ManagementGatewayService,
     Wicketd { baseboard: Baseboard },
     Dendrite { asic: DendriteAsic },
+    Lldpd { baseboard: Baseboard },
     Tfport { pkt_source: String, asic: DendriteAsic },
     Uplink,
     MgDdm { mode: String },
@@ -483,6 +485,7 @@ impl crate::smf_helper::Service for SwitchService {
             SwitchService::ManagementGatewayService => "mgs",
             SwitchService::Wicketd { .. } => "wicketd",
             SwitchService::Dendrite { .. } => "dendrite",
+            SwitchService::Lldpd { .. } => "lldpd",
             SwitchService::Tfport { .. } => "tfport",
             SwitchService::Uplink { .. } => "uplink",
             SwitchService::MgDdm { .. } => "mg-ddm",
@@ -1019,7 +1022,10 @@ impl ServiceManager {
         let mut devices = vec![];
         for svc_details in zone_args.sled_local_services() {
             match svc_details {
-                SwitchService::Dendrite { asic: DendriteAsic::TofinoAsic } => {
+                SwitchService::Dendrite {
+                    asic: DendriteAsic::TofinoAsic,
+                    ..
+                } => {
                     if let Ok(Some(n)) = tofino::get_tofino() {
                         if let Ok(device_path) = n.device_path() {
                             devices.push(device_path);
@@ -1032,6 +1038,7 @@ impl ServiceManager {
                 }
                 SwitchService::Dendrite {
                     asic: DendriteAsic::SoftNpuPropolisDevice,
+                    ..
                 } => {
                     devices.push("/dev/tty03".into());
                 }
@@ -2374,6 +2381,15 @@ impl ServiceManager {
             None
         };
 
+        let sidecar_revision = match &self.inner.sidecar_revision {
+            SidecarRevision::Physical(rev) => rev.to_string(),
+            SidecarRevision::SoftZone(rev)
+            | SidecarRevision::SoftPropolis(rev) => format!(
+                "softnpu_front_{}_rear_{}",
+                rev.front_port_count, rev.rear_port_count
+            ),
+        };
+
         if let Some(gateway) = maybe_gateway {
             running_zone.add_default_route(gateway).map_err(|err| {
                 Error::ZoneCommand { intent: "Adding Route".to_string(), err }
@@ -2596,19 +2612,7 @@ impl ServiceManager {
                                         "config/port_config",
                                         "/opt/oxide/dendrite/misc/sidecar_config.toml",
                                     )?;
-                                    let sidecar_revision =
-                                        match self.inner.sidecar_revision {
-                                            SidecarRevision::Physical(ref rev) => rev,
-                                            _ => {
-                                                return Err(Error::SidecarRevision(
-                                                    anyhow::anyhow!(
-                                                        "expected physical \
-                                                        sidecar revision"
-                                                    ),
-                                                ))
-                                            }
-                                        };
-                                    smfh.setprop("config/board_rev", sidecar_revision)?;
+                                    smfh.setprop("config/board_rev", &sidecar_revision)?;
                                 }
                                 DendriteAsic::TofinoStub => smfh.setprop(
                                     "config/port_config",
@@ -2710,6 +2714,39 @@ impl ServiceManager {
 
                             smfh.refresh()?;
                         }
+                        SwitchService::Lldpd { baseboard } => {
+                            info!(self.inner.log, "Setting up lldpd service");
+
+                            match baseboard {
+                                Baseboard::Gimlet {
+                                    identifier, model, ..
+                                }
+                                | Baseboard::Pc { identifier, model, .. } => {
+                                    smfh.setprop(
+                                        "config/scrimlet_id",
+                                        identifier,
+                                    )?;
+                                    smfh.setprop(
+                                        "config/scrimlet_model",
+                                        model,
+                                    )?;
+                                }
+                                Baseboard::Unknown => {}
+                            }
+                            smfh.setprop(
+                                "config/board_rev",
+                                &sidecar_revision,
+                            )?;
+
+                            smfh.delpropvalue("config/address", "*")?;
+                            for address in &request.zone.addresses {
+                                smfh.addpropvalue(
+                                    "config/address",
+                                    &format!("[{}]:{}", address, LLDP_PORT),
+                                )?;
+                            }
+                            smfh.refresh()?;
+                        }
                         SwitchService::Uplink => {
                             // Nothing to do here - this service is special and
                             // configured in
@@ -3509,13 +3546,14 @@ impl ServiceManager {
             | SledMode::Scrimlet { asic: DendriteAsic::TofinoAsic } => {
                 vec![
                     SwitchService::Dendrite { asic: DendriteAsic::TofinoAsic },
+                    SwitchService::Lldpd { baseboard: baseboard.clone() },
                     SwitchService::ManagementGatewayService,
                     SwitchService::Tfport {
                         pkt_source: "tfpkt0".to_string(),
                         asic: DendriteAsic::TofinoAsic,
                     },
                     SwitchService::Uplink,
-                    SwitchService::Wicketd { baseboard },
+                    SwitchService::Wicketd { baseboard: baseboard.clone() },
                     SwitchService::Mgd,
                     SwitchService::MgDdm { mode: "transit".to_string() },
                 ]
@@ -3527,9 +3565,10 @@ impl ServiceManager {
                 data_links = vec!["vioif0".to_owned()];
                 vec![
                     SwitchService::Dendrite { asic },
+                    SwitchService::Lldpd { baseboard: baseboard.clone() },
                     SwitchService::ManagementGatewayService,
                     SwitchService::Uplink,
-                    SwitchService::Wicketd { baseboard },
+                    SwitchService::Wicketd { baseboard: baseboard.clone() },
                     SwitchService::Mgd,
                     SwitchService::MgDdm { mode: "transit".to_string() },
                     SwitchService::Tfport {
@@ -3557,9 +3596,10 @@ impl ServiceManager {
                 }
                 vec![
                     SwitchService::Dendrite { asic },
+                    SwitchService::Lldpd { baseboard: baseboard.clone() },
                     SwitchService::ManagementGatewayService,
                     SwitchService::Uplink,
-                    SwitchService::Wicketd { baseboard },
+                    SwitchService::Wicketd { baseboard: baseboard.clone() },
                     SwitchService::Mgd,
                     SwitchService::MgDdm { mode: "transit".to_string() },
                     SwitchService::Tfport {
@@ -3823,7 +3863,10 @@ impl ServiceManager {
                             smfh.refresh()?;
                         }
                         SwitchService::Dendrite { .. } => {
-                            info!(self.inner.log, "configuring dendrite zone");
+                            info!(
+                                self.inner.log,
+                                "configuring dendrite service"
+                            );
                             if let Some(info) = self.inner.sled_info.get() {
                                 smfh.setprop("config/rack_id", info.rack_id)?;
                                 smfh.setprop(
@@ -3881,6 +3924,17 @@ impl ServiceManager {
                                 );
                             }
                         }
+                        SwitchService::Lldpd { .. } => {
+                            info!(self.inner.log, "configuring lldp service");
+                            smfh.delpropvalue("config/address", "*")?;
+                            for address in &request.addresses {
+                                smfh.addpropvalue(
+                                    "config/address",
+                                    &format!("[{}]:{}", address, LLDP_PORT),
+                                )?;
+                            }
+                            smfh.refresh()?;
+                        }
                         SwitchService::Tfport { .. } => {
                             // Since tfport and dpd communicate using localhost,
                             // the tfport service shouldn't need to be
diff --git a/tools/update_lldp.sh b/tools/update_lldp.sh
new file mode 100755
index 0000000000..29cf8517af
--- /dev/null
+++ b/tools/update_lldp.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+set -o pipefail
+set -o errexit
+
+SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+function usage {
+    echo "usage: $0 [-c COMMIT] [-n]"
+    echo
+    echo "  -c COMMIT   Ask to update Lldp to a specific commit."
+    echo "              If this is unset, Github is queried."
+    echo "  -n          Dry-run"
+    exit 1
+}
+
+PACKAGES=(
+  "lldp"
+)
+
+CRATES=(
+  "lldp"
+)
+
+REPO="oxidecomputer/lldp"
+
+. "$SOURCE_DIR/update_helpers.sh"
+
+function main {
+    TARGET_COMMIT=""
+    DRY_RUN=""
+    while getopts "c:n" o; do
+      case "${o}" in
+        c)
+          TARGET_COMMIT="$OPTARG"
+          ;;
+        n)
+          DRY_RUN="yes"
+          ;;
+        *)
+          usage
+          ;;
+      esac
+    done
+
+    TARGET_COMMIT=$(get_latest_commit_from_gh "$REPO" "$TARGET_COMMIT")
+    install_toml2json
+    do_update_packages "$TARGET_COMMIT" "$DRY_RUN" "$REPO" "${PACKAGES[@]}"
+    do_update_crates "$TARGET_COMMIT" "$DRY_RUN" "$REPO" "${CRATES[@]}"
+}
+
+main "$@"

From 2d8bdf93ba98cdfd680fab79abf7bcd407f4a37c Mon Sep 17 00:00:00 2001
From: James MacMahon <james@oxide.computer>
Date: Tue, 12 Mar 2024 15:13:16 -0400
Subject: [PATCH 099/157] Add Error::not_found variant (#5242)

Add a variant to our external Error so that a 404 can be returned
without requiring a ResourceType - adding one may not make sense for a
particular endpoint.
---
 common/src/api/external/error.rs | 29 +++++++++++++++++++++++++++++
 nexus/src/app/session.rs         |  3 ++-
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/common/src/api/external/error.rs b/common/src/api/external/error.rs
index 2cb3dc0d6e..ef64aa3bbb 100644
--- a/common/src/api/external/error.rs
+++ b/common/src/api/external/error.rs
@@ -71,6 +71,11 @@ pub enum Error {
 
     #[error("Conflict: {}", .message.display_internal())]
     Conflict { message: MessagePair },
+
+    /// A generic 404 response. If there is an applicable ResourceType, use
+    /// ObjectNotFound instead.
+    #[error("Not found: {}", .message.display_internal())]
+    NotFound { message: MessagePair },
 }
 
 /// Represents an error message which has an external component, along with
@@ -199,6 +204,7 @@ impl Error {
             | Error::InsufficientCapacity { .. }
             | Error::InternalError { .. }
             | Error::TypeVersionMismatch { .. }
+            | Error::NotFound { .. }
             | Error::Conflict { .. } => false,
         }
     }
@@ -300,6 +306,15 @@ impl Error {
         Error::Conflict { message: MessagePair::new(message.into()) }
     }
 
+    /// Generates an [`Error::NotFound`] with a specific message.
+    ///
+    /// This is used in cases where a generic 404 is required. For cases where
+    /// there is a ResourceType, use a function that produces
+    /// [`Error::ObjectNotFound`] instead.
+    pub fn non_resourcetype_not_found(message: impl Into<String>) -> Error {
+        Error::NotFound { message: MessagePair::new(message.into()) }
+    }
+
     /// Given an [`Error`] with an internal message, return the same error with
     /// `context` prepended to it to provide more context
     ///
@@ -354,6 +369,9 @@ impl Error {
             Error::Conflict { message } => Error::Conflict {
                 message: message.with_internal_context(context),
             },
+            Error::NotFound { message } => Error::NotFound {
+                message: message.with_internal_context(context),
+            },
         }
     }
 }
@@ -475,6 +493,17 @@ impl From<Error> for HttpError {
                     internal_message,
                 }
             }
+
+            Error::NotFound { message } => {
+                let (internal_message, external_message) =
+                    message.into_internal_external();
+                HttpError {
+                    status_code: http::StatusCode::NOT_FOUND,
+                    error_code: Some(String::from("Not Found")),
+                    external_message,
+                    internal_message,
+                }
+            }
         }
     }
 }
diff --git a/nexus/src/app/session.rs b/nexus/src/app/session.rs
index 7adf1c9bdd..dd3665161a 100644
--- a/nexus/src/app/session.rs
+++ b/nexus/src/app/session.rs
@@ -156,7 +156,8 @@ impl super::Nexus {
                 | Error::ServiceUnavailable { .. }
                 | Error::InsufficientCapacity { .. }
                 | Error::TypeVersionMismatch { .. }
-                | Error::Conflict { .. } => {
+                | Error::Conflict { .. }
+                | Error::NotFound { .. } => {
                     Reason::UnknownError { source: error }
                 }
             })?;

From fdbbbe54cc706ff92c1a327d20ab40febfa623a6 Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Tue, 12 Mar 2024 17:00:01 -0400
Subject: [PATCH 100/157] Blueprint execution: Send OPTE firewall rules for
 services (#5233)

This is the reworked and caught-up-with-main #5157 (which I'll close
momentarily). I believe I addressed all the comments on the PR except
those related to its extension trait style and the `nexus-capabilities`
crate. This introduces a more focused `nexus-networking` crate that
exposes free functions instead of extension traits.

I'll run this through madrid before merging, but I believe it should
work (absent any mistakes I made merging changes across branches) now
that #5202 has landed.

Closes #4886
---
 Cargo.lock                                    |  17 +
 Cargo.toml                                    |   3 +
 nexus/Cargo.toml                              |   1 +
 nexus/networking/Cargo.toml                   |  16 +
 nexus/networking/src/firewall_rules.rs        | 441 ++++++++++++++++++
 nexus/networking/src/lib.rs                   |  12 +
 nexus/networking/src/sled_client.rs           |  53 +++
 nexus/reconfigurator/execution/Cargo.toml     |   1 +
 nexus/reconfigurator/execution/src/lib.rs     |  21 +-
 .../execution/src/omicron_zones.rs            |  26 +-
 .../src/app/background/blueprint_execution.rs |   5 +-
 nexus/src/app/sled.rs                         |  41 +-
 nexus/src/app/vpc.rs                          | 412 +---------------
 13 files changed, 611 insertions(+), 438 deletions(-)
 create mode 100644 nexus/networking/Cargo.toml
 create mode 100644 nexus/networking/src/firewall_rules.rs
 create mode 100644 nexus/networking/src/lib.rs
 create mode 100644 nexus/networking/src/sled_client.rs

diff --git a/Cargo.lock b/Cargo.lock
index 5900cf3dec..e50918bba5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4562,6 +4562,21 @@ dependencies = [
  "syn 2.0.51",
 ]
 
+[[package]]
+name = "nexus-networking"
+version = "0.1.0"
+dependencies = [
+ "futures",
+ "ipnetwork",
+ "nexus-db-queries",
+ "omicron-common",
+ "omicron-workspace-hack",
+ "reqwest",
+ "sled-agent-client",
+ "slog",
+ "uuid",
+]
+
 [[package]]
 name = "nexus-reconfigurator-execution"
 version = "0.1.0"
@@ -4578,6 +4593,7 @@ dependencies = [
  "nexus-db-model",
  "nexus-db-queries",
  "nexus-inventory",
+ "nexus-networking",
  "nexus-reconfigurator-planning",
  "nexus-test-utils",
  "nexus-test-utils-macros",
@@ -5176,6 +5192,7 @@ dependencies = [
  "nexus-db-queries",
  "nexus-defaults",
  "nexus-inventory",
+ "nexus-networking",
  "nexus-reconfigurator-execution",
  "nexus-reconfigurator-planning",
  "nexus-reconfigurator-preparation",
diff --git a/Cargo.toml b/Cargo.toml
index f291d9642a..45c45c9d56 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -45,6 +45,7 @@ members = [
     "nexus/defaults",
     "nexus/inventory",
     "nexus/macros-common",
+    "nexus/networking",
     "nexus/reconfigurator/execution",
     "nexus/reconfigurator/planning",
     "nexus/reconfigurator/preparation",
@@ -120,6 +121,7 @@ default-members = [
     "nexus-config",
     "nexus/authz-macros",
     "nexus/macros-common",
+    "nexus/networking",
     "nexus/db-macros",
     "nexus/db-model",
     "nexus/db-queries",
@@ -272,6 +274,7 @@ nexus-db-queries = { path = "nexus/db-queries" }
 nexus-defaults = { path = "nexus/defaults" }
 nexus-inventory = { path = "nexus/inventory" }
 nexus-macros-common = { path = "nexus/macros-common" }
+nexus-networking = { path = "nexus/networking" }
 nexus-reconfigurator-execution = { path = "nexus/reconfigurator/execution" }
 nexus-reconfigurator-planning = { path = "nexus/reconfigurator/planning" }
 nexus-reconfigurator-preparation = { path = "nexus/reconfigurator/preparation" }
diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml
index 0712ba6743..2b8845f6f5 100644
--- a/nexus/Cargo.toml
+++ b/nexus/Cargo.toml
@@ -41,6 +41,7 @@ mime_guess.workspace = true
 # Not under "dev-dependencies"; these also need to be implemented for
 # integration tests.
 nexus-config.workspace = true
+nexus-networking.workspace = true
 nexus-test-interface.workspace = true
 num-integer.workspace = true
 once_cell.workspace = true
diff --git a/nexus/networking/Cargo.toml b/nexus/networking/Cargo.toml
new file mode 100644
index 0000000000..11f6d83993
--- /dev/null
+++ b/nexus/networking/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name = "nexus-networking"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+
+[dependencies]
+futures.workspace = true
+ipnetwork.workspace = true
+nexus-db-queries.workspace = true
+omicron-common.workspace = true
+reqwest.workspace = true
+sled-agent-client.workspace = true
+slog.workspace = true
+uuid.workspace = true
+omicron-workspace-hack.workspace = true
diff --git a/nexus/networking/src/firewall_rules.rs b/nexus/networking/src/firewall_rules.rs
new file mode 100644
index 0000000000..85c457d522
--- /dev/null
+++ b/nexus/networking/src/firewall_rules.rs
@@ -0,0 +1,441 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Functionality related to firewall rules.
+
+use futures::future::join_all;
+use ipnetwork::IpNetwork;
+use nexus_db_queries::authz;
+use nexus_db_queries::context::OpContext;
+use nexus_db_queries::db;
+use nexus_db_queries::db::identity::Asset;
+use nexus_db_queries::db::identity::Resource;
+use nexus_db_queries::db::lookup;
+use nexus_db_queries::db::lookup::LookupPath;
+use nexus_db_queries::db::model::Name;
+use nexus_db_queries::db::DataStore;
+use omicron_common::api::external;
+use omicron_common::api::external::Error;
+use omicron_common::api::external::IpNet;
+use omicron_common::api::external::ListResultVec;
+use omicron_common::api::internal::nexus::HostIdentifier;
+use omicron_common::api::internal::shared::NetworkInterface;
+use slog::debug;
+use slog::info;
+use slog::warn;
+use slog::Logger;
+use std::collections::{HashMap, HashSet};
+use std::net::IpAddr;
+use uuid::Uuid;
+
+pub async fn vpc_list_firewall_rules(
+    datastore: &DataStore,
+    opctx: &OpContext,
+    vpc_lookup: &lookup::Vpc<'_>,
+) -> ListResultVec<db::model::VpcFirewallRule> {
+    let (.., authz_vpc) = vpc_lookup.lookup_for(authz::Action::Read).await?;
+    let rules = datastore.vpc_list_firewall_rules(&opctx, &authz_vpc).await?;
+    Ok(rules)
+}
+
+pub async fn resolve_firewall_rules_for_sled_agent(
+    datastore: &DataStore,
+    opctx: &OpContext,
+    vpc: &db::model::Vpc,
+    rules: &[db::model::VpcFirewallRule],
+    log: &Logger,
+) -> Result<Vec<sled_agent_client::types::VpcFirewallRule>, Error> {
+    // Collect the names of instances, subnets, and VPCs that are either
+    // targets or host filters. We have to find the sleds for all the
+    // targets, and we'll need information about the IP addresses or
+    // subnets for things that are specified as host filters as well.
+    let mut instances: HashSet<Name> = HashSet::new();
+    let mut subnets: HashSet<Name> = HashSet::new();
+    let mut vpcs: HashSet<Name> = HashSet::new();
+    for rule in rules {
+        for target in &rule.targets {
+            match &target.0 {
+                external::VpcFirewallRuleTarget::Instance(name) => {
+                    instances.insert(name.clone().into());
+                }
+                external::VpcFirewallRuleTarget::Subnet(name) => {
+                    subnets.insert(name.clone().into());
+                }
+                external::VpcFirewallRuleTarget::Vpc(name) => {
+                    if name != vpc.name() {
+                        return Err(Error::invalid_request(
+                            "cross-VPC firewall target unsupported",
+                        ));
+                    }
+                    vpcs.insert(name.clone().into());
+                }
+                external::VpcFirewallRuleTarget::Ip(_)
+                | external::VpcFirewallRuleTarget::IpNet(_) => {
+                    vpcs.insert(vpc.name().clone().into());
+                }
+            }
+        }
+
+        for host in rule.filter_hosts.iter().flatten() {
+            match &host.0 {
+                external::VpcFirewallRuleHostFilter::Instance(name) => {
+                    instances.insert(name.clone().into());
+                }
+                external::VpcFirewallRuleHostFilter::Subnet(name) => {
+                    subnets.insert(name.clone().into());
+                }
+                external::VpcFirewallRuleHostFilter::Vpc(name) => {
+                    if name != vpc.name() {
+                        return Err(Error::invalid_request(
+                            "cross-VPC firewall host filter unsupported",
+                        ));
+                    }
+                    vpcs.insert(name.clone().into());
+                }
+                // We don't need to resolve anything for Ip(Net)s.
+                external::VpcFirewallRuleHostFilter::Ip(_) => (),
+                external::VpcFirewallRuleHostFilter::IpNet(_) => (),
+            }
+        }
+    }
+
+    // Resolve named instances, VPCs, and subnets.
+    // TODO-correctness: It's possible the resolving queries produce
+    // inconsistent results due to concurrent changes. They should be
+    // transactional.
+    type NetMap = HashMap<external::Name, Vec<IpNetwork>>;
+    type NicMap = HashMap<external::Name, Vec<NetworkInterface>>;
+    let no_networks: Vec<IpNetwork> = Vec::new();
+    let no_interfaces: Vec<NetworkInterface> = Vec::new();
+
+    let mut instance_interfaces: NicMap = HashMap::new();
+    for instance_name in &instances {
+        if let Ok((.., authz_instance)) = LookupPath::new(opctx, datastore)
+            .project_id(vpc.project_id)
+            .instance_name(instance_name)
+            .lookup_for(authz::Action::ListChildren)
+            .await
+        {
+            for iface in datastore
+                .derive_guest_network_interface_info(opctx, &authz_instance)
+                .await?
+            {
+                instance_interfaces
+                    .entry(instance_name.0.clone())
+                    .or_insert_with(Vec::new)
+                    .push(iface);
+            }
+        }
+    }
+
+    let mut vpc_interfaces: NicMap = HashMap::new();
+    for vpc_name in &vpcs {
+        if let Ok((.., authz_vpc)) = LookupPath::new(opctx, datastore)
+            .project_id(vpc.project_id)
+            .vpc_name(vpc_name)
+            .lookup_for(authz::Action::ListChildren)
+            .await
+        {
+            for iface in datastore
+                .derive_vpc_network_interface_info(opctx, &authz_vpc)
+                .await?
+            {
+                vpc_interfaces
+                    .entry(vpc_name.0.clone())
+                    .or_insert_with(Vec::new)
+                    .push(iface);
+            }
+        }
+    }
+
+    let mut subnet_interfaces: NicMap = HashMap::new();
+    for subnet_name in &subnets {
+        if let Ok((.., authz_subnet)) = LookupPath::new(opctx, datastore)
+            .project_id(vpc.project_id)
+            .vpc_name(&Name::from(vpc.name().clone()))
+            .vpc_subnet_name(subnet_name)
+            .lookup_for(authz::Action::ListChildren)
+            .await
+        {
+            for iface in datastore
+                .derive_subnet_network_interface_info(opctx, &authz_subnet)
+                .await?
+            {
+                subnet_interfaces
+                    .entry(subnet_name.0.clone())
+                    .or_insert_with(Vec::new)
+                    .push(iface);
+            }
+        }
+    }
+
+    let subnet_networks: NetMap = datastore
+        .resolve_vpc_subnets_to_ip_networks(vpc, subnets)
+        .await?
+        .into_iter()
+        .map(|(name, v)| (name.0, v))
+        .collect();
+
+    debug!(
+        log,
+        "resolved names for firewall rules";
+        "instance_interfaces" => ?instance_interfaces,
+        "vpc_interfaces" => ?vpc_interfaces,
+        "subnet_interfaces" => ?subnet_interfaces,
+        "subnet_networks" => ?subnet_networks,
+    );
+
+    // Compile resolved rules for the sled agents.
+    let mut sled_agent_rules = Vec::with_capacity(rules.len());
+    for rule in rules {
+        // TODO: what is the correct behavior when a name is not found?
+        // Options:
+        // (1) Fail update request (though note this can still arise
+        //     from things like instance deletion)
+        // (2) Allow update request, ignore this rule (but store it
+        //     in case it becomes valid later). This is consistent
+        //     with the semantics of the rules. Rules with bad
+        //     references should likely at least be flagged to users.
+        // We currently adopt option (2), as this allows users to add
+        // firewall rules (including default rules) before instances
+        // and their interfaces are instantiated.
+
+        // Collect unique network interface targets.
+        // This would be easier if `NetworkInterface` were `Hash`,
+        // but that's not easy because it's a generated type. We
+        // use the pair (VNI, MAC) as a unique interface identifier.
+        let mut nics = HashSet::new();
+        let mut targets = Vec::with_capacity(rule.targets.len());
+        let mut push_target_nic = |nic: &NetworkInterface| {
+            if nics.insert((nic.vni, *nic.mac)) {
+                targets.push(nic.clone());
+            }
+        };
+        for target in &rule.targets {
+            match &target.0 {
+                external::VpcFirewallRuleTarget::Vpc(name) => {
+                    vpc_interfaces
+                        .get(&name)
+                        .unwrap_or(&no_interfaces)
+                        .iter()
+                        .for_each(&mut push_target_nic);
+                }
+                external::VpcFirewallRuleTarget::Subnet(name) => {
+                    subnet_interfaces
+                        .get(&name)
+                        .unwrap_or(&no_interfaces)
+                        .iter()
+                        .for_each(&mut push_target_nic);
+                }
+                external::VpcFirewallRuleTarget::Instance(name) => {
+                    instance_interfaces
+                        .get(&name)
+                        .unwrap_or(&no_interfaces)
+                        .iter()
+                        .for_each(&mut push_target_nic);
+                }
+                external::VpcFirewallRuleTarget::Ip(addr) => {
+                    vpc_interfaces
+                        .get(vpc.name())
+                        .unwrap_or(&no_interfaces)
+                        .iter()
+                        .filter(|nic| nic.ip == *addr)
+                        .for_each(&mut push_target_nic);
+                }
+                external::VpcFirewallRuleTarget::IpNet(net) => {
+                    vpc_interfaces
+                        .get(vpc.name())
+                        .unwrap_or(&no_interfaces)
+                        .iter()
+                        .filter(|nic| match (net, nic.ip) {
+                            (IpNet::V4(net), IpAddr::V4(ip)) => {
+                                net.contains(ip)
+                            }
+                            (IpNet::V6(net), IpAddr::V6(ip)) => {
+                                net.contains(ip)
+                            }
+                            (_, _) => false,
+                        })
+                        .for_each(&mut push_target_nic);
+                }
+            }
+        }
+        if !rule.targets.is_empty() && targets.is_empty() {
+            // Target not found; skip this rule.
+            continue;
+        }
+
+        let filter_hosts = match &rule.filter_hosts {
+            None => None,
+            Some(hosts) => {
+                let mut host_addrs = Vec::with_capacity(hosts.len());
+                for host in hosts {
+                    match &host.0 {
+                        external::VpcFirewallRuleHostFilter::Instance(name) => {
+                            for interface in instance_interfaces
+                                .get(&name)
+                                .unwrap_or(&no_interfaces)
+                            {
+                                host_addrs.push(
+                                    HostIdentifier::Ip(IpNet::from(
+                                        interface.ip,
+                                    ))
+                                    .into(),
+                                )
+                            }
+                        }
+                        external::VpcFirewallRuleHostFilter::Subnet(name) => {
+                            for subnet in subnet_networks
+                                .get(&name)
+                                .unwrap_or(&no_networks)
+                            {
+                                host_addrs.push(
+                                    HostIdentifier::Ip(IpNet::from(*subnet))
+                                        .into(),
+                                );
+                            }
+                        }
+                        external::VpcFirewallRuleHostFilter::Ip(addr) => {
+                            host_addrs.push(
+                                HostIdentifier::Ip(IpNet::from(*addr)).into(),
+                            )
+                        }
+                        external::VpcFirewallRuleHostFilter::IpNet(net) => {
+                            host_addrs.push(HostIdentifier::Ip(*net).into())
+                        }
+                        external::VpcFirewallRuleHostFilter::Vpc(name) => {
+                            for interface in vpc_interfaces
+                                .get(&name)
+                                .unwrap_or(&no_interfaces)
+                            {
+                                host_addrs.push(
+                                    HostIdentifier::Vpc(interface.vni).into(),
+                                )
+                            }
+                        }
+                    }
+                }
+                if !hosts.is_empty() && host_addrs.is_empty() {
+                    // Filter host not found; skip this rule.
+                    continue;
+                }
+                Some(host_addrs)
+            }
+        };
+
+        let filter_ports = rule
+            .filter_ports
+            .as_ref()
+            .map(|ports| ports.iter().map(|v| v.0.into()).collect());
+
+        let filter_protocols = rule
+            .filter_protocols
+            .as_ref()
+            .map(|protocols| protocols.iter().map(|v| v.0.into()).collect());
+
+        sled_agent_rules.push(sled_agent_client::types::VpcFirewallRule {
+            status: rule.status.0.into(),
+            direction: rule.direction.0.into(),
+            targets,
+            filter_hosts,
+            filter_ports,
+            filter_protocols,
+            action: rule.action.0.into(),
+            priority: rule.priority.0 .0,
+        });
+    }
+    debug!(
+        log,
+        "resolved firewall rules for sled agents";
+        "sled_agent_rules" => ?sled_agent_rules,
+    );
+
+    Ok(sled_agent_rules)
+}
+
+pub async fn send_sled_agents_firewall_rules(
+    datastore: &DataStore,
+    opctx: &OpContext,
+    vpc: &db::model::Vpc,
+    rules: &[db::model::VpcFirewallRule],
+    sleds_filter: &[Uuid],
+    sled_lookup_opctx: &OpContext,
+    log: &Logger,
+) -> Result<(), Error> {
+    let rules_for_sled = resolve_firewall_rules_for_sled_agent(
+        datastore, opctx, &vpc, rules, log,
+    )
+    .await?;
+    debug!(log, "resolved {} rules for sleds", rules_for_sled.len());
+    let sled_rules_request =
+        sled_agent_client::types::VpcFirewallRulesEnsureBody {
+            vni: vpc.vni.0,
+            rules: rules_for_sled,
+        };
+
+    let vpc_to_sleds =
+        datastore.vpc_resolve_to_sleds(vpc.id(), sleds_filter).await?;
+    debug!(
+        log, "resolved sleds for vpc {}", vpc.name();
+        "vpc_to_sled" => ?vpc_to_sleds,
+    );
+
+    let mut sled_requests = Vec::with_capacity(vpc_to_sleds.len());
+    for sled in &vpc_to_sleds {
+        let sled_id = sled.id();
+        let vpc_id = vpc.id();
+        let sled_rules_request = sled_rules_request.clone();
+        sled_requests.push(async move {
+            crate::sled_client(datastore, sled_lookup_opctx, sled_id, log)
+                .await?
+                .vpc_firewall_rules_put(&vpc_id, &sled_rules_request)
+                .await
+                .map_err(|e| Error::internal_error(&e.to_string()))
+        });
+    }
+
+    debug!(log, "sending firewall rules to sled agents");
+    let results = join_all(sled_requests).await;
+    // TODO-correctness: handle more than one failure in the sled-agent requests
+    //   https://github.com/oxidecomputer/omicron/issues/1791
+    for (sled, result) in vpc_to_sleds.iter().zip(results) {
+        if let Err(e) = result {
+            warn!(log, "failed to update firewall rules on sled agent";
+                      "sled_id" => %sled.id(),
+                      "vpc_id" => %vpc.id(),
+                      "error" => %e);
+            return Err(e);
+        }
+    }
+    info!(log, "updated firewall rules on {} sleds", vpc_to_sleds.len());
+
+    Ok(())
+}
+
+/// Ensure firewall rules for internal services get reflected on all the
+/// relevant sleds.
+pub async fn plumb_service_firewall_rules(
+    datastore: &DataStore,
+    opctx: &OpContext,
+    sleds_filter: &[Uuid],
+    sled_lookup_opctx: &OpContext,
+    log: &Logger,
+) -> Result<(), Error> {
+    let svcs_vpc = LookupPath::new(opctx, datastore)
+        .vpc_id(*db::fixed_data::vpc::SERVICES_VPC_ID);
+    let svcs_fw_rules =
+        vpc_list_firewall_rules(datastore, opctx, &svcs_vpc).await?;
+    let (_, _, _, svcs_vpc) = svcs_vpc.fetch().await?;
+    send_sled_agents_firewall_rules(
+        datastore,
+        opctx,
+        &svcs_vpc,
+        &svcs_fw_rules,
+        sleds_filter,
+        sled_lookup_opctx,
+        log,
+    )
+    .await?;
+    Ok(())
+}
diff --git a/nexus/networking/src/lib.rs b/nexus/networking/src/lib.rs
new file mode 100644
index 0000000000..98278a5c9c
--- /dev/null
+++ b/nexus/networking/src/lib.rs
@@ -0,0 +1,12 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Networking functionality shared between Nexus proper and its background
+//! tasks or sagas.
+
+mod firewall_rules;
+mod sled_client;
+
+pub use firewall_rules::*;
+pub use sled_client::*;
diff --git a/nexus/networking/src/sled_client.rs b/nexus/networking/src/sled_client.rs
new file mode 100644
index 0000000000..073f073356
--- /dev/null
+++ b/nexus/networking/src/sled_client.rs
@@ -0,0 +1,53 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Functionality for constructing sled-agent clients.
+
+use nexus_db_queries::context::OpContext;
+use nexus_db_queries::db::lookup;
+use nexus_db_queries::db::lookup::LookupPath;
+use nexus_db_queries::db::DataStore;
+use omicron_common::api::external::Error;
+use omicron_common::api::external::LookupResult;
+use sled_agent_client::Client as SledAgentClient;
+use slog::o;
+use slog::Logger;
+use std::net::SocketAddrV6;
+use uuid::Uuid;
+
+pub fn sled_lookup<'a>(
+    datastore: &'a DataStore,
+    opctx: &'a OpContext,
+    sled_id: Uuid,
+) -> LookupResult<lookup::Sled<'a>> {
+    let sled = LookupPath::new(opctx, datastore).sled_id(sled_id);
+    Ok(sled)
+}
+
+pub async fn sled_client(
+    datastore: &DataStore,
+    lookup_opctx: &OpContext,
+    sled_id: Uuid,
+    log: &Logger,
+) -> Result<SledAgentClient, Error> {
+    let (.., sled) =
+        sled_lookup(datastore, lookup_opctx, sled_id)?.fetch().await?;
+
+    Ok(sled_client_from_address(sled_id, sled.address(), log))
+}
+
+pub fn sled_client_from_address(
+    sled_id: Uuid,
+    address: SocketAddrV6,
+    log: &Logger,
+) -> SledAgentClient {
+    let log = log.new(o!("SledAgent" => sled_id.to_string()));
+    let dur = std::time::Duration::from_secs(60);
+    let client = reqwest::ClientBuilder::new()
+        .connect_timeout(dur)
+        .timeout(dur)
+        .build()
+        .unwrap();
+    SledAgentClient::new_with_client(&format!("http://{address}"), client, log)
+}
diff --git a/nexus/reconfigurator/execution/Cargo.toml b/nexus/reconfigurator/execution/Cargo.toml
index 62155d9783..d48f4c6b5b 100644
--- a/nexus/reconfigurator/execution/Cargo.toml
+++ b/nexus/reconfigurator/execution/Cargo.toml
@@ -15,6 +15,7 @@ internal-dns.workspace = true
 nexus-config.workspace = true
 nexus-db-model.workspace = true
 nexus-db-queries.workspace = true
+nexus-networking.workspace = true
 nexus-types.workspace = true
 omicron-common.workspace = true
 reqwest.workspace = true
diff --git a/nexus/reconfigurator/execution/src/lib.rs b/nexus/reconfigurator/execution/src/lib.rs
index 74b4764d7a..386e9dc1c8 100644
--- a/nexus/reconfigurator/execution/src/lib.rs
+++ b/nexus/reconfigurator/execution/src/lib.rs
@@ -68,7 +68,7 @@ where
     info!(
         opctx.log,
         "attempting to realize blueprint";
-        "blueprint_id" => ?blueprint.id
+        "blueprint_id" => %blueprint.id
     );
 
     resource_allocation::ensure_zone_resources_allocated(
@@ -90,6 +90,25 @@ where
     omicron_zones::deploy_zones(&opctx, &sleds_by_id, &blueprint.omicron_zones)
         .await?;
 
+    // After deploying omicron zones, we may need to refresh OPTE service
+    // firewall rules. This is an idempotent operation, so we don't attempt
+    // to optimize out calling it in unnecessary cases, although it is only
+    // needed in cases where we've changed the set of services on one or more
+    // sleds, or the sleds have lost their firewall rules for some reason.
+    // Fixing the latter case is a side effect and should really be handled by a
+    // firewall-rule-specific RPW; once that RPW exists, we could trigger it
+    // here instead of pluming firewall rules ourselves.
+    nexus_networking::plumb_service_firewall_rules(
+        datastore,
+        &opctx,
+        &[],
+        &opctx,
+        &opctx.log,
+    )
+    .await
+    .context("failed to plumb service firewall rules to sleds")
+    .map_err(|err| vec![err])?;
+
     datasets::ensure_crucible_dataset_records_exist(
         &opctx,
         datastore,
diff --git a/nexus/reconfigurator/execution/src/omicron_zones.rs b/nexus/reconfigurator/execution/src/omicron_zones.rs
index 1d5c4444b1..336e40df16 100644
--- a/nexus/reconfigurator/execution/src/omicron_zones.rs
+++ b/nexus/reconfigurator/execution/src/omicron_zones.rs
@@ -11,7 +11,6 @@ use futures::stream;
 use futures::StreamExt;
 use nexus_db_queries::context::OpContext;
 use nexus_types::deployment::OmicronZonesConfig;
-use sled_agent_client::Client as SledAgentClient;
 use slog::info;
 use slog::warn;
 use std::collections::BTreeMap;
@@ -34,7 +33,11 @@ pub(crate) async fn deploy_zones(
                     return Some(err);
                 }
             };
-            let client = sled_client(opctx, &db_sled);
+            let client = nexus_networking::sled_client_from_address(
+                *sled_id,
+                db_sled.sled_agent_address,
+                &opctx.log,
+            );
             let result =
                 client.omicron_zones_put(&config).await.with_context(|| {
                     format!("Failed to put {config:#?} to sled {sled_id}")
@@ -65,25 +68,6 @@ pub(crate) async fn deploy_zones(
     }
 }
 
-// This is a modified copy of the functionality from `nexus/src/app/sled.rs`.
-// There's no good way to access this functionality right now since it is a
-// method on the `Nexus` type. We want to have a more constrained type we can
-// pass into background tasks for this type of functionality, but for now we
-// just copy the functionality.
-fn sled_client(opctx: &OpContext, sled: &Sled) -> SledAgentClient {
-    let dur = std::time::Duration::from_secs(60);
-    let client = reqwest::ClientBuilder::new()
-        .connect_timeout(dur)
-        .timeout(dur)
-        .build()
-        .unwrap();
-    SledAgentClient::new_with_client(
-        &format!("http://{}", sled.sled_agent_address),
-        client,
-        opctx.log.clone(),
-    )
-}
-
 #[cfg(test)]
 mod test {
     use super::deploy_zones;
diff --git a/nexus/src/app/background/blueprint_execution.rs b/nexus/src/app/background/blueprint_execution.rs
index ceb3cd7a05..7024f8be75 100644
--- a/nexus/src/app/background/blueprint_execution.rs
+++ b/nexus/src/app/background/blueprint_execution.rs
@@ -109,6 +109,7 @@ mod test {
     use nexus_db_model::{
         ByteCount, SledBaseboard, SledSystemHardware, SledUpdate,
     };
+    use nexus_db_queries::authn;
     use nexus_db_queries::context::OpContext;
     use nexus_test_utils_macros::nexus_test;
     use nexus_types::deployment::OmicronZonesConfig;
@@ -158,8 +159,10 @@ mod test {
         // Set up the test.
         let nexus = &cptestctx.server.apictx().nexus;
         let datastore = nexus.datastore();
-        let opctx = OpContext::for_tests(
+        let opctx = OpContext::for_background(
             cptestctx.logctx.log.clone(),
+            nexus.authz.clone(),
+            authn::Context::internal_api(),
             datastore.clone(),
         );
 
diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs
index 88f70c7d0d..aad6e1122f 100644
--- a/nexus/src/app/sled.rs
+++ b/nexus/src/app/sled.rs
@@ -31,8 +31,7 @@ impl super::Nexus {
         opctx: &'a OpContext,
         sled_id: &Uuid,
     ) -> LookupResult<lookup::Sled<'a>> {
-        let sled = LookupPath::new(opctx, &self.db_datastore).sled_id(*sled_id);
-        Ok(sled)
+        nexus_networking::sled_lookup(&self.db_datastore, opctx, *sled_id)
     }
 
     // TODO-robustness we should have a limit on how many sled agents there can
@@ -103,21 +102,14 @@ impl super::Nexus {
         // Frankly, returning an "Arc" here without a connection pool is a
         // little silly; it's not actually used if each client connection exists
         // as a one-shot.
-        let (.., sled) =
-            self.sled_lookup(&self.opctx_alloc, id)?.fetch().await?;
-
-        let log = self.log.new(o!("SledAgent" => id.clone().to_string()));
-        let dur = std::time::Duration::from_secs(60);
-        let client = reqwest::ClientBuilder::new()
-            .connect_timeout(dur)
-            .timeout(dur)
-            .build()
-            .unwrap();
-        Ok(Arc::new(SledAgentClient::new_with_client(
-            &format!("http://{}", sled.address()),
-            client,
-            log,
-        )))
+        let client = nexus_networking::sled_client(
+            &self.db_datastore,
+            &self.opctx_alloc,
+            *id,
+            &self.log,
+        )
+        .await?;
+        Ok(Arc::new(client))
     }
 
     pub(crate) async fn reserve_on_random_sled(
@@ -286,18 +278,13 @@ impl super::Nexus {
         opctx: &OpContext,
         sleds_filter: &[Uuid],
     ) -> Result<(), Error> {
-        let svcs_vpc = LookupPath::new(opctx, &self.db_datastore)
-            .vpc_id(*db::fixed_data::vpc::SERVICES_VPC_ID);
-        let svcs_fw_rules =
-            self.vpc_list_firewall_rules(opctx, &svcs_vpc).await?;
-        let (_, _, _, svcs_vpc) = svcs_vpc.fetch().await?;
-        self.send_sled_agents_firewall_rules(
+        nexus_networking::plumb_service_firewall_rules(
+            &self.db_datastore,
             opctx,
-            &svcs_vpc,
-            &svcs_fw_rules,
             sleds_filter,
+            &self.opctx_alloc,
+            &self.log,
         )
-        .await?;
-        Ok(())
+        .await
     }
 }
diff --git a/nexus/src/app/vpc.rs b/nexus/src/app/vpc.rs
index 44b676b853..0950e65b83 100644
--- a/nexus/src/app/vpc.rs
+++ b/nexus/src/app/vpc.rs
@@ -10,8 +10,6 @@ use nexus_db_queries::authn;
 use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db;
-use nexus_db_queries::db::identity::Asset;
-use nexus_db_queries::db::identity::Resource;
 use nexus_db_queries::db::lookup;
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_db_queries::db::model::Name;
@@ -22,20 +20,12 @@ use omicron_common::api::external::CreateResult;
 use omicron_common::api::external::DeleteResult;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::InternalContext;
-use omicron_common::api::external::IpNet;
 use omicron_common::api::external::ListResultVec;
 use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::LookupType;
 use omicron_common::api::external::NameOrId;
 use omicron_common::api::external::UpdateResult;
 use omicron_common::api::external::VpcFirewallRuleUpdateParams;
-use omicron_common::api::internal::nexus::HostIdentifier;
-
-use futures::future::join_all;
-use ipnetwork::IpNetwork;
-use omicron_common::api::internal::shared::NetworkInterface;
-use std::collections::{HashMap, HashSet};
-use std::net::IpAddr;
 use std::sync::Arc;
 use uuid::Uuid;
 
@@ -170,13 +160,12 @@ impl super::Nexus {
         opctx: &OpContext,
         vpc_lookup: &lookup::Vpc<'_>,
     ) -> ListResultVec<db::model::VpcFirewallRule> {
-        let (.., authz_vpc) =
-            vpc_lookup.lookup_for(authz::Action::Read).await?;
-        let rules = self
-            .db_datastore
-            .vpc_list_firewall_rules(&opctx, &authz_vpc)
-            .await?;
-        Ok(rules)
+        nexus_networking::vpc_list_firewall_rules(
+            &self.db_datastore,
+            opctx,
+            vpc_lookup,
+        )
+        .await
     }
 
     pub(crate) async fn vpc_update_firewall_rules(
@@ -252,56 +241,16 @@ impl super::Nexus {
         rules: &[db::model::VpcFirewallRule],
         sleds_filter: &[Uuid],
     ) -> Result<(), Error> {
-        let rules_for_sled = self
-            .resolve_firewall_rules_for_sled_agent(opctx, &vpc, rules)
-            .await?;
-        debug!(self.log, "resolved {} rules for sleds", rules_for_sled.len());
-        let sled_rules_request =
-            sled_agent_client::types::VpcFirewallRulesEnsureBody {
-                vni: vpc.vni.0,
-                rules: rules_for_sled,
-            };
-
-        let vpc_to_sleds = self
-            .db_datastore
-            .vpc_resolve_to_sleds(vpc.id(), sleds_filter)
-            .await?;
-        debug!(self.log, "resolved sleds for vpc {}", vpc.name(); "vpc_to_sled" => ?vpc_to_sleds);
-
-        let mut sled_requests = Vec::with_capacity(vpc_to_sleds.len());
-        for sled in &vpc_to_sleds {
-            let sled_id = sled.id();
-            let vpc_id = vpc.id();
-            let sled_rules_request = sled_rules_request.clone();
-            sled_requests.push(async move {
-                self.sled_client(&sled_id)
-                    .await?
-                    .vpc_firewall_rules_put(&vpc_id, &sled_rules_request)
-                    .await
-                    .map_err(|e| Error::internal_error(&e.to_string()))
-            });
-        }
-
-        debug!(self.log, "sending firewall rules to sled agents");
-        let results = join_all(sled_requests).await;
-        // TODO-correctness: handle more than one failure in the sled-agent requests
-        //   https://github.com/oxidecomputer/omicron/issues/1791
-        for (sled, result) in vpc_to_sleds.iter().zip(results) {
-            if let Err(e) = result {
-                warn!(self.log, "failed to update firewall rules on sled agent";
-                      "sled_id" => %sled.id(),
-                      "vpc_id" => %vpc.id(),
-                      "error" => %e);
-                return Err(e);
-            }
-        }
-        info!(
-            self.log,
-            "updated firewall rules on {} sleds",
-            vpc_to_sleds.len()
-        );
-
-        Ok(())
+        nexus_networking::send_sled_agents_firewall_rules(
+            &self.db_datastore,
+            opctx,
+            vpc,
+            rules,
+            sleds_filter,
+            &self.opctx_alloc,
+            &self.log,
+        )
+        .await
     }
 
     pub(crate) async fn resolve_firewall_rules_for_sled_agent(
@@ -310,326 +259,13 @@ impl super::Nexus {
         vpc: &db::model::Vpc,
         rules: &[db::model::VpcFirewallRule],
     ) -> Result<Vec<sled_agent_client::types::VpcFirewallRule>, Error> {
-        // Collect the names of instances, subnets, and VPCs that are either
-        // targets or host filters. We have to find the sleds for all the
-        // targets, and we'll need information about the IP addresses or
-        // subnets for things that are specified as host filters as well.
-        let mut instances: HashSet<Name> = HashSet::new();
-        let mut subnets: HashSet<Name> = HashSet::new();
-        let mut vpcs: HashSet<Name> = HashSet::new();
-        for rule in rules {
-            for target in &rule.targets {
-                match &target.0 {
-                    external::VpcFirewallRuleTarget::Instance(name) => {
-                        instances.insert(name.clone().into());
-                    }
-                    external::VpcFirewallRuleTarget::Subnet(name) => {
-                        subnets.insert(name.clone().into());
-                    }
-                    external::VpcFirewallRuleTarget::Vpc(name) => {
-                        if name != vpc.name() {
-                            return Err(Error::invalid_request(
-                                "cross-VPC firewall target unsupported",
-                            ));
-                        }
-                        vpcs.insert(name.clone().into());
-                    }
-                    external::VpcFirewallRuleTarget::Ip(_)
-                    | external::VpcFirewallRuleTarget::IpNet(_) => {
-                        vpcs.insert(vpc.name().clone().into());
-                    }
-                }
-            }
-
-            for host in rule.filter_hosts.iter().flatten() {
-                match &host.0 {
-                    external::VpcFirewallRuleHostFilter::Instance(name) => {
-                        instances.insert(name.clone().into());
-                    }
-                    external::VpcFirewallRuleHostFilter::Subnet(name) => {
-                        subnets.insert(name.clone().into());
-                    }
-                    external::VpcFirewallRuleHostFilter::Vpc(name) => {
-                        if name != vpc.name() {
-                            return Err(Error::invalid_request(
-                                "cross-VPC firewall host filter unsupported",
-                            ));
-                        }
-                        vpcs.insert(name.clone().into());
-                    }
-                    // We don't need to resolve anything for Ip(Net)s.
-                    external::VpcFirewallRuleHostFilter::Ip(_) => (),
-                    external::VpcFirewallRuleHostFilter::IpNet(_) => (),
-                }
-            }
-        }
-
-        // Resolve named instances, VPCs, and subnets.
-        // TODO-correctness: It's possible the resolving queries produce
-        // inconsistent results due to concurrent changes. They should be
-        // transactional.
-        type NetMap = HashMap<external::Name, Vec<IpNetwork>>;
-        type NicMap = HashMap<external::Name, Vec<NetworkInterface>>;
-        let no_networks: Vec<IpNetwork> = Vec::new();
-        let no_interfaces: Vec<NetworkInterface> = Vec::new();
-
-        let mut instance_interfaces: NicMap = HashMap::new();
-        for instance_name in &instances {
-            if let Ok((.., authz_instance)) =
-                LookupPath::new(opctx, &self.db_datastore)
-                    .project_id(vpc.project_id)
-                    .instance_name(instance_name)
-                    .lookup_for(authz::Action::ListChildren)
-                    .await
-            {
-                for iface in self
-                    .db_datastore
-                    .derive_guest_network_interface_info(opctx, &authz_instance)
-                    .await?
-                {
-                    instance_interfaces
-                        .entry(instance_name.0.clone())
-                        .or_insert_with(Vec::new)
-                        .push(iface);
-                }
-            }
-        }
-
-        let mut vpc_interfaces: NicMap = HashMap::new();
-        for vpc_name in &vpcs {
-            if let Ok((.., authz_vpc)) =
-                LookupPath::new(opctx, &self.db_datastore)
-                    .project_id(vpc.project_id)
-                    .vpc_name(vpc_name)
-                    .lookup_for(authz::Action::ListChildren)
-                    .await
-            {
-                for iface in self
-                    .db_datastore
-                    .derive_vpc_network_interface_info(opctx, &authz_vpc)
-                    .await?
-                {
-                    vpc_interfaces
-                        .entry(vpc_name.0.clone())
-                        .or_insert_with(Vec::new)
-                        .push(iface);
-                }
-            }
-        }
-
-        let mut subnet_interfaces: NicMap = HashMap::new();
-        for subnet_name in &subnets {
-            if let Ok((.., authz_subnet)) =
-                LookupPath::new(opctx, &self.db_datastore)
-                    .project_id(vpc.project_id)
-                    .vpc_name(&Name::from(vpc.name().clone()))
-                    .vpc_subnet_name(subnet_name)
-                    .lookup_for(authz::Action::ListChildren)
-                    .await
-            {
-                for iface in self
-                    .db_datastore
-                    .derive_subnet_network_interface_info(opctx, &authz_subnet)
-                    .await?
-                {
-                    subnet_interfaces
-                        .entry(subnet_name.0.clone())
-                        .or_insert_with(Vec::new)
-                        .push(iface);
-                }
-            }
-        }
-
-        let subnet_networks: NetMap = self
-            .db_datastore
-            .resolve_vpc_subnets_to_ip_networks(vpc, subnets)
-            .await?
-            .into_iter()
-            .map(|(name, v)| (name.0, v))
-            .collect();
-
-        debug!(
-            self.log,
-            "resolved names for firewall rules";
-            "instance_interfaces" => ?instance_interfaces,
-            "vpc_interfaces" => ?vpc_interfaces,
-            "subnet_interfaces" => ?subnet_interfaces,
-            "subnet_networks" => ?subnet_networks,
-        );
-
-        // Compile resolved rules for the sled agents.
-        let mut sled_agent_rules = Vec::with_capacity(rules.len());
-        for rule in rules {
-            // TODO: what is the correct behavior when a name is not found?
-            // Options:
-            // (1) Fail update request (though note this can still arise
-            //     from things like instance deletion)
-            // (2) Allow update request, ignore this rule (but store it
-            //     in case it becomes valid later). This is consistent
-            //     with the semantics of the rules. Rules with bad
-            //     references should likely at least be flagged to users.
-            // We currently adopt option (2), as this allows users to add
-            // firewall rules (including default rules) before instances
-            // and their interfaces are instantiated.
-
-            // Collect unique network interface targets.
-            // This would be easier if `NetworkInterface` were `Hash`,
-            // but that's not easy because it's a generated type. We
-            // use the pair (VNI, MAC) as a unique interface identifier.
-            let mut nics = HashSet::new();
-            let mut targets = Vec::with_capacity(rule.targets.len());
-            let mut push_target_nic = |nic: &NetworkInterface| {
-                if nics.insert((nic.vni, *nic.mac)) {
-                    targets.push(nic.clone());
-                }
-            };
-            for target in &rule.targets {
-                match &target.0 {
-                    external::VpcFirewallRuleTarget::Vpc(name) => {
-                        vpc_interfaces
-                            .get(&name)
-                            .unwrap_or(&no_interfaces)
-                            .iter()
-                            .for_each(&mut push_target_nic);
-                    }
-                    external::VpcFirewallRuleTarget::Subnet(name) => {
-                        subnet_interfaces
-                            .get(&name)
-                            .unwrap_or(&no_interfaces)
-                            .iter()
-                            .for_each(&mut push_target_nic);
-                    }
-                    external::VpcFirewallRuleTarget::Instance(name) => {
-                        instance_interfaces
-                            .get(&name)
-                            .unwrap_or(&no_interfaces)
-                            .iter()
-                            .for_each(&mut push_target_nic);
-                    }
-                    external::VpcFirewallRuleTarget::Ip(addr) => {
-                        vpc_interfaces
-                            .get(vpc.name())
-                            .unwrap_or(&no_interfaces)
-                            .iter()
-                            .filter(|nic| nic.ip == *addr)
-                            .for_each(&mut push_target_nic);
-                    }
-                    external::VpcFirewallRuleTarget::IpNet(net) => {
-                        vpc_interfaces
-                            .get(vpc.name())
-                            .unwrap_or(&no_interfaces)
-                            .iter()
-                            .filter(|nic| match (net, nic.ip) {
-                                (IpNet::V4(net), IpAddr::V4(ip)) => {
-                                    net.contains(ip)
-                                }
-                                (IpNet::V6(net), IpAddr::V6(ip)) => {
-                                    net.contains(ip)
-                                }
-                                (_, _) => false,
-                            })
-                            .for_each(&mut push_target_nic);
-                    }
-                }
-            }
-            if !rule.targets.is_empty() && targets.is_empty() {
-                // Target not found; skip this rule.
-                continue;
-            }
-
-            let filter_hosts = match &rule.filter_hosts {
-                None => None,
-                Some(hosts) => {
-                    let mut host_addrs = Vec::with_capacity(hosts.len());
-                    for host in hosts {
-                        match &host.0 {
-                            external::VpcFirewallRuleHostFilter::Instance(
-                                name,
-                            ) => {
-                                for interface in instance_interfaces
-                                    .get(&name)
-                                    .unwrap_or(&no_interfaces)
-                                {
-                                    host_addrs.push(
-                                        HostIdentifier::Ip(IpNet::from(
-                                            interface.ip,
-                                        ))
-                                        .into(),
-                                    )
-                                }
-                            }
-                            external::VpcFirewallRuleHostFilter::Subnet(
-                                name,
-                            ) => {
-                                for subnet in subnet_networks
-                                    .get(&name)
-                                    .unwrap_or(&no_networks)
-                                {
-                                    host_addrs.push(
-                                        HostIdentifier::Ip(IpNet::from(
-                                            *subnet,
-                                        ))
-                                        .into(),
-                                    );
-                                }
-                            }
-                            external::VpcFirewallRuleHostFilter::Ip(addr) => {
-                                host_addrs.push(
-                                    HostIdentifier::Ip(IpNet::from(*addr))
-                                        .into(),
-                                )
-                            }
-                            external::VpcFirewallRuleHostFilter::IpNet(net) => {
-                                host_addrs.push(HostIdentifier::Ip(*net).into())
-                            }
-                            external::VpcFirewallRuleHostFilter::Vpc(name) => {
-                                for interface in vpc_interfaces
-                                    .get(&name)
-                                    .unwrap_or(&no_interfaces)
-                                {
-                                    host_addrs.push(
-                                        HostIdentifier::Vpc(interface.vni)
-                                            .into(),
-                                    )
-                                }
-                            }
-                        }
-                    }
-                    if !hosts.is_empty() && host_addrs.is_empty() {
-                        // Filter host not found; skip this rule.
-                        continue;
-                    }
-                    Some(host_addrs)
-                }
-            };
-
-            let filter_ports = rule
-                .filter_ports
-                .as_ref()
-                .map(|ports| ports.iter().map(|v| v.0.into()).collect());
-
-            let filter_protocols =
-                rule.filter_protocols.as_ref().map(|protocols| {
-                    protocols.iter().map(|v| v.0.into()).collect()
-                });
-
-            sled_agent_rules.push(sled_agent_client::types::VpcFirewallRule {
-                status: rule.status.0.into(),
-                direction: rule.direction.0.into(),
-                targets,
-                filter_hosts,
-                filter_ports,
-                filter_protocols,
-                action: rule.action.0.into(),
-                priority: rule.priority.0 .0,
-            });
-        }
-        debug!(
-            self.log,
-            "resolved firewall rules for sled agents";
-            "sled_agent_rules" => ?sled_agent_rules,
-        );
-
-        Ok(sled_agent_rules)
+        nexus_networking::resolve_firewall_rules_for_sled_agent(
+            &self.db_datastore,
+            opctx,
+            vpc,
+            rules,
+            &self.log,
+        )
+        .await
     }
 }

From 85150a7436bb6dfa12f98bf843956e1a4fb9ddab Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Tue, 12 Mar 2024 20:07:00 -0400
Subject: [PATCH 101/157] Split out a `sled-hardware-types` crate (#5245)

On main as of #5158, we unexpectedly get some binaries depending on
`libnvme` that shouldn't. In release builds:

```
installinator
omicron-dev
omicron-package
services-ledger-check-migrate
sled-agent
sled-agent-sim
wicketd
```

and in debug builds, all of the above plus `omdb`. We don't really care
about this for binaries that don't run on the rack, so stripping the
list down to binaries we do care about:

```
installinator
omdb (debug only)
sled-agent
wicketd
```

It's correct and expected that installinator and sled-agent depend on
libnvme, but omdb shouldn't (and doesn't in release), and wicketd _must
not_, as libnvme isn't available in the switch zone.

This PR fixes that incorrect dependency by splitting the parts of the
`sled-hardware` crate that wicketd and omdb depended on (directly or
transitively) into a new `sled-hardware-types` crate. On this PR, we are
left with only the following needing libnvme, in both debug and release:

```
installinator
omicron-dev
omicron-package
services-ledger-check-migrate
sled-agent
sled-agent-sim
```

I assume with a bit more work we could trim out everything but
`installinator` and `sled-agent`, and that might be worth doing, but
we'd like to land this ASAP so as to not break any updates performed off
of `main`. Separately, we could also imagine a CI check that we don't
have unexpected library dependencies present in the final binaries;
@papertigers is going to work on that.
---
 Cargo.lock                                    | 24 ++++-
 Cargo.toml                                    |  3 +
 bootstore/Cargo.toml                          |  2 +-
 bootstore/src/schemes/v0/fsm.rs               |  2 +-
 bootstore/src/schemes/v0/messages.rs          |  2 +-
 bootstore/src/schemes/v0/peer.rs              |  2 +-
 bootstore/src/schemes/v0/peer_networking.rs   |  2 +-
 bootstore/src/schemes/v0/request_manager.rs   |  2 +-
 bootstore/src/schemes/v0/share_pkg.rs         |  2 +-
 bootstore/src/schemes/v0/storage.rs           |  2 +-
 bootstore/tests/common/generators.rs          |  2 +-
 bootstore/tests/common/mod.rs                 |  2 +-
 bootstore/tests/v0-fsm-proptest-learner.rs    |  2 +-
 .../tests/v0-fsm-proptest-rack-coordinator.rs |  2 +-
 clients/bootstrap-agent-client/Cargo.toml     |  2 +-
 clients/bootstrap-agent-client/src/lib.rs     | 22 +++--
 clients/ddm-admin-client/Cargo.toml           |  2 +-
 clients/ddm-admin-client/src/lib.rs           |  6 +-
 clients/mg-admin-client/Cargo.toml            |  1 -
 dev-tools/omdb/Cargo.toml                     |  2 +-
 dev-tools/omdb/src/bin/omdb/db.rs             |  2 +-
 installinator/Cargo.toml                      |  1 +
 installinator/src/bootstrap.rs                |  2 +-
 installinator/src/peers.rs                    |  2 +-
 sled-agent/Cargo.toml                         |  1 +
 sled-agent/src/bin/sled-agent-sim.rs          |  2 +-
 sled-agent/src/bootstrap/bootstore_setup.rs   |  4 +-
 sled-agent/src/bootstrap/http_entrypoints.rs  |  2 +-
 sled-agent/src/bootstrap/params.rs            |  2 +-
 sled-agent/src/bootstrap/pre_server.rs        |  5 +-
 sled-agent/src/hardware_monitor.rs            |  3 +-
 sled-agent/src/metrics.rs                     |  2 +-
 sled-agent/src/nexus.rs                       |  6 +-
 sled-agent/src/params.rs                      |  2 +-
 sled-agent/src/rack_setup/service.rs          |  2 +-
 sled-agent/src/services.rs                    |  4 +-
 sled-agent/src/sim/config.rs                  |  2 +-
 sled-agent/src/sled_agent.rs                  |  8 +-
 sled-hardware/Cargo.toml                      |  1 +
 sled-hardware/src/illumos/mod.rs              |  4 +-
 sled-hardware/src/lib.rs                      | 89 -----------------
 sled-hardware/src/non_illumos/mod.rs          |  3 +-
 sled-hardware/src/underlay.rs                 | 66 -------------
 sled-hardware/types/Cargo.toml                | 15 +++
 sled-hardware/types/src/lib.rs                | 97 +++++++++++++++++++
 sled-hardware/types/src/underlay.rs           | 72 ++++++++++++++
 wicketd/Cargo.toml                            |  2 +-
 wicketd/src/bin/wicketd.rs                    |  2 +-
 wicketd/src/bootstrap_addrs.rs                |  4 +-
 wicketd/src/context.rs                        |  2 +-
 wicketd/src/http_entrypoints.rs               |  2 +-
 wicketd/src/lib.rs                            |  2 +-
 wicketd/src/rss_config.rs                     |  2 +-
 53 files changed, 279 insertions(+), 222 deletions(-)
 create mode 100644 sled-hardware/types/Cargo.toml
 create mode 100644 sled-hardware/types/src/lib.rs
 create mode 100644 sled-hardware/types/src/underlay.rs

diff --git a/Cargo.lock b/Cargo.lock
index e50918bba5..0d10f6b675 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -637,7 +637,7 @@ dependencies = [
  "serde",
  "serde_with",
  "sha3",
- "sled-hardware",
+ "sled-hardware-types",
  "slog",
  "slog-async",
  "slog-term",
@@ -660,7 +660,7 @@ dependencies = [
  "reqwest",
  "schemars",
  "serde",
- "sled-hardware",
+ "sled-hardware-types",
  "slog",
  "uuid",
 ]
@@ -1595,7 +1595,7 @@ dependencies = [
  "rustfmt-wrapper",
  "serde",
  "serde_json",
- "sled-hardware",
+ "sled-hardware-types",
  "slog",
  "thiserror",
  "tokio",
@@ -3556,6 +3556,7 @@ dependencies = [
  "reqwest",
  "sha2",
  "sled-hardware",
+ "sled-hardware-types",
  "sled-storage",
  "slog",
  "slog-async",
@@ -4181,7 +4182,6 @@ dependencies = [
  "rustfmt-wrapper",
  "serde",
  "serde_json",
- "sled-hardware",
  "slog",
  "thiserror",
  "tokio",
@@ -5447,6 +5447,7 @@ dependencies = [
  "sha3",
  "sled-agent-client",
  "sled-hardware",
+ "sled-hardware-types",
  "sled-storage",
  "slog",
  "slog-async",
@@ -8346,6 +8347,7 @@ dependencies = [
  "rand 0.8.5",
  "schemars",
  "serde",
+ "sled-hardware-types",
  "slog",
  "slog-error-chain",
  "thiserror",
@@ -8354,6 +8356,18 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "sled-hardware-types"
+version = "0.1.0"
+dependencies = [
+ "illumos-utils",
+ "macaddr",
+ "omicron-common",
+ "omicron-workspace-hack",
+ "schemars",
+ "serde",
+]
+
 [[package]]
 name = "sled-storage"
 version = "0.1.0"
@@ -10623,7 +10637,7 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2",
- "sled-hardware",
+ "sled-hardware-types",
  "slog",
  "slog-dtrace",
  "subprocess",
diff --git a/Cargo.toml b/Cargo.toml
index 45c45c9d56..733eee1b32 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -64,6 +64,7 @@ members = [
     "rpaths",
     "sled-agent",
     "sled-hardware",
+    "sled-hardware/types",
     "sled-storage",
     "sp-sim",
     "test-utils",
@@ -142,6 +143,7 @@ default-members = [
     "rpaths",
     "sled-agent",
     "sled-hardware",
+    "sled-hardware/types",
     "sled-storage",
     "sp-sim",
     "test-utils",
@@ -373,6 +375,7 @@ similar-asserts = "1.5.0"
 sled = "0.34"
 sled-agent-client = { path = "clients/sled-agent-client" }
 sled-hardware = { path = "sled-hardware" }
+sled-hardware-types = { path = "sled-hardware/types" }
 sled-storage = { path = "sled-storage" }
 slog = { version = "2.7", features = [ "dynamic-keys", "max_level_trace", "release_max_level_debug" ] }
 slog-async = "2.8"
diff --git a/bootstore/Cargo.toml b/bootstore/Cargo.toml
index 37280f6dcb..5e9bcd1ef4 100644
--- a/bootstore/Cargo.toml
+++ b/bootstore/Cargo.toml
@@ -22,7 +22,7 @@ secrecy.workspace = true
 serde.workspace = true
 serde_with.workspace = true
 sha3.workspace = true
-sled-hardware.workspace = true
+sled-hardware-types.workspace = true
 slog.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
diff --git a/bootstore/src/schemes/v0/fsm.rs b/bootstore/src/schemes/v0/fsm.rs
index 6382ca7df1..4f2bcc2d11 100644
--- a/bootstore/src/schemes/v0/fsm.rs
+++ b/bootstore/src/schemes/v0/fsm.rs
@@ -23,7 +23,7 @@ use secrecy::ExposeSecret;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 use sha3::{Digest, Sha3_256};
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use std::collections::{BTreeMap, BTreeSet};
 use std::fmt::Debug;
 use std::time::Instant;
diff --git a/bootstore/src/schemes/v0/messages.rs b/bootstore/src/schemes/v0/messages.rs
index db6ec4f281..a524df5c88 100644
--- a/bootstore/src/schemes/v0/messages.rs
+++ b/bootstore/src/schemes/v0/messages.rs
@@ -7,7 +7,7 @@
 use super::{LearnedSharePkg, RackUuid, Share, SharePkg};
 use derive_more::From;
 use serde::{Deserialize, Serialize};
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use std::net::SocketAddrV6;
 use thiserror::Error;
 use uuid::Uuid;
diff --git a/bootstore/src/schemes/v0/peer.rs b/bootstore/src/schemes/v0/peer.rs
index 1175e64143..efb916a61f 100644
--- a/bootstore/src/schemes/v0/peer.rs
+++ b/bootstore/src/schemes/v0/peer.rs
@@ -14,7 +14,7 @@ use super::{ApiError, ApiOutput, Fsm, FsmConfig, RackUuid};
 use crate::trust_quorum::RackSecret;
 use camino::Utf8PathBuf;
 use derive_more::From;
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use slog::{error, info, o, warn, Logger};
 use std::collections::{BTreeMap, BTreeSet};
 use std::net::{SocketAddr, SocketAddrV6};
diff --git a/bootstore/src/schemes/v0/peer_networking.rs b/bootstore/src/schemes/v0/peer_networking.rs
index 33673cdab8..13afd27fa2 100644
--- a/bootstore/src/schemes/v0/peer_networking.rs
+++ b/bootstore/src/schemes/v0/peer_networking.rs
@@ -11,7 +11,7 @@ use crate::schemes::Hello;
 use bytes::Buf;
 use derive_more::From;
 use serde::{Deserialize, Serialize};
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use slog::{debug, error, info, o, warn, Logger};
 use std::collections::VecDeque;
 use std::io::Cursor;
diff --git a/bootstore/src/schemes/v0/request_manager.rs b/bootstore/src/schemes/v0/request_manager.rs
index 90466fdc07..2ad907aaa1 100644
--- a/bootstore/src/schemes/v0/request_manager.rs
+++ b/bootstore/src/schemes/v0/request_manager.rs
@@ -7,7 +7,7 @@
 use super::{
     Envelope, FsmConfig, Msg, RackUuid, Request, RequestType, Share, SharePkg,
 };
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use std::collections::{BTreeMap, BTreeSet};
 use std::time::Instant;
 use uuid::Uuid;
diff --git a/bootstore/src/schemes/v0/share_pkg.rs b/bootstore/src/schemes/v0/share_pkg.rs
index 2446f1904d..f89d5b7313 100644
--- a/bootstore/src/schemes/v0/share_pkg.rs
+++ b/bootstore/src/schemes/v0/share_pkg.rs
@@ -12,7 +12,7 @@ use rand::{rngs::OsRng, RngCore};
 use secrecy::{ExposeSecret, Secret};
 use serde::{Deserialize, Serialize};
 use sha3::{Digest, Sha3_256};
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use std::collections::BTreeSet;
 use std::fmt;
 use uuid::Uuid;
diff --git a/bootstore/src/schemes/v0/storage.rs b/bootstore/src/schemes/v0/storage.rs
index 327acc6058..624d0ee6ed 100644
--- a/bootstore/src/schemes/v0/storage.rs
+++ b/bootstore/src/schemes/v0/storage.rs
@@ -15,7 +15,7 @@ use super::{Fsm, FsmConfig, State};
 use camino::Utf8PathBuf;
 use omicron_common::ledger::{Ledger, Ledgerable};
 use serde::{Deserialize, Serialize};
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use slog::{info, Logger};
 
 /// A persistent version of `Fsm::State`
diff --git a/bootstore/tests/common/generators.rs b/bootstore/tests/common/generators.rs
index 1760aefc96..4f1a2fdbc9 100644
--- a/bootstore/tests/common/generators.rs
+++ b/bootstore/tests/common/generators.rs
@@ -6,7 +6,7 @@
 
 use bootstore::schemes::v0::{FsmConfig, MsgError};
 use proptest::prelude::*;
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use std::collections::BTreeSet;
 use std::ops::RangeInclusive;
 use std::time::Duration;
diff --git a/bootstore/tests/common/mod.rs b/bootstore/tests/common/mod.rs
index 0cff91cfa0..ed396efde5 100644
--- a/bootstore/tests/common/mod.rs
+++ b/bootstore/tests/common/mod.rs
@@ -12,7 +12,7 @@ use bootstore::schemes::v0::{
     ApiError, Envelope, Fsm, FsmConfig, Msg, MsgError, RackUuid, Request,
     RequestType, Response, ResponseType,
 };
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use std::collections::{BTreeMap, BTreeSet};
 use std::time::Instant;
 
diff --git a/bootstore/tests/v0-fsm-proptest-learner.rs b/bootstore/tests/v0-fsm-proptest-learner.rs
index 2295f5fc8e..b70c3c3a9d 100644
--- a/bootstore/tests/v0-fsm-proptest-learner.rs
+++ b/bootstore/tests/v0-fsm-proptest-learner.rs
@@ -19,7 +19,7 @@ use bootstore::trust_quorum::RackSecret;
 use common::CommonTestState;
 use proptest::prelude::*;
 use secrecy::ExposeSecret;
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use std::collections::{BTreeMap, BTreeSet};
 use uuid::Uuid;
 
diff --git a/bootstore/tests/v0-fsm-proptest-rack-coordinator.rs b/bootstore/tests/v0-fsm-proptest-rack-coordinator.rs
index 8bf149568a..0de1f83ba7 100644
--- a/bootstore/tests/v0-fsm-proptest-rack-coordinator.rs
+++ b/bootstore/tests/v0-fsm-proptest-rack-coordinator.rs
@@ -20,7 +20,7 @@ use bootstore::schemes::v0::{
     Request, RequestType, Response, ResponseType, Share,
 };
 use proptest::prelude::*;
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use std::collections::{BTreeMap, BTreeSet};
 use uuid::Uuid;
 
diff --git a/clients/bootstrap-agent-client/Cargo.toml b/clients/bootstrap-agent-client/Cargo.toml
index 3474c5814a..ea5d5358f9 100644
--- a/clients/bootstrap-agent-client/Cargo.toml
+++ b/clients/bootstrap-agent-client/Cargo.toml
@@ -12,7 +12,7 @@ regress.workspace = true
 reqwest = { workspace = true, features = [ "json", "rustls-tls", "stream" ] }
 schemars.workspace = true
 serde.workspace = true
-sled-hardware.workspace = true
+sled-hardware-types.workspace = true
 slog.workspace = true
 uuid.workspace = true
 omicron-workspace-hack.workspace = true
diff --git a/clients/bootstrap-agent-client/src/lib.rs b/clients/bootstrap-agent-client/src/lib.rs
index 19ecb599f3..61ebd04e1b 100644
--- a/clients/bootstrap-agent-client/src/lib.rs
+++ b/clients/bootstrap-agent-client/src/lib.rs
@@ -31,32 +31,36 @@ impl omicron_common::api::external::ClientError for types::Error {
     }
 }
 
-impl From<types::Baseboard> for sled_hardware::Baseboard {
+impl From<types::Baseboard> for sled_hardware_types::Baseboard {
     fn from(value: types::Baseboard) -> Self {
         match value {
             types::Baseboard::Gimlet { identifier, model, revision } => {
-                sled_hardware::Baseboard::new_gimlet(
+                sled_hardware_types::Baseboard::new_gimlet(
                     identifier, model, revision,
                 )
             }
-            types::Baseboard::Unknown => sled_hardware::Baseboard::unknown(),
+            types::Baseboard::Unknown => {
+                sled_hardware_types::Baseboard::unknown()
+            }
             types::Baseboard::Pc { identifier, model } => {
-                sled_hardware::Baseboard::new_pc(identifier, model)
+                sled_hardware_types::Baseboard::new_pc(identifier, model)
             }
         }
     }
 }
 
-impl From<sled_hardware::Baseboard> for types::Baseboard {
-    fn from(value: sled_hardware::Baseboard) -> Self {
+impl From<sled_hardware_types::Baseboard> for types::Baseboard {
+    fn from(value: sled_hardware_types::Baseboard) -> Self {
         match value {
-            sled_hardware::Baseboard::Gimlet {
+            sled_hardware_types::Baseboard::Gimlet {
                 identifier,
                 model,
                 revision,
             } => types::Baseboard::Gimlet { identifier, model, revision },
-            sled_hardware::Baseboard::Unknown => types::Baseboard::Unknown,
-            sled_hardware::Baseboard::Pc { identifier, model } => {
+            sled_hardware_types::Baseboard::Unknown => {
+                types::Baseboard::Unknown
+            }
+            sled_hardware_types::Baseboard::Pc { identifier, model } => {
                 types::Baseboard::Pc { identifier, model }
             }
         }
diff --git a/clients/ddm-admin-client/Cargo.toml b/clients/ddm-admin-client/Cargo.toml
index 4d00f329e7..f4aee0a961 100644
--- a/clients/ddm-admin-client/Cargo.toml
+++ b/clients/ddm-admin-client/Cargo.toml
@@ -14,7 +14,7 @@ thiserror.workspace = true
 tokio.workspace = true
 
 omicron-common.workspace = true
-sled-hardware.workspace = true
+sled-hardware-types.workspace = true
 omicron-workspace-hack.workspace = true
 
 [build-dependencies]
diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs
index c9dea15964..b88ea51e00 100644
--- a/clients/ddm-admin-client/src/lib.rs
+++ b/clients/ddm-admin-client/src/lib.rs
@@ -26,9 +26,9 @@ use omicron_common::address::Ipv6Subnet;
 use omicron_common::address::SLED_PREFIX;
 use omicron_common::backoff::retry_notify;
 use omicron_common::backoff::retry_policy_internal_service_aggressive;
-use sled_hardware::underlay::BootstrapInterface;
-use sled_hardware::underlay::BOOTSTRAP_MASK;
-use sled_hardware::underlay::BOOTSTRAP_PREFIX;
+use sled_hardware_types::underlay::BootstrapInterface;
+use sled_hardware_types::underlay::BOOTSTRAP_MASK;
+use sled_hardware_types::underlay::BOOTSTRAP_PREFIX;
 use slog::info;
 use slog::Logger;
 use std::net::Ipv6Addr;
diff --git a/clients/mg-admin-client/Cargo.toml b/clients/mg-admin-client/Cargo.toml
index c444fee32f..e426fa2371 100644
--- a/clients/mg-admin-client/Cargo.toml
+++ b/clients/mg-admin-client/Cargo.toml
@@ -13,7 +13,6 @@ slog.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
 omicron-common.workspace = true
-sled-hardware.workspace = true
 omicron-workspace-hack.workspace = true
 
 [build-dependencies]
diff --git a/dev-tools/omdb/Cargo.toml b/dev-tools/omdb/Cargo.toml
index 2b0480be2d..1b697561a9 100644
--- a/dev-tools/omdb/Cargo.toml
+++ b/dev-tools/omdb/Cargo.toml
@@ -49,11 +49,11 @@ unicode-width.workspace = true
 uuid.workspace = true
 ipnetwork.workspace = true
 omicron-workspace-hack.workspace = true
-nexus-test-utils.workspace = true
 multimap.workspace = true
 
 [dev-dependencies]
 expectorate.workspace = true
+nexus-test-utils.workspace = true
 nexus-test-utils-macros.workspace = true
 omicron-nexus.workspace = true
 omicron-test-utils.workspace = true
diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs
index 98e7748054..1b99a815ef 100644
--- a/dev-tools/omdb/src/bin/omdb/db.rs
+++ b/dev-tools/omdb/src/bin/omdb/db.rs
@@ -77,9 +77,9 @@ use nexus_db_queries::db::datastore::InstanceAndActiveVmm;
 use nexus_db_queries::db::identity::Asset;
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_db_queries::db::model::ServiceKind;
+use nexus_db_queries::db::queries::ALLOW_FULL_TABLE_SCAN_SQL;
 use nexus_db_queries::db::DataStore;
 use nexus_reconfigurator_preparation::policy_from_db;
-use nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL;
 use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::OmicronZoneType;
 use nexus_types::deployment::UnstableReconfiguratorState;
diff --git a/installinator/Cargo.toml b/installinator/Cargo.toml
index 43966d1202..1f92913aa1 100644
--- a/installinator/Cargo.toml
+++ b/installinator/Cargo.toml
@@ -27,6 +27,7 @@ omicron-common.workspace = true
 reqwest.workspace = true
 sha2.workspace = true
 sled-hardware.workspace = true
+sled-hardware-types.workspace = true
 sled-storage.workspace = true
 slog.workspace = true
 slog-async.workspace = true
diff --git a/installinator/src/bootstrap.rs b/installinator/src/bootstrap.rs
index 71c76809db..59541df28e 100644
--- a/installinator/src/bootstrap.rs
+++ b/installinator/src/bootstrap.rs
@@ -16,7 +16,7 @@ use illumos_utils::dladm::Dladm;
 use illumos_utils::zone::Zones;
 use omicron_common::address::Ipv6Subnet;
 use sled_hardware::underlay;
-use sled_hardware::underlay::BootstrapInterface;
+use sled_hardware_types::underlay::BootstrapInterface;
 use slog::info;
 use slog::Logger;
 
diff --git a/installinator/src/peers.rs b/installinator/src/peers.rs
index 3627ae861b..33ba0de0e5 100644
--- a/installinator/src/peers.rs
+++ b/installinator/src/peers.rs
@@ -25,7 +25,7 @@ use itertools::Itertools;
 use omicron_common::address::BOOTSTRAP_ARTIFACT_PORT;
 use omicron_common::update::ArtifactHashId;
 use reqwest::StatusCode;
-use sled_hardware::underlay::BootstrapInterface;
+use sled_hardware_types::underlay::BootstrapInterface;
 use tokio::{sync::mpsc, time::Instant};
 use update_engine::events::ProgressUnits;
 use uuid::Uuid;
diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml
index 7f46a277ce..239887e789 100644
--- a/sled-agent/Cargo.toml
+++ b/sled-agent/Cargo.toml
@@ -63,6 +63,7 @@ serde_json = { workspace = true, features = ["raw_value"] }
 sha3.workspace = true
 sled-agent-client.workspace = true
 sled-hardware.workspace = true
+sled-hardware-types.workspace = true
 sled-storage.workspace = true
 slog.workspace = true
 slog-async.workspace = true
diff --git a/sled-agent/src/bin/sled-agent-sim.rs b/sled-agent/src/bin/sled-agent-sim.rs
index 8de9a3c423..fcd7d02623 100644
--- a/sled-agent/src/bin/sled-agent-sim.rs
+++ b/sled-agent/src/bin/sled-agent-sim.rs
@@ -20,7 +20,7 @@ use omicron_sled_agent::sim::{
     run_standalone_server, Config, ConfigHardware, ConfigStorage, ConfigZpool,
     SimMode,
 };
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use std::net::SocketAddr;
 use std::net::SocketAddrV6;
 use uuid::Uuid;
diff --git a/sled-agent/src/bootstrap/bootstore_setup.rs b/sled-agent/src/bootstrap/bootstore_setup.rs
index 9eb0a87c03..120c7df000 100644
--- a/sled-agent/src/bootstrap/bootstore_setup.rs
+++ b/sled-agent/src/bootstrap/bootstore_setup.rs
@@ -12,8 +12,8 @@ use super::server::StartError;
 use bootstore::schemes::v0 as bootstore;
 use camino::Utf8PathBuf;
 use ddm_admin_client::Client as DdmAdminClient;
-use sled_hardware::underlay::BootstrapInterface;
-use sled_hardware::Baseboard;
+use sled_hardware_types::underlay::BootstrapInterface;
+use sled_hardware_types::Baseboard;
 use sled_storage::dataset::CLUSTER_DATASET;
 use sled_storage::resources::StorageResources;
 use slog::Logger;
diff --git a/sled-agent/src/bootstrap/http_entrypoints.rs b/sled-agent/src/bootstrap/http_entrypoints.rs
index 7c32bf48a5..68098f431e 100644
--- a/sled-agent/src/bootstrap/http_entrypoints.rs
+++ b/sled-agent/src/bootstrap/http_entrypoints.rs
@@ -23,7 +23,7 @@ use http::StatusCode;
 use omicron_common::api::external::Error;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use sled_storage::manager::StorageHandle;
 use slog::Logger;
 use std::net::Ipv6Addr;
diff --git a/sled-agent/src/bootstrap/params.rs b/sled-agent/src/bootstrap/params.rs
index 48444af8d4..4835edf4a9 100644
--- a/sled-agent/src/bootstrap/params.rs
+++ b/sled-agent/src/bootstrap/params.rs
@@ -12,7 +12,7 @@ use omicron_common::ledger::Ledgerable;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use sha3::{Digest, Sha3_256};
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use std::borrow::Cow;
 use std::collections::BTreeSet;
 use std::net::{IpAddr, Ipv6Addr, SocketAddrV6};
diff --git a/sled-agent/src/bootstrap/pre_server.rs b/sled-agent/src/bootstrap/pre_server.rs
index e61e15d370..6769af8ca2 100644
--- a/sled-agent/src/bootstrap/pre_server.rs
+++ b/sled-agent/src/bootstrap/pre_server.rs
@@ -38,6 +38,7 @@ use omicron_common::FileKv;
 use sled_hardware::underlay;
 use sled_hardware::DendriteAsic;
 use sled_hardware::SledMode;
+use sled_hardware_types::underlay::BootstrapInterface;
 use slog::Drain;
 use slog::Logger;
 use std::net::IpAddr;
@@ -344,7 +345,7 @@ pub(crate) struct BootstrapNetworking {
 impl BootstrapNetworking {
     fn setup(config: &Config) -> Result<Self, StartError> {
         let link_for_mac = config.get_link().map_err(StartError::ConfigLink)?;
-        let global_zone_bootstrap_ip = underlay::BootstrapInterface::GlobalZone
+        let global_zone_bootstrap_ip = BootstrapInterface::GlobalZone
             .ip(&link_for_mac)
             .map_err(StartError::BootstrapLinkMac)?;
 
@@ -381,7 +382,7 @@ impl BootstrapNetworking {
                 IpAddr::V6(addr) => addr,
             };
 
-        let switch_zone_bootstrap_ip = underlay::BootstrapInterface::SwitchZone
+        let switch_zone_bootstrap_ip = BootstrapInterface::SwitchZone
             .ip(&link_for_mac)
             .map_err(StartError::BootstrapLinkMac)?;
 
diff --git a/sled-agent/src/hardware_monitor.rs b/sled-agent/src/hardware_monitor.rs
index 5dcd060f98..34e9d8c9c7 100644
--- a/sled-agent/src/hardware_monitor.rs
+++ b/sled-agent/src/hardware_monitor.rs
@@ -8,7 +8,8 @@
 
 use crate::services::ServiceManager;
 use crate::sled_agent::SledAgent;
-use sled_hardware::{Baseboard, HardwareManager, HardwareUpdate};
+use sled_hardware::{HardwareManager, HardwareUpdate};
+use sled_hardware_types::Baseboard;
 use sled_storage::disk::RawDisk;
 use sled_storage::manager::StorageHandle;
 use slog::Logger;
diff --git a/sled-agent/src/metrics.rs b/sled-agent/src/metrics.rs
index a9d5acfff8..c41037fead 100644
--- a/sled-agent/src/metrics.rs
+++ b/sled-agent/src/metrics.rs
@@ -6,7 +6,7 @@
 
 use oximeter::types::MetricsError;
 use oximeter::types::ProducerRegistry;
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use slog::Logger;
 use std::sync::Arc;
 use std::time::Duration;
diff --git a/sled-agent/src/nexus.rs b/sled-agent/src/nexus.rs
index 7e7d60f6a4..59ae797ab4 100644
--- a/sled-agent/src/nexus.rs
+++ b/sled-agent/src/nexus.rs
@@ -140,7 +140,9 @@ impl ConvertInto<nexus_client::types::PhysicalDiskKind>
     }
 }
 
-impl ConvertInto<nexus_client::types::Baseboard> for sled_hardware::Baseboard {
+impl ConvertInto<nexus_client::types::Baseboard>
+    for sled_hardware_types::Baseboard
+{
     fn convert(self) -> nexus_client::types::Baseboard {
         nexus_client::types::Baseboard {
             serial: self.identifier().to_string(),
@@ -644,7 +646,7 @@ mod test {
         ByteCount, Error, Generation, LookupType, MessagePair, ResourceType,
     };
     use omicron_test_utils::dev::test_setup_log;
-    use sled_hardware::Baseboard;
+    use sled_hardware_types::Baseboard;
 
     /// Pretend to be CRDB storing info about a sled-agent
     #[derive(Default, Clone)]
diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs
index f30c910efc..82813d3f1e 100644
--- a/sled-agent/src/params.rs
+++ b/sled-agent/src/params.rs
@@ -21,8 +21,8 @@ use omicron_common::api::internal::shared::{
 };
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
-use sled_hardware::Baseboard;
 pub use sled_hardware::DendriteAsic;
+use sled_hardware_types::Baseboard;
 use sled_storage::dataset::DatasetKind;
 use sled_storage::dataset::DatasetName;
 use std::collections::BTreeSet;
diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs
index 2788e189cc..453495a870 100644
--- a/sled-agent/src/rack_setup/service.rs
+++ b/sled-agent/src/rack_setup/service.rs
@@ -103,7 +103,7 @@ use serde::{Deserialize, Serialize};
 use sled_agent_client::{
     types as SledAgentTypes, Client as SledAgentClient, Error as SledAgentError,
 };
-use sled_hardware::underlay::BootstrapInterface;
+use sled_hardware_types::underlay::BootstrapInterface;
 use sled_storage::dataset::CONFIG_DATASET;
 use sled_storage::manager::StorageHandle;
 use slog::Logger;
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
index 2be4b8ec45..11ccfad69b 100644
--- a/sled-agent/src/services.rs
+++ b/sled-agent/src/services.rs
@@ -91,9 +91,9 @@ use once_cell::sync::OnceCell;
 use rand::prelude::SliceRandom;
 use sled_hardware::is_gimlet;
 use sled_hardware::underlay;
-use sled_hardware::underlay::BOOTSTRAP_PREFIX;
-use sled_hardware::Baseboard;
 use sled_hardware::SledMode;
+use sled_hardware_types::underlay::BOOTSTRAP_PREFIX;
+use sled_hardware_types::Baseboard;
 use sled_storage::dataset::{
     DatasetKind, DatasetName, CONFIG_DATASET, INSTALL_DATASET, ZONE_DATASET,
 };
diff --git a/sled-agent/src/sim/config.rs b/sled-agent/src/sim/config.rs
index 7a20dd5709..8da6f8773f 100644
--- a/sled-agent/src/sim/config.rs
+++ b/sled-agent/src/sim/config.rs
@@ -9,7 +9,7 @@ use camino::Utf8Path;
 use dropshot::ConfigDropshot;
 use serde::Deserialize;
 use serde::Serialize;
-pub use sled_hardware::Baseboard;
+pub use sled_hardware_types::Baseboard;
 use std::net::Ipv6Addr;
 use std::net::{IpAddr, SocketAddr};
 use uuid::Uuid;
diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs
index 4a21a6fe89..cff13dd43a 100644
--- a/sled-agent/src/sled_agent.rs
+++ b/sled-agent/src/sled_agent.rs
@@ -66,7 +66,9 @@ use omicron_common::backoff::{
     retry_policy_internal_service_aggressive, BackoffError,
 };
 use oximeter::types::ProducerRegistry;
-use sled_hardware::{underlay, Baseboard, HardwareManager};
+use sled_hardware::{underlay, HardwareManager};
+use sled_hardware_types::underlay::BootstrapInterface;
+use sled_hardware_types::Baseboard;
 use sled_storage::manager::StorageHandle;
 use slog::Logger;
 use std::collections::BTreeMap;
@@ -1173,9 +1175,7 @@ pub async fn sled_add(
     // Get all known bootstrap addresses via DDM
     let ddm_admin_client = DdmAdminClient::localhost(&log)?;
     let addrs = ddm_admin_client
-        .derive_bootstrap_addrs_from_prefixes(&[
-            underlay::BootstrapInterface::GlobalZone,
-        ])
+        .derive_bootstrap_addrs_from_prefixes(&[BootstrapInterface::GlobalZone])
         .await?;
 
     // Create a set of futures to concurrently map the baseboard to bootstrap ip
diff --git a/sled-hardware/Cargo.toml b/sled-hardware/Cargo.toml
index 28cec20041..1c914e2897 100644
--- a/sled-hardware/Cargo.toml
+++ b/sled-hardware/Cargo.toml
@@ -18,6 +18,7 @@ omicron-common.workspace = true
 rand.workspace = true
 schemars.workspace = true
 serde.workspace = true
+sled-hardware-types.workspace = true
 slog.workspace = true
 slog-error-chain.workspace = true
 thiserror.workspace = true
diff --git a/sled-hardware/src/illumos/mod.rs b/sled-hardware/src/illumos/mod.rs
index f2db424bda..7dd6f9e20d 100644
--- a/sled-hardware/src/illumos/mod.rs
+++ b/sled-hardware/src/illumos/mod.rs
@@ -3,13 +3,13 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 use crate::{
-    Baseboard, DendriteAsic, DiskVariant, HardwareUpdate, SledMode,
-    UnparsedDisk,
+    DendriteAsic, DiskVariant, HardwareUpdate, SledMode, UnparsedDisk,
 };
 use camino::Utf8PathBuf;
 use gethostname::gethostname;
 use illumos_devinfo::{DevInfo, DevLinkType, DevLinks, Node, Property};
 use omicron_common::disk::DiskIdentity;
+use sled_hardware_types::Baseboard;
 use slog::debug;
 use slog::error;
 use slog::info;
diff --git a/sled-hardware/src/lib.rs b/sled-hardware/src/lib.rs
index 2e3fd4a576..607f72e25c 100644
--- a/sled-hardware/src/lib.rs
+++ b/sled-hardware/src/lib.rs
@@ -74,92 +74,3 @@ pub enum SledMode {
     /// Force sled to run as a Scrimlet
     Scrimlet { asic: DendriteAsic },
 }
-
-/// Describes properties that should uniquely identify a Gimlet.
-#[derive(
-    Clone,
-    Debug,
-    PartialOrd,
-    Ord,
-    PartialEq,
-    Eq,
-    Hash,
-    Serialize,
-    Deserialize,
-    JsonSchema,
-)]
-#[serde(tag = "type", rename_all = "snake_case")]
-pub enum Baseboard {
-    Gimlet { identifier: String, model: String, revision: i64 },
-
-    Unknown,
-
-    Pc { identifier: String, model: String },
-}
-
-impl Baseboard {
-    #[allow(dead_code)]
-    pub fn new_gimlet(
-        identifier: String,
-        model: String,
-        revision: i64,
-    ) -> Self {
-        Self::Gimlet { identifier, model, revision }
-    }
-
-    pub fn new_pc(identifier: String, model: String) -> Self {
-        Self::Pc { identifier, model }
-    }
-
-    // XXX This should be removed, but it requires a refactor in how devices are
-    // polled.
-    pub fn unknown() -> Self {
-        Self::Unknown
-    }
-
-    pub fn type_string(&self) -> &str {
-        match &self {
-            Self::Gimlet { .. } => "gimlet",
-            Self::Pc { .. } => "pc",
-            Self::Unknown => "unknown",
-        }
-    }
-
-    pub fn identifier(&self) -> &str {
-        match &self {
-            Self::Gimlet { identifier, .. } => &identifier,
-            Self::Pc { identifier, .. } => &identifier,
-            Self::Unknown => "unknown",
-        }
-    }
-
-    pub fn model(&self) -> &str {
-        match self {
-            Self::Gimlet { model, .. } => &model,
-            Self::Pc { model, .. } => &model,
-            Self::Unknown => "unknown",
-        }
-    }
-
-    pub fn revision(&self) -> i64 {
-        match self {
-            Self::Gimlet { revision, .. } => *revision,
-            Self::Pc { .. } => 0,
-            Self::Unknown => 0,
-        }
-    }
-}
-
-impl std::fmt::Display for Baseboard {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Baseboard::Gimlet { identifier, model, revision } => {
-                write!(f, "gimlet-{identifier}-{model}-{revision}")
-            }
-            Baseboard::Unknown => write!(f, "unknown"),
-            Baseboard::Pc { identifier, model } => {
-                write!(f, "pc-{identifier}-{model}")
-            }
-        }
-    }
-}
diff --git a/sled-hardware/src/non_illumos/mod.rs b/sled-hardware/src/non_illumos/mod.rs
index 8518aae495..e990567b7c 100644
--- a/sled-hardware/src/non_illumos/mod.rs
+++ b/sled-hardware/src/non_illumos/mod.rs
@@ -5,8 +5,9 @@
 use crate::disk::{
     DiskPaths, DiskVariant, Partition, PooledDiskError, UnparsedDisk,
 };
-use crate::{Baseboard, SledMode};
+use crate::SledMode;
 use omicron_common::disk::DiskIdentity;
+use sled_hardware_types::Baseboard;
 use slog::Logger;
 use std::collections::HashSet;
 use tokio::sync::broadcast;
diff --git a/sled-hardware/src/underlay.rs b/sled-hardware/src/underlay.rs
index 48d1dd2d64..e0b40f443b 100644
--- a/sled-hardware/src/underlay.rs
+++ b/sled-hardware/src/underlay.rs
@@ -7,7 +7,6 @@
 use crate::is_gimlet;
 use illumos_utils::addrobj;
 use illumos_utils::addrobj::AddrObject;
-use illumos_utils::dladm;
 use illumos_utils::dladm::Dladm;
 use illumos_utils::dladm::FindPhysicalLinkError;
 use illumos_utils::dladm::GetLinkpropError;
@@ -15,14 +14,6 @@ use illumos_utils::dladm::PhysicalLink;
 use illumos_utils::dladm::SetLinkpropError;
 use illumos_utils::dladm::CHELSIO_LINK_PREFIX;
 use illumos_utils::zone::Zones;
-use omicron_common::api::external::MacAddr;
-use std::net::Ipv6Addr;
-
-/// Initial octet of IPv6 for bootstrap addresses.
-pub const BOOTSTRAP_PREFIX: u16 = 0xfdb0;
-
-/// IPv6 prefix mask for bootstrap addresses.
-pub const BOOTSTRAP_MASK: u8 = 64;
 
 #[derive(thiserror::Error, Debug)]
 pub enum Error {
@@ -107,60 +98,3 @@ pub fn ensure_links_have_global_zone_link_local_v6_addresses(
 
     Ok(addr_objs)
 }
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum BootstrapInterface {
-    GlobalZone,
-    SwitchZone,
-}
-
-impl BootstrapInterface {
-    pub fn interface_id(self) -> u64 {
-        match self {
-            BootstrapInterface::GlobalZone => 1,
-            BootstrapInterface::SwitchZone => 2,
-        }
-    }
-
-    // TODO(https://github.com/oxidecomputer/omicron/issues/945): This address
-    // could be randomly generated when it no longer needs to be durable.
-    pub fn ip(
-        self,
-        link: &PhysicalLink,
-    ) -> Result<Ipv6Addr, dladm::GetMacError> {
-        let mac = Dladm::get_mac(link)?;
-        Ok(mac_to_bootstrap_ip(mac, self.interface_id()))
-    }
-}
-
-fn mac_to_bootstrap_ip(mac: MacAddr, interface_id: u64) -> Ipv6Addr {
-    let mac_bytes = mac.into_array();
-    assert_eq!(6, mac_bytes.len());
-
-    Ipv6Addr::new(
-        BOOTSTRAP_PREFIX,
-        ((mac_bytes[0] as u16) << 8) | mac_bytes[1] as u16,
-        ((mac_bytes[2] as u16) << 8) | mac_bytes[3] as u16,
-        ((mac_bytes[4] as u16) << 8) | mac_bytes[5] as u16,
-        (interface_id >> 48 & 0xffff).try_into().unwrap(),
-        (interface_id >> 32 & 0xffff).try_into().unwrap(),
-        (interface_id >> 16 & 0xffff).try_into().unwrap(),
-        (interface_id & 0xfff).try_into().unwrap(),
-    )
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use macaddr::MacAddr6;
-
-    #[test]
-    fn test_mac_to_bootstrap_ip() {
-        let mac = MacAddr("a8:40:25:10:00:01".parse::<MacAddr6>().unwrap());
-
-        assert_eq!(
-            mac_to_bootstrap_ip(mac, 1),
-            "fdb0:a840:2510:1::1".parse::<Ipv6Addr>().unwrap(),
-        );
-    }
-}
diff --git a/sled-hardware/types/Cargo.toml b/sled-hardware/types/Cargo.toml
new file mode 100644
index 0000000000..d6eefa49ac
--- /dev/null
+++ b/sled-hardware/types/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+name = "sled-hardware-types"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+
+[dependencies]
+illumos-utils.workspace = true
+omicron-common.workspace = true
+schemars.workspace = true
+serde.workspace = true
+omicron-workspace-hack.workspace = true
+
+[dev-dependencies]
+macaddr.workspace = true
diff --git a/sled-hardware/types/src/lib.rs b/sled-hardware/types/src/lib.rs
new file mode 100644
index 0000000000..e589498ff8
--- /dev/null
+++ b/sled-hardware/types/src/lib.rs
@@ -0,0 +1,97 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+pub mod underlay;
+
+/// Describes properties that should uniquely identify a Gimlet.
+#[derive(
+    Clone,
+    Debug,
+    PartialOrd,
+    Ord,
+    PartialEq,
+    Eq,
+    Hash,
+    Serialize,
+    Deserialize,
+    JsonSchema,
+)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum Baseboard {
+    Gimlet { identifier: String, model: String, revision: i64 },
+
+    Unknown,
+
+    Pc { identifier: String, model: String },
+}
+
+impl Baseboard {
+    #[allow(dead_code)]
+    pub fn new_gimlet(
+        identifier: String,
+        model: String,
+        revision: i64,
+    ) -> Self {
+        Self::Gimlet { identifier, model, revision }
+    }
+
+    pub fn new_pc(identifier: String, model: String) -> Self {
+        Self::Pc { identifier, model }
+    }
+
+    // XXX This should be removed, but it requires a refactor in how devices are
+    // polled.
+    pub fn unknown() -> Self {
+        Self::Unknown
+    }
+
+    pub fn type_string(&self) -> &str {
+        match &self {
+            Self::Gimlet { .. } => "gimlet",
+            Self::Pc { .. } => "pc",
+            Self::Unknown => "unknown",
+        }
+    }
+
+    pub fn identifier(&self) -> &str {
+        match &self {
+            Self::Gimlet { identifier, .. } => &identifier,
+            Self::Pc { identifier, .. } => &identifier,
+            Self::Unknown => "unknown",
+        }
+    }
+
+    pub fn model(&self) -> &str {
+        match self {
+            Self::Gimlet { model, .. } => &model,
+            Self::Pc { model, .. } => &model,
+            Self::Unknown => "unknown",
+        }
+    }
+
+    pub fn revision(&self) -> i64 {
+        match self {
+            Self::Gimlet { revision, .. } => *revision,
+            Self::Pc { .. } => 0,
+            Self::Unknown => 0,
+        }
+    }
+}
+
+impl std::fmt::Display for Baseboard {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Baseboard::Gimlet { identifier, model, revision } => {
+                write!(f, "gimlet-{identifier}-{model}-{revision}")
+            }
+            Baseboard::Unknown => write!(f, "unknown"),
+            Baseboard::Pc { identifier, model } => {
+                write!(f, "pc-{identifier}-{model}")
+            }
+        }
+    }
+}
diff --git a/sled-hardware/types/src/underlay.rs b/sled-hardware/types/src/underlay.rs
new file mode 100644
index 0000000000..bbeb43bd4d
--- /dev/null
+++ b/sled-hardware/types/src/underlay.rs
@@ -0,0 +1,72 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use illumos_utils::dladm;
+use illumos_utils::dladm::Dladm;
+use illumos_utils::dladm::PhysicalLink;
+use omicron_common::api::external::MacAddr;
+use std::net::Ipv6Addr;
+
+/// Initial octet of IPv6 for bootstrap addresses.
+pub const BOOTSTRAP_PREFIX: u16 = 0xfdb0;
+
+/// IPv6 prefix mask for bootstrap addresses.
+pub const BOOTSTRAP_MASK: u8 = 64;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BootstrapInterface {
+    GlobalZone,
+    SwitchZone,
+}
+
+impl BootstrapInterface {
+    pub fn interface_id(self) -> u64 {
+        match self {
+            BootstrapInterface::GlobalZone => 1,
+            BootstrapInterface::SwitchZone => 2,
+        }
+    }
+
+    // TODO(https://github.com/oxidecomputer/omicron/issues/945): This address
+    // could be randomly generated when it no longer needs to be durable.
+    pub fn ip(
+        self,
+        link: &PhysicalLink,
+    ) -> Result<Ipv6Addr, dladm::GetMacError> {
+        let mac = Dladm::get_mac(link)?;
+        Ok(mac_to_bootstrap_ip(mac, self.interface_id()))
+    }
+}
+
+fn mac_to_bootstrap_ip(mac: MacAddr, interface_id: u64) -> Ipv6Addr {
+    let mac_bytes = mac.into_array();
+    assert_eq!(6, mac_bytes.len());
+
+    Ipv6Addr::new(
+        BOOTSTRAP_PREFIX,
+        ((mac_bytes[0] as u16) << 8) | mac_bytes[1] as u16,
+        ((mac_bytes[2] as u16) << 8) | mac_bytes[3] as u16,
+        ((mac_bytes[4] as u16) << 8) | mac_bytes[5] as u16,
+        (interface_id >> 48 & 0xffff).try_into().unwrap(),
+        (interface_id >> 32 & 0xffff).try_into().unwrap(),
+        (interface_id >> 16 & 0xffff).try_into().unwrap(),
+        (interface_id & 0xfff).try_into().unwrap(),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use macaddr::MacAddr6;
+
+    #[test]
+    fn test_mac_to_bootstrap_ip() {
+        let mac = MacAddr("a8:40:25:10:00:01".parse::<MacAddr6>().unwrap());
+
+        assert_eq!(
+            mac_to_bootstrap_ip(mac, 1),
+            "fdb0:a840:2510:1::1".parse::<Ipv6Addr>().unwrap(),
+        );
+    }
+}
diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml
index 26e54eb3bc..4b6e6d6f5f 100644
--- a/wicketd/Cargo.toml
+++ b/wicketd/Cargo.toml
@@ -53,7 +53,7 @@ installinator-common.workspace = true
 omicron-certificates.workspace = true
 omicron-common.workspace = true
 omicron-passwords.workspace = true
-sled-hardware.workspace = true
+sled-hardware-types.workspace = true
 tufaceous-lib.workspace = true
 update-common.workspace = true
 update-engine.workspace = true
diff --git a/wicketd/src/bin/wicketd.rs b/wicketd/src/bin/wicketd.rs
index 24fa802c79..4037bc4c23 100644
--- a/wicketd/src/bin/wicketd.rs
+++ b/wicketd/src/bin/wicketd.rs
@@ -11,7 +11,7 @@ use omicron_common::{
     address::Ipv6Subnet,
     cmd::{fatal, CmdError},
 };
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use std::net::{Ipv6Addr, SocketAddrV6};
 use std::path::PathBuf;
 use wicketd::{self, run_openapi, Config, Server, SmfConfigValues};
diff --git a/wicketd/src/bootstrap_addrs.rs b/wicketd/src/bootstrap_addrs.rs
index f49e5f8ba8..f7256ae6ac 100644
--- a/wicketd/src/bootstrap_addrs.rs
+++ b/wicketd/src/bootstrap_addrs.rs
@@ -4,8 +4,8 @@
 
 use ddm_admin_client::Client as DdmAdminClient;
 use futures::stream::FuturesUnordered;
-use sled_hardware::underlay::BootstrapInterface;
-use sled_hardware::Baseboard;
+use sled_hardware_types::underlay::BootstrapInterface;
+use sled_hardware_types::Baseboard;
 use slog::warn;
 use slog::Logger;
 use std::collections::BTreeMap;
diff --git a/wicketd/src/context.rs b/wicketd/src/context.rs
index eeecc3fa64..68d04f35dc 100644
--- a/wicketd/src/context.rs
+++ b/wicketd/src/context.rs
@@ -14,7 +14,7 @@ use anyhow::bail;
 use anyhow::Result;
 use gateway_client::types::SpIdentifier;
 use internal_dns::resolver::Resolver;
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use slog::info;
 use std::net::Ipv6Addr;
 use std::net::SocketAddrV6;
diff --git a/wicketd/src/http_entrypoints.rs b/wicketd/src/http_entrypoints.rs
index 9748a93bd5..e87581b7be 100644
--- a/wicketd/src/http_entrypoints.rs
+++ b/wicketd/src/http_entrypoints.rs
@@ -38,7 +38,7 @@ use omicron_common::update::ArtifactId;
 use schemars::JsonSchema;
 use serde::Deserialize;
 use serde::Serialize;
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use slog::o;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
diff --git a/wicketd/src/lib.rs b/wicketd/src/lib.rs
index 32188d77de..5926fc468d 100644
--- a/wicketd/src/lib.rs
+++ b/wicketd/src/lib.rs
@@ -32,7 +32,7 @@ use nexus_proxy::NexusTcpProxy;
 use omicron_common::address::{Ipv6Subnet, AZ_PREFIX};
 use omicron_common::FileKv;
 use preflight_check::PreflightCheckerHandler;
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use slog::{debug, error, o, Drain};
 use std::sync::{Mutex, OnceLock};
 use std::time::Duration;
diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs
index 4bc1a6b62b..3fc123480d 100644
--- a/wicketd/src/rss_config.rs
+++ b/wicketd/src/rss_config.rs
@@ -26,7 +26,7 @@ use gateway_client::types::SpType;
 use omicron_certificates::CertificateError;
 use omicron_common::address;
 use omicron_common::address::Ipv4Range;
-use sled_hardware::Baseboard;
+use sled_hardware_types::Baseboard;
 use slog::warn;
 use std::collections::BTreeSet;
 use std::mem;

From 6d44cf529c8821e7fc107fe3b15bc2377ba96872 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Wed, 13 Mar 2024 06:29:25 +0000
Subject: [PATCH 102/157] fix(deps): update rust crate vsss-rs to 3.4.0 (#5115)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock           | 8 ++++----
 bootstore/Cargo.toml | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 0d10f6b675..edc352b2d6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1448,9 +1448,9 @@ dependencies = [
 
 [[package]]
 name = "curve25519-dalek"
-version = "4.1.1"
+version = "4.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e89b8c6a2e4b1f45971ad09761aafb85514a84744b67a95e32c3cc1352d1f65c"
+checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -10293,9 +10293,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "vsss-rs"
-version = "3.3.4"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "196bbee60607a195bc850e94f0e040bd090e45794ad8df0e9c5a422b9975a00f"
+checksum = "14d9b5a415d1bbbf1a9c5982babc0d8054fa10effe9c5b7086cd2fdd36be79d9"
 dependencies = [
  "curve25519-dalek",
  "elliptic-curve",
diff --git a/bootstore/Cargo.toml b/bootstore/Cargo.toml
index 5e9bcd1ef4..97db1d52b7 100644
--- a/bootstore/Cargo.toml
+++ b/bootstore/Cargo.toml
@@ -27,7 +27,7 @@ slog.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
 uuid.workspace = true
-vsss-rs = { version = "3.3.4", features = ["std", "curve25519"] }
+vsss-rs = { version = "3.4.0", features = ["std", "curve25519"] }
 zeroize.workspace = true
 
 # See omicron-rpaths for more about the "pq-sys" dependency.

From 2317dfe3dd24e72f58c8d6da0f758f292e397a8f Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Wed, 13 Mar 2024 00:36:50 -0700
Subject: [PATCH 103/157] chore(deps): update rust crate base64 to 0.22.0
 (#5217)

---
 Cargo.lock | 52 +++++++++++++++++++++++++++++-----------------------
 Cargo.toml |  2 +-
 2 files changed, 30 insertions(+), 24 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index edc352b2d6..1b19e61eb0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -411,6 +411,12 @@ version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
+[[package]]
+name = "base64"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51"
+
 [[package]]
 name = "base64ct"
 version = "1.6.0"
@@ -2020,7 +2026,7 @@ source = "git+https://github.com/oxidecomputer/dropshot?branch=main#29ae98d1f909
 dependencies = [
  "async-stream",
  "async-trait",
- "base64",
+ "base64 0.21.7",
  "bytes",
  "camino",
  "chrono",
@@ -2210,7 +2216,7 @@ dependencies = [
  "anstyle",
  "anyhow",
  "async-trait",
- "base64",
+ "base64 0.22.0",
  "chrono",
  "clap 4.5.1",
  "colored",
@@ -2642,7 +2648,7 @@ dependencies = [
 name = "gateway-client"
 version = "0.1.0"
 dependencies = [
- "base64",
+ "base64 0.22.0",
  "chrono",
  "gateway-messages",
  "omicron-workspace-hack",
@@ -2941,7 +2947,7 @@ version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "bytes",
  "headers-core",
  "http 0.2.12",
@@ -4441,7 +4447,7 @@ dependencies = [
  "async-bb8-diesel",
  "async-trait",
  "authz-macros",
- "base64",
+ "base64 0.22.0",
  "bb8",
  "camino",
  "camino-tempfile",
@@ -4530,7 +4536,7 @@ name = "nexus-inventory"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "base64",
+ "base64 0.22.0",
  "chrono",
  "expectorate",
  "futures",
@@ -4718,7 +4724,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "api_identity",
- "base64",
+ "base64 0.22.0",
  "chrono",
  "dns-service-client",
  "futures",
@@ -5105,7 +5111,7 @@ name = "omicron-gateway"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "base64",
+ "base64 0.22.0",
  "camino",
  "clap 4.5.1",
  "dropshot",
@@ -5151,7 +5157,7 @@ dependencies = [
  "assert_matches",
  "async-bb8-diesel",
  "async-trait",
- "base64",
+ "base64 0.22.0",
  "buf-list",
  "bytes",
  "camino",
@@ -5383,7 +5389,7 @@ dependencies = [
  "anyhow",
  "assert_matches",
  "async-trait",
- "base64",
+ "base64 0.22.0",
  "bootstore",
  "bootstrap-agent-client",
  "bytes",
@@ -5833,7 +5839,7 @@ name = "oxide-client"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "base64",
+ "base64 0.22.0",
  "chrono",
  "futures",
  "http 0.2.12",
@@ -6276,7 +6282,7 @@ version = "3.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3163d2912b7c3b52d651a055f2c7eec9ba5cd22d26ef75b8dd3a59980b185923"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "serde",
 ]
 
@@ -6545,7 +6551,7 @@ version = "0.6.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "byteorder",
  "bytes",
  "fallible-iterator",
@@ -6782,7 +6788,7 @@ version = "0.1.0"
 source = "git+https://github.com/oxidecomputer/propolis?rev=a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c#a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c"
 dependencies = [
  "async-trait",
- "base64",
+ "base64 0.21.7",
  "futures",
  "progenitor",
  "rand 0.8.5",
@@ -6804,7 +6810,7 @@ source = "git+https://github.com/oxidecomputer/propolis?rev=a9fdbc464db4cbd4ac0f
 dependencies = [
  "anyhow",
  "atty",
- "base64",
+ "base64 0.21.7",
  "clap 4.5.1",
  "dropshot",
  "futures",
@@ -7272,7 +7278,7 @@ version = "0.11.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "bytes",
  "cookie 0.17.0",
  "cookie_store",
@@ -7370,7 +7376,7 @@ version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "bitflags 2.4.2",
  "serde",
  "serde_derive",
@@ -7540,7 +7546,7 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a5885493fdf0be6cdff808d1533ce878d21cfa49c7086fa00c66355cd9141bfc"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "blake2b_simd",
  "constant_time_eq 0.3.0",
  "crossbeam-utils",
@@ -7647,7 +7653,7 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2"
 dependencies = [
- "base64",
+ "base64 0.21.7",
 ]
 
 [[package]]
@@ -7656,7 +7662,7 @@ version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "rustls-pki-types",
 ]
 
@@ -7767,7 +7773,7 @@ name = "samael"
 version = "0.0.14"
 source = "git+https://github.com/oxidecomputer/samael?branch=oxide/omicron#9e609a8f6fa0dd84e3bb8f579f46bd780c8be62b"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "bindgen",
  "chrono",
  "data-encoding",
@@ -8117,7 +8123,7 @@ version = "3.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "15d167997bd841ec232f5b2b8e0e26606df2e7caa4c31b95ea9ca52b200bd270"
 dependencies = [
- "base64",
+ "base64 0.21.7",
  "chrono",
  "hex",
  "indexmap 1.9.3",
@@ -10590,7 +10596,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
- "base64",
+ "base64 0.22.0",
  "bootstrap-agent-client",
  "buf-list",
  "bytes",
diff --git a/Cargo.toml b/Cargo.toml
index 733eee1b32..f97c6e6b27 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -172,7 +172,7 @@ async-trait = "0.1.77"
 atomicwrites = "0.4.3"
 authz-macros = { path = "nexus/authz-macros" }
 backoff = { version = "0.4.0", features = [ "tokio" ] }
-base64 = "0.21.7"
+base64 = "0.22.0"
 bb8 = "0.8.3"
 bcs = "0.1.6"
 bincode = "1.3.3"

From fd769f8896ce8e62e8fff6e7f8dec6260de65264 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Wed, 13 Mar 2024 00:39:03 -0700
Subject: [PATCH 104/157] chore(deps): update rust crate regress to 0.9.0
 (#5160)

---
 Cargo.lock | 30 ++++++++++++++++++++----------
 Cargo.toml |  2 +-
 2 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 1b19e61eb0..b40855b336 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -662,7 +662,7 @@ dependencies = [
  "omicron-common",
  "omicron-workspace-hack",
  "progenitor",
- "regress",
+ "regress 0.9.0",
  "reqwest",
  "schemars",
  "serde",
@@ -2008,7 +2008,7 @@ dependencies = [
  "progenitor-client",
  "quote",
  "rand 0.8.5",
- "regress",
+ "regress 0.9.0",
  "reqwest",
  "rustfmt-wrapper",
  "schemars",
@@ -3449,7 +3449,7 @@ dependencies = [
  "opte-ioctl",
  "oxide-vpc",
  "oxlog",
- "regress",
+ "regress 0.9.0",
  "schemars",
  "serde",
  "serde_json",
@@ -3586,7 +3586,7 @@ dependencies = [
  "installinator-common",
  "omicron-workspace-hack",
  "progenitor",
- "regress",
+ "regress 0.9.0",
  "reqwest",
  "schemars",
  "serde",
@@ -4371,7 +4371,7 @@ dependencies = [
  "omicron-passwords",
  "omicron-workspace-hack",
  "progenitor",
- "regress",
+ "regress 0.9.0",
  "reqwest",
  "schemars",
  "serde",
@@ -5055,7 +5055,7 @@ dependencies = [
  "progenitor",
  "proptest",
  "rand 0.8.5",
- "regress",
+ "regress 0.9.0",
  "reqwest",
  "schemars",
  "semver 1.0.22",
@@ -5847,7 +5847,7 @@ dependencies = [
  "omicron-workspace-hack",
  "progenitor",
  "rand 0.8.5",
- "regress",
+ "regress 0.9.0",
  "reqwest",
  "serde",
  "serde_json",
@@ -7257,6 +7257,16 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "regress"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d06f9a1f7cd8473611ba1a480cf35f9c5cffc2954336ba90a982fdb7e7d7f51e"
+dependencies = [
+ "hashbrown 0.14.3",
+ "memchr",
+]
+
 [[package]]
 name = "relative-path"
 version = "1.9.0"
@@ -8324,7 +8334,7 @@ dependencies = [
  "omicron-common",
  "omicron-workspace-hack",
  "progenitor",
- "regress",
+ "regress 0.9.0",
  "reqwest",
  "schemars",
  "serde",
@@ -9932,7 +9942,7 @@ dependencies = [
  "log",
  "proc-macro2",
  "quote",
- "regress",
+ "regress 0.8.0",
  "schemars",
  "serde_json",
  "syn 2.0.51",
@@ -10674,7 +10684,7 @@ dependencies = [
  "ipnetwork",
  "omicron-workspace-hack",
  "progenitor",
- "regress",
+ "regress 0.9.0",
  "reqwest",
  "schemars",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index f97c6e6b27..0b2e5b58b2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -345,7 +345,7 @@ rcgen = "0.12.1"
 reedline = "0.30.0"
 ref-cast = "1.0"
 regex = "1.10.3"
-regress = "0.8.0"
+regress = "0.9.0"
 reqwest = { version = "0.11", default-features = false }
 ring = "0.17.8"
 rpassword = "7.3.1"

From 13a28f06278f8c7e730e58126acb3d47a6879d86 Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Wed, 13 Mar 2024 08:51:55 +0000
Subject: [PATCH 105/157] chore(deps): update taiki-e/install-action digest to
 34c5068 (#5250)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [taiki-e/install-action](https://togithub.com/taiki-e/install-action)
| action | digest | [`4978b3c` ->
`34c5068`](https://togithub.com/taiki-e/install-action/compare/4978b3c...34c5068)
|

---

### Configuration

📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone
America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone
America/Los_Angeles.

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update
again.

---

- [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check
this box

---

This PR has been generated by [Renovate
Bot](https://togithub.com/renovatebot/renovate).

<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNy4yNDEuMCIsInVwZGF0ZWRJblZlciI6IjM3LjI0MS4wIiwidGFyZ2V0QnJhbmNoIjoibWFpbiJ9-->

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index 557cfedbd3..ae4622290d 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@4978b3c8549425a60eb2d68446d0b58cef63ad0a # v2
+        uses: taiki-e/install-action@34c5068afd12dd215738693ffa556632845e203d # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From 4f101be78f2e0f2b22e01609ffd47105e2305e94 Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Wed, 13 Mar 2024 11:44:27 -0400
Subject: [PATCH 106/157] Add internal API and omdb command to expunge a sled
 (#5234)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The omdb interface might be overly cautious; feedback welcome. You can
try this against `omicron-dev run-all`; it looks roughly like:

```
note: using Nexus URL http://[::1]:12221
note: using database URL postgresql://root@[::1]:42285/omicron?sslmode=disable
note: database schema version matches expected (39.0.0)
WARNING: sled b6d65341-167c-41df-9b5c-41cded99c229 IS PRESENT in the most recent inventory collection; are you sure you want to mark it expunged?
WARNING: This operation will PERMANENTLY and IRRECOVABLY mark sled b6d65341-167c-41df-9b5c-41cded99c229 (sim-b6d65341) expunged. To proceed, type the sled's serial number.
sled serial〉sim-b6d65341
expunged sled b6d65341-167c-41df-9b5c-41cded99c229 (previous policy: InService(Provisionable))
```

This handles the "internal" half of #5134
---
 Cargo.lock                                 |   1 +
 dev-tools/omdb/Cargo.toml                  |   1 +
 dev-tools/omdb/src/bin/omdb/db.rs          | 112 ++++++++------
 dev-tools/omdb/src/bin/omdb/main.rs        |  29 +++-
 dev-tools/omdb/src/bin/omdb/nexus.rs       | 171 +++++++++++++++++++--
 nexus/db-model/src/sled.rs                 |   4 +
 nexus/src/app/sled.rs                      |  18 +++
 nexus/src/internal_api/http_entrypoints.rs |  28 ++++
 nexus/types/src/external_api/params.rs     |   1 +
 openapi/nexus-internal.json                | 111 +++++++++++++
 10 files changed, 413 insertions(+), 63 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index b40855b336..255433250e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5311,6 +5311,7 @@ dependencies = [
  "oximeter-client",
  "pq-sys",
  "ratatui",
+ "reedline",
  "serde",
  "serde_json",
  "sled-agent-client",
diff --git a/dev-tools/omdb/Cargo.toml b/dev-tools/omdb/Cargo.toml
index 1b697561a9..813a4b9552 100644
--- a/dev-tools/omdb/Cargo.toml
+++ b/dev-tools/omdb/Cargo.toml
@@ -36,6 +36,7 @@ oximeter-client.workspace = true
 # See omicron-rpaths for more about the "pq-sys" dependency.
 pq-sys = "*"
 ratatui.workspace = true
+reedline.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 sled-agent-client.workspace = true
diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs
index 1b99a815ef..cc498d0ad1 100644
--- a/dev-tools/omdb/src/bin/omdb/db.rs
+++ b/dev-tools/omdb/src/bin/omdb/db.rs
@@ -148,9 +148,8 @@ impl Display for MaybeSledId {
 
 #[derive(Debug, Args)]
 pub struct DbArgs {
-    /// URL of the database SQL interface
-    #[clap(long, env("OMDB_DB_URL"))]
-    db_url: Option<PostgresConfigWithUrl>,
+    #[clap(flatten)]
+    db_url_opts: DbUrlOptions,
 
     #[clap(flatten)]
     fetch_opts: DbFetchOptions,
@@ -159,6 +158,71 @@ pub struct DbArgs {
     command: DbCommands,
 }
 
+#[derive(Debug, Args)]
+pub struct DbUrlOptions {
+    /// URL of the database SQL interface
+    #[clap(long, env("OMDB_DB_URL"))]
+    db_url: Option<PostgresConfigWithUrl>,
+}
+
+impl DbUrlOptions {
+    async fn resolve_pg_url(
+        &self,
+        omdb: &Omdb,
+        log: &slog::Logger,
+    ) -> anyhow::Result<PostgresConfigWithUrl> {
+        match &self.db_url {
+            Some(cli_or_env_url) => Ok(cli_or_env_url.clone()),
+            None => {
+                eprintln!(
+                    "note: database URL not specified.  Will search DNS."
+                );
+                eprintln!("note: (override with --db-url or OMDB_DB_URL)");
+                let addrs = omdb
+                    .dns_lookup_all(
+                        log.clone(),
+                        internal_dns::ServiceName::Cockroach,
+                    )
+                    .await?;
+
+                format!(
+                    "postgresql://root@{}/omicron?sslmode=disable",
+                    addrs
+                        .into_iter()
+                        .map(|a| a.to_string())
+                        .collect::<Vec<_>>()
+                        .join(",")
+                )
+                .parse()
+                .context("failed to parse constructed postgres URL")
+            }
+        }
+    }
+
+    pub async fn connect(
+        &self,
+        omdb: &Omdb,
+        log: &slog::Logger,
+    ) -> anyhow::Result<Arc<DataStore>> {
+        let db_url = self.resolve_pg_url(omdb, log).await?;
+        eprintln!("note: using database URL {}", &db_url);
+
+        let db_config = db::Config { url: db_url.clone() };
+        let pool = Arc::new(db::Pool::new(&log.clone(), &db_config));
+
+        // Being a dev tool, we want to try this operation even if the schema
+        // doesn't match what we expect.  So we use `DataStore::new_unchecked()`
+        // here.  We will then check the schema version explicitly and warn the
+        // user if it doesn't match.
+        let datastore = Arc::new(
+            DataStore::new_unchecked(log.clone(), pool)
+                .map_err(|e| anyhow!(e).context("creating datastore"))?,
+        );
+        check_schema_version(&datastore).await;
+        Ok(datastore)
+    }
+}
+
 #[derive(Debug, Args)]
 pub struct DbFetchOptions {
     /// limit to apply to queries that fetch rows
@@ -405,47 +469,7 @@ impl DbArgs {
         omdb: &Omdb,
         log: &slog::Logger,
     ) -> Result<(), anyhow::Error> {
-        let db_url = match &self.db_url {
-            Some(cli_or_env_url) => cli_or_env_url.clone(),
-            None => {
-                eprintln!(
-                    "note: database URL not specified.  Will search DNS."
-                );
-                eprintln!("note: (override with --db-url or OMDB_DB_URL)");
-                let addrs = omdb
-                    .dns_lookup_all(
-                        log.clone(),
-                        internal_dns::ServiceName::Cockroach,
-                    )
-                    .await?;
-
-                format!(
-                    "postgresql://root@{}/omicron?sslmode=disable",
-                    addrs
-                        .into_iter()
-                        .map(|a| a.to_string())
-                        .collect::<Vec<_>>()
-                        .join(",")
-                )
-                .parse()
-                .context("failed to parse constructed postgres URL")?
-            }
-        };
-        eprintln!("note: using database URL {}", &db_url);
-
-        let db_config = db::Config { url: db_url.clone() };
-        let pool = Arc::new(db::Pool::new(&log.clone(), &db_config));
-
-        // Being a dev tool, we want to try this operation even if the schema
-        // doesn't match what we expect.  So we use `DataStore::new_unchecked()`
-        // here.  We will then check the schema version explicitly and warn the
-        // user if it doesn't match.
-        let datastore = Arc::new(
-            DataStore::new_unchecked(log.clone(), pool)
-                .map_err(|e| anyhow!(e).context("creating datastore"))?,
-        );
-        check_schema_version(&datastore).await;
-
+        let datastore = self.db_url_opts.connect(omdb, log).await?;
         let opctx = OpContext::for_tests(log.clone(), datastore.clone());
         match &self.command {
             DbCommands::Rack(RackArgs { command: RackCommands::List }) => {
diff --git a/dev-tools/omdb/src/bin/omdb/main.rs b/dev-tools/omdb/src/bin/omdb/main.rs
index 0fb9ba0121..17de22c2fa 100644
--- a/dev-tools/omdb/src/bin/omdb/main.rs
+++ b/dev-tools/omdb/src/bin/omdb/main.rs
@@ -92,16 +92,29 @@ struct Omdb {
     command: OmdbCommands,
 }
 
-impl Omdb {
-    fn check_allow_destructive(&self) -> anyhow::Result<()> {
-        anyhow::ensure!(
-            self.allow_destructive,
-            "This command is potentially destructive. \
-             Pass the `-w` / `--destructive` flag to allow it."
-        );
-        Ok(())
+mod check_allow_destructive {
+    /// Zero-size type that potentially-destructive functions can accept to
+    /// ensure `Omdb::check_allow_destructive` has been called.
+    // This is tucked away inside a module to prevent it from being constructed
+    // by anything other than `Omdb::check_allow_destructive`.
+    #[must_use]
+    pub(crate) struct DestructiveOperationToken(());
+
+    impl super::Omdb {
+        pub(crate) fn check_allow_destructive(
+            &self,
+        ) -> anyhow::Result<DestructiveOperationToken> {
+            anyhow::ensure!(
+                self.allow_destructive,
+                "This command is potentially destructive. \
+                 Pass the `-w` / `--destructive` flag to allow it."
+            );
+            Ok(DestructiveOperationToken(()))
+        }
     }
+}
 
+impl Omdb {
     async fn dns_lookup_all(
         &self,
         log: slog::Logger,
diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs
index 4705da2a32..ecf2a8072d 100644
--- a/dev-tools/omdb/src/bin/omdb/nexus.rs
+++ b/dev-tools/omdb/src/bin/omdb/nexus.rs
@@ -4,7 +4,10 @@
 
 //! omdb commands that query or update specific Nexus instances
 
+use crate::check_allow_destructive::DestructiveOperationToken;
+use crate::db::DbUrlOptions;
 use crate::Omdb;
+use anyhow::bail;
 use anyhow::Context;
 use chrono::DateTime;
 use chrono::SecondsFormat;
@@ -17,7 +20,13 @@ use nexus_client::types::ActivationReason;
 use nexus_client::types::BackgroundTask;
 use nexus_client::types::CurrentStatus;
 use nexus_client::types::LastResult;
+use nexus_client::types::SledSelector;
 use nexus_client::types::UninitializedSledId;
+use nexus_db_queries::db::lookup::LookupPath;
+use nexus_types::inventory::BaseboardId;
+use reedline::DefaultPrompt;
+use reedline::DefaultPromptSegment;
+use reedline::Reedline;
 use serde::Deserialize;
 use slog_error_chain::InlineErrorChain;
 use std::collections::BTreeMap;
@@ -36,6 +45,7 @@ pub struct NexusArgs {
 }
 
 /// Subcommands for the "omdb nexus" subcommand
+#[allow(clippy::large_enum_variant)]
 #[derive(Debug, Subcommand)]
 enum NexusCommands {
     /// print information about background tasks
@@ -145,11 +155,14 @@ struct SledsArgs {
 }
 
 #[derive(Debug, Subcommand)]
+#[allow(clippy::large_enum_variant)]
 enum SledsCommands {
     /// List all uninitialized sleds
     ListUninitialized,
     /// Add an uninitialized sled
     Add(SledAddArgs),
+    /// Expunge a sled (DANGEROUS)
+    Expunge(SledExpungeArgs),
 }
 
 #[derive(Debug, Args)]
@@ -160,6 +173,17 @@ struct SledAddArgs {
     part: String,
 }
 
+#[derive(Debug, Args)]
+struct SledExpungeArgs {
+    // expunge is _extremely_ dangerous, so we also require a database
+    // connection to perform some safety checks
+    #[clap(flatten)]
+    db_url_opts: DbUrlOptions,
+
+    /// sled ID
+    sled_id: Uuid,
+}
+
 impl NexusArgs {
     /// Run a `omdb nexus` subcommand.
     pub(crate) async fn run_cmd(
@@ -212,8 +236,8 @@ impl NexusArgs {
             NexusCommands::Blueprints(BlueprintsArgs {
                 command: BlueprintsCommands::Delete(args),
             }) => {
-                omdb.check_allow_destructive()?;
-                cmd_nexus_blueprints_delete(&client, args).await
+                let token = omdb.check_allow_destructive()?;
+                cmd_nexus_blueprints_delete(&client, args, token).await
             }
             NexusCommands::Blueprints(BlueprintsArgs {
                 command:
@@ -227,21 +251,23 @@ impl NexusArgs {
                         command: BlueprintTargetCommands::Set(args),
                     }),
             }) => {
-                omdb.check_allow_destructive()?;
-                cmd_nexus_blueprints_target_set(&client, args).await
+                let token = omdb.check_allow_destructive()?;
+                cmd_nexus_blueprints_target_set(&client, args, token).await
             }
             NexusCommands::Blueprints(BlueprintsArgs {
                 command: BlueprintsCommands::Regenerate,
             }) => {
-                omdb.check_allow_destructive()?;
-                cmd_nexus_blueprints_regenerate(&client).await
+                let token = omdb.check_allow_destructive()?;
+                cmd_nexus_blueprints_regenerate(&client, token).await
             }
             NexusCommands::Blueprints(BlueprintsArgs {
                 command: BlueprintsCommands::GenerateFromCollection(args),
             }) => {
-                omdb.check_allow_destructive()?;
-                cmd_nexus_blueprints_generate_from_collection(&client, args)
-                    .await
+                let token = omdb.check_allow_destructive()?;
+                cmd_nexus_blueprints_generate_from_collection(
+                    &client, args, token,
+                )
+                .await
             }
 
             NexusCommands::Sleds(SledsArgs {
@@ -250,8 +276,14 @@ impl NexusArgs {
             NexusCommands::Sleds(SledsArgs {
                 command: SledsCommands::Add(args),
             }) => {
-                omdb.check_allow_destructive()?;
-                cmd_nexus_sled_add(&client, args).await
+                let token = omdb.check_allow_destructive()?;
+                cmd_nexus_sled_add(&client, args, token).await
+            }
+            NexusCommands::Sleds(SledsArgs {
+                command: SledsCommands::Expunge(args),
+            }) => {
+                let token = omdb.check_allow_destructive()?;
+                cmd_nexus_sled_expunge(&client, args, omdb, log, token).await
             }
         }
     }
@@ -909,6 +941,7 @@ async fn cmd_nexus_blueprints_diff(
 async fn cmd_nexus_blueprints_delete(
     client: &nexus_client::Client,
     args: &BlueprintIdArgs,
+    _destruction_token: DestructiveOperationToken,
 ) -> Result<(), anyhow::Error> {
     let _ = client
         .blueprint_delete(&args.blueprint_id)
@@ -934,6 +967,7 @@ async fn cmd_nexus_blueprints_target_show(
 async fn cmd_nexus_blueprints_target_set(
     client: &nexus_client::Client,
     args: &BlueprintTargetSetArgs,
+    _destruction_token: DestructiveOperationToken,
 ) -> Result<(), anyhow::Error> {
     let enabled = match args.enabled {
         BlueprintTargetSetEnabled::Enabled => true,
@@ -969,6 +1003,7 @@ async fn cmd_nexus_blueprints_target_set(
 async fn cmd_nexus_blueprints_generate_from_collection(
     client: &nexus_client::Client,
     args: &CollectionIdArgs,
+    _destruction_token: DestructiveOperationToken,
 ) -> Result<(), anyhow::Error> {
     let blueprint = client
         .blueprint_generate_from_collection(
@@ -984,6 +1019,7 @@ async fn cmd_nexus_blueprints_generate_from_collection(
 
 async fn cmd_nexus_blueprints_regenerate(
     client: &nexus_client::Client,
+    _destruction_token: DestructiveOperationToken,
 ) -> Result<(), anyhow::Error> {
     let blueprint =
         client.blueprint_regenerate().await.context("generating blueprint")?;
@@ -1037,6 +1073,7 @@ async fn cmd_nexus_sleds_list_uninitialized(
 async fn cmd_nexus_sled_add(
     client: &nexus_client::Client,
     args: &SledAddArgs,
+    _destruction_token: DestructiveOperationToken,
 ) -> Result<(), anyhow::Error> {
     client
         .sled_add(&UninitializedSledId {
@@ -1048,3 +1085,115 @@ async fn cmd_nexus_sled_add(
     eprintln!("added sled {} ({})", args.serial, args.part);
     Ok(())
 }
+
+/// Runs `omdb nexus sleds expunge`
+async fn cmd_nexus_sled_expunge(
+    client: &nexus_client::Client,
+    args: &SledExpungeArgs,
+    omdb: &Omdb,
+    log: &slog::Logger,
+    _destruction_token: DestructiveOperationToken,
+) -> Result<(), anyhow::Error> {
+    // This is an extremely dangerous and irreversible operation. We put a
+    // couple of safeguards in place to ensure this cannot be called without
+    // due consideration:
+    //
+    // 1. We'll require manual input on stdin to confirm the sled to be removed
+    // 2. We'll warn sternly if the sled-to-be-expunged is still present in the
+    //    most recent inventory collection
+    use nexus_db_queries::context::OpContext;
+
+    let datastore = args.db_url_opts.connect(omdb, log).await?;
+    let opctx = OpContext::for_tests(log.clone(), datastore.clone());
+    let opctx = &opctx;
+
+    // First, we need to look up the sled so we know its serial number.
+    let (_authz_sled, sled) = LookupPath::new(opctx, &datastore)
+        .sled_id(args.sled_id)
+        .fetch()
+        .await
+        .with_context(|| format!("failed to find sled {}", args.sled_id))?;
+
+    // Helper to get confirmation messages from the user.
+    let mut line_editor = Reedline::create();
+    let mut read_with_prompt = move |message: &str| {
+        let prompt = DefaultPrompt::new(
+            DefaultPromptSegment::Basic(message.to_string()),
+            DefaultPromptSegment::Empty,
+        );
+        if let Ok(reedline::Signal::Success(input)) =
+            line_editor.read_line(&prompt)
+        {
+            Ok(input)
+        } else {
+            bail!("expungement aborted")
+        }
+    };
+
+    // Now check whether its sled-agent or SP were found in the most recent
+    // inventory collection.
+    match datastore
+        .inventory_get_latest_collection(opctx)
+        .await
+        .context("loading latest collection")?
+    {
+        Some(collection) => {
+            let sled_baseboard = BaseboardId {
+                part_number: sled.part_number().to_string(),
+                serial_number: sled.serial_number().to_string(),
+            };
+            // Check the collection for either the sled-id or the baseboard. In
+            // general a sled-agent's self report should result in both the
+            // sled-id and the baseboard being present in the collection, but
+            // there are some test environments (e.g., `omicron-dev run-all`)
+            // where that isn't guaranteed.
+            let sled_present_in_collection =
+                collection.sled_agents.contains_key(&args.sled_id)
+                    || collection.baseboards.contains(&sled_baseboard);
+            if sled_present_in_collection {
+                eprintln!(
+                    "WARNING: sled {} is PRESENT in the most recent inventory \
+                     collection (spotted at {}). It is dangerous to expunge a \
+                     sled that is still running. Are you sure you want to \
+                     proceed anyway?",
+                    args.sled_id, collection.time_done,
+                );
+                let confirm = read_with_prompt("y/N")?;
+                if confirm != "y" {
+                    eprintln!("expungement not confirmed: aborting");
+                    return Ok(());
+                }
+            }
+        }
+        None => {
+            eprintln!(
+                "WARNING: cannot verify that the sled is physically gone \
+                 because there are no inventory collections present. Please \
+                 make sure that the sled has been physically removed."
+            );
+        }
+    }
+
+    eprintln!(
+        "WARNING: This operation will PERMANENTLY and IRRECOVABLY mark sled \
+        {} ({}) expunged. To proceed, type the sled's serial number.",
+        args.sled_id,
+        sled.serial_number(),
+    );
+    let confirm = read_with_prompt("sled serial number")?;
+    if confirm != sled.serial_number() {
+        eprintln!("sled serial number not confirmed: aborting");
+        return Ok(());
+    }
+
+    let old_policy = client
+        .sled_expunge(&SledSelector { sled: args.sled_id })
+        .await
+        .context("expunging sled")?
+        .into_inner();
+    eprintln!(
+        "expunged sled {} (previous policy: {old_policy:?})",
+        args.sled_id
+    );
+    Ok(())
+}
diff --git a/nexus/db-model/src/sled.rs b/nexus/db-model/src/sled.rs
index c2cbb65bd6..a603f28d57 100644
--- a/nexus/db-model/src/sled.rs
+++ b/nexus/db-model/src/sled.rs
@@ -100,6 +100,10 @@ impl Sled {
         &self.serial_number
     }
 
+    pub fn part_number(&self) -> &str {
+        &self.part_number
+    }
+
     /// The policy here is the `views::SledPolicy` because we expect external
     /// users to always use that.
     pub fn policy(&self) -> views::SledPolicy {
diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs
index aad6e1122f..4ad8dc36ce 100644
--- a/nexus/src/app/sled.rs
+++ b/nexus/src/app/sled.rs
@@ -14,6 +14,7 @@ use nexus_db_queries::db;
 use nexus_db_queries::db::lookup;
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_db_queries::db::model::DatasetKind;
+use nexus_types::external_api::views::SledPolicy;
 use nexus_types::external_api::views::SledProvisionPolicy;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::Error;
@@ -73,6 +74,23 @@ impl super::Nexus {
         Ok(())
     }
 
+    /// Mark a sled as expunged
+    ///
+    /// This is an irreversible process! It should only be called after
+    /// sufficient warning to the operator.
+    ///
+    /// This is idempotent, and it returns the old policy of the sled.
+    pub(crate) async fn sled_expunge(
+        &self,
+        opctx: &OpContext,
+        sled_id: Uuid,
+    ) -> Result<SledPolicy, Error> {
+        let sled_lookup = self.sled_lookup(opctx, &sled_id)?;
+        let (authz_sled,) =
+            sled_lookup.lookup_for(authz::Action::Modify).await?;
+        self.db_datastore.sled_set_policy_to_expunged(opctx, &authz_sled).await
+    }
+
     pub(crate) async fn sled_request_firewall_rules(
         &self,
         opctx: &OpContext,
diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs
index 7f8211dc8e..d6649cd944 100644
--- a/nexus/src/internal_api/http_entrypoints.rs
+++ b/nexus/src/internal_api/http_entrypoints.rs
@@ -30,8 +30,10 @@ use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::BlueprintMetadata;
 use nexus_types::deployment::BlueprintTarget;
 use nexus_types::deployment::BlueprintTargetSet;
+use nexus_types::external_api::params::SledSelector;
 use nexus_types::external_api::params::UninitializedSledId;
 use nexus_types::external_api::shared::UninitializedSled;
+use nexus_types::external_api::views::SledPolicy;
 use nexus_types::internal_api::params::SwitchPutRequest;
 use nexus_types::internal_api::params::SwitchPutResponse;
 use nexus_types::internal_api::views::to_list;
@@ -94,6 +96,7 @@ pub(crate) fn internal_api() -> NexusApiDescription {
 
         api.register(sled_list_uninitialized)?;
         api.register(sled_add)?;
+        api.register(sled_expunge)?;
 
         api.register(probes_get)?;
 
@@ -868,6 +871,31 @@ async fn sled_add(
     apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
 }
 
+/// Mark a sled as expunged
+///
+/// This is an irreversible process! It should only be called after
+/// sufficient warning to the operator.
+///
+/// This is idempotent, and it returns the old policy of the sled.
+#[endpoint {
+    method = POST,
+    path = "/sleds/expunge",
+}]
+async fn sled_expunge(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    sled: TypedBody<SledSelector>,
+) -> Result<HttpResponseOk<SledPolicy>, HttpError> {
+    let apictx = rqctx.context();
+    let nexus = &apictx.nexus;
+    let handler = async {
+        let opctx = crate::context::op_context_for_internal_api(&rqctx).await;
+        let previous_policy =
+            nexus.sled_expunge(&opctx, sled.into_inner().sled).await?;
+        Ok(HttpResponseOk(previous_policy))
+    };
+    apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
 /// Path parameters for probes
 #[derive(Deserialize, JsonSchema)]
 struct ProbePathParam {
diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs
index c6ae47d27c..490661a819 100644
--- a/nexus/types/src/external_api/params.rs
+++ b/nexus/types/src/external_api/params.rs
@@ -92,6 +92,7 @@ id_path_param!(SwitchPath, switch_id, "switch");
 // Internal API parameters
 id_path_param!(BlueprintPath, blueprint_id, "blueprint");
 
+#[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema, PartialEq)]
 pub struct SledSelector {
     /// ID of the sled
     pub sled: Uuid,
diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json
index 9e18c7d6bc..cc831fff73 100644
--- a/openapi/nexus-internal.json
+++ b/openapi/nexus-internal.json
@@ -1047,6 +1047,41 @@
         }
       }
     },
+    "/sleds/expunge": {
+      "post": {
+        "summary": "Mark a sled as expunged",
+        "description": "This is an irreversible process! It should only be called after sufficient warning to the operator.\nThis is idempotent, and it returns the old policy of the sled.",
+        "operationId": "sled_expunge",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/SledSelector"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/SledPolicy"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/sleds/uninitialized": {
       "get": {
         "summary": "List uninitialized sleds",
@@ -6701,6 +6736,69 @@
           "vmm_state"
         ]
       },
+      "SledPolicy": {
+        "description": "The operator-defined policy of a sled.",
+        "oneOf": [
+          {
+            "description": "The operator has indicated that the sled is in-service.",
+            "type": "object",
+            "properties": {
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "in_service"
+                ]
+              },
+              "provision_policy": {
+                "description": "Determines whether new resources can be provisioned onto the sled.",
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/SledProvisionPolicy"
+                  }
+                ]
+              }
+            },
+            "required": [
+              "kind",
+              "provision_policy"
+            ]
+          },
+          {
+            "description": "The operator has indicated that the sled has been permanently removed from service.\n\nThis is a terminal state: once a particular sled ID is expunged, it will never return to service. (The actual hardware may be reused, but it will be treated as a brand-new sled.)\n\nAn expunged sled is always non-provisionable.",
+            "type": "object",
+            "properties": {
+              "kind": {
+                "type": "string",
+                "enum": [
+                  "expunged"
+                ]
+              }
+            },
+            "required": [
+              "kind"
+            ]
+          }
+        ]
+      },
+      "SledProvisionPolicy": {
+        "description": "The operator-defined provision policy of a sled.\n\nThis controls whether new resources are going to be provisioned on this sled.",
+        "oneOf": [
+          {
+            "description": "New resources will be provisioned on this sled.",
+            "type": "string",
+            "enum": [
+              "provisionable"
+            ]
+          },
+          {
+            "description": "New resources will not be provisioned on this sled. However, if the sled is currently in service, existing resources will continue to be on this sled unless manually migrated off.",
+            "type": "string",
+            "enum": [
+              "non_provisionable"
+            ]
+          }
+        ]
+      },
       "SledRole": {
         "description": "Describes the role of the sled within the rack.\n\nNote that this may change if the sled is physically moved within the rack.",
         "oneOf": [
@@ -6720,6 +6818,19 @@
           }
         ]
       },
+      "SledSelector": {
+        "type": "object",
+        "properties": {
+          "sled": {
+            "description": "ID of the sled",
+            "type": "string",
+            "format": "uuid"
+          }
+        },
+        "required": [
+          "sled"
+        ]
+      },
       "SourceNatConfig": {
         "description": "An IP address and port range used for source NAT, i.e., making outbound network connections from guests or services.",
         "type": "object",

From 20b9b00462160e8fd3e1d5b5f5da0db7d72e0148 Mon Sep 17 00:00:00 2001
From: Rain <rain@oxide.computer>
Date: Wed, 13 Mar 2024 11:34:04 -0700
Subject: [PATCH 107/157] [reconfigurator] improvements to blueprint displays
 and tests (#5248)

Bundles up a few changes:

1. Moves `fmt::Debug` for `Blueprint` to a `display` method.
`fmt::Debug` output should ideally look close enough to standard
`fmt::Debug` output, so moving this to some kind of `fmt::Display` is
good.

    But why move this to its own struct? Two reasons:

1. It can be somewhat surprising to get a massive multi-line output just
by printing a `fmt::Display` impl. Using `display` makes this explicit.
2. In the future, if we want more output customization (e.g. colorized
output), the struct gives us a place to hang that customization off of.

2. By the same logic as above, moves diff display to its own struct as
well.
3. Adds snapshot tests to ensure that changes to display output aren't
surprising.
4. To achieve this, adds determinism to each test by using a tree of
RNGs. Initially these RNGs are seeded from entropy, but we can also
choose to seed them from any fixed hashable value.

This tree of RNG approach is pretty standard in things like roguelikes,
where I first came across it. It works quite well.
---
 Cargo.lock                                    |  12 +
 Cargo.toml                                    |   1 +
 dev-tools/omdb/src/bin/omdb/nexus.rs          |   2 +-
 dev-tools/reconfigurator-cli/src/main.rs      |   6 +-
 .../db-queries/src/db/datastore/deployment.rs |   2 +-
 nexus/reconfigurator/planning/Cargo.toml      |   3 +
 .../planning/src/blueprint_builder.rs         | 248 ++++++++++++++----
 nexus/reconfigurator/planning/src/planner.rs  | 147 +++++++----
 .../output/planner_basic_add_sled_2_3.txt     |  48 ++++
 .../output/planner_basic_add_sled_3_5.txt     |  59 +++++
 .../output/planner_nonprovisionable_1_2.txt   |  86 ++++++
 nexus/types/src/deployment.rs                 |  73 ++++--
 12 files changed, 567 insertions(+), 120 deletions(-)
 create mode 100644 nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt
 create mode 100644 nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt
 create mode 100644 nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt

diff --git a/Cargo.lock b/Cargo.lock
index 255433250e..9acd070208 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4624,6 +4624,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "chrono",
+ "expectorate",
  "gateway-client",
  "indexmap 2.2.5",
  "internal-dns",
@@ -4635,6 +4636,8 @@ dependencies = [
  "omicron-common",
  "omicron-test-utils",
  "omicron-workspace-hack",
+ "rand 0.8.5",
+ "rand_seeder",
  "sled-agent-client",
  "slog",
  "thiserror",
@@ -7030,6 +7033,15 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "rand_seeder"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf2890aaef0aa82719a50e808de264f9484b74b442e1a3a0e5ee38243ac40bdb"
+dependencies = [
+ "rand_core 0.6.4",
+]
+
 [[package]]
 name = "rand_xorshift"
 version = "0.3.0"
diff --git a/Cargo.toml b/Cargo.toml
index 0b2e5b58b2..1ac1e3d2eb 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -339,6 +339,7 @@ propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev
 proptest = "1.4.0"
 quote = "1.0"
 rand = "0.8.5"
+rand_seeder = "0.2.3"
 ratatui = "0.26.1"
 rayon = "1.9"
 rcgen = "0.12.1"
diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs
index ecf2a8072d..f27e84f4a2 100644
--- a/dev-tools/omdb/src/bin/omdb/nexus.rs
+++ b/dev-tools/omdb/src/bin/omdb/nexus.rs
@@ -934,7 +934,7 @@ async fn cmd_nexus_blueprints_diff(
     let b2 = client.blueprint_view(&args.blueprint2_id).await.with_context(
         || format!("fetching blueprint {}", args.blueprint2_id),
     )?;
-    println!("{}", b1.diff_sleds(&b2));
+    println!("{}", b1.diff_sleds(&b2).display());
     Ok(())
 }
 
diff --git a/dev-tools/reconfigurator-cli/src/main.rs b/dev-tools/reconfigurator-cli/src/main.rs
index 9cdf09a6f9..fab4237ebd 100644
--- a/dev-tools/reconfigurator-cli/src/main.rs
+++ b/dev-tools/reconfigurator-cli/src/main.rs
@@ -509,7 +509,7 @@ fn cmd_blueprint_show(
         .blueprints
         .get(&args.blueprint_id)
         .ok_or_else(|| anyhow!("no such blueprint: {}", args.blueprint_id))?;
-    Ok(Some(format!("{:?}", blueprint)))
+    Ok(Some(format!("{}", blueprint.display())))
 }
 
 fn cmd_blueprint_diff(
@@ -528,7 +528,7 @@ fn cmd_blueprint_diff(
         .ok_or_else(|| anyhow!("no such blueprint: {}", blueprint2_id))?;
 
     let diff = blueprint1.diff_sleds(&blueprint2);
-    Ok(Some(diff.to_string()))
+    Ok(Some(diff.display().to_string()))
 }
 
 fn cmd_blueprint_diff_inventory(
@@ -547,7 +547,7 @@ fn cmd_blueprint_diff_inventory(
 
     let zones = collection.all_omicron_zones().map(|z| z.id).collect();
     let diff = blueprint.diff_sleds_from_collection(&collection, &zones);
-    Ok(Some(diff.to_string()))
+    Ok(Some(diff.display().to_string()))
 }
 
 fn cmd_save(
diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs
index 5fc17c88d0..7e80f1be6b 100644
--- a/nexus/db-queries/src/db/datastore/deployment.rs
+++ b/nexus/db-queries/src/db/datastore/deployment.rs
@@ -1392,7 +1392,7 @@ mod tests {
             .blueprint_read(&opctx, &authz_blueprint2)
             .await
             .expect("failed to read collection back");
-        println!("diff: {}", blueprint2.diff_sleds(&blueprint_read));
+        println!("diff: {}", blueprint2.diff_sleds(&blueprint_read).display());
         assert_eq!(blueprint2, blueprint_read);
         assert_eq!(blueprint2.internal_dns_version, new_dns_version);
         {
diff --git a/nexus/reconfigurator/planning/Cargo.toml b/nexus/reconfigurator/planning/Cargo.toml
index 41ea17d56e..52111bab94 100644
--- a/nexus/reconfigurator/planning/Cargo.toml
+++ b/nexus/reconfigurator/planning/Cargo.toml
@@ -15,6 +15,8 @@ nexus-config.workspace = true
 nexus-inventory.workspace = true
 nexus-types.workspace = true
 omicron-common.workspace = true
+rand.workspace = true
+rand_seeder.workspace = true
 sled-agent-client.workspace = true
 slog.workspace = true
 thiserror.workspace = true
@@ -23,5 +25,6 @@ uuid.workspace = true
 omicron-workspace-hack.workspace = true
 
 [dev-dependencies]
+expectorate.workspace = true
 omicron-test-utils.workspace = true
 sled-agent-client.workspace = true
diff --git a/nexus/reconfigurator/planning/src/blueprint_builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder.rs
index 26d0d1e23f..7a99343bf9 100644
--- a/nexus/reconfigurator/planning/src/blueprint_builder.rs
+++ b/nexus/reconfigurator/planning/src/blueprint_builder.rs
@@ -35,11 +35,15 @@ use omicron_common::api::external::MacAddr;
 use omicron_common::api::external::Vni;
 use omicron_common::api::internal::shared::NetworkInterface;
 use omicron_common::api::internal::shared::NetworkInterfaceKind;
+use rand::rngs::StdRng;
+use rand::RngCore;
+use rand::SeedableRng;
 use slog::o;
 use slog::Logger;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::collections::HashSet;
+use std::hash::Hash;
 use std::net::IpAddr;
 use std::net::Ipv4Addr;
 use std::net::Ipv6Addr;
@@ -127,6 +131,9 @@ pub struct BlueprintBuilder<'a> {
 
     // Iterator of available MAC addresses in the system address range
     available_system_macs: Box<dyn Iterator<Item = MacAddr>>,
+
+    // Random number generator for new UUIDs
+    rng: BlueprintBuilderRng,
 }
 
 impl<'a> BlueprintBuilder<'a> {
@@ -137,6 +144,42 @@ impl<'a> BlueprintBuilder<'a> {
         internal_dns_version: Generation,
         policy: &'a Policy,
         creator: &str,
+    ) -> Result<Blueprint, Error> {
+        Self::build_initial_impl(
+            collection,
+            internal_dns_version,
+            policy,
+            creator,
+            BlueprintBuilderRng::new(),
+        )
+    }
+
+    /// A version of [`Self::build_initial_from_collection`] that allows the
+    /// blueprint ID to be generated from a random seed.
+    pub fn build_initial_from_collection_seeded<H: Hash>(
+        collection: &'a Collection,
+        internal_dns_version: Generation,
+        policy: &'a Policy,
+        creator: &str,
+        seed: H,
+    ) -> Result<Blueprint, Error> {
+        let mut rng = BlueprintBuilderRng::new();
+        rng.set_seed(seed);
+        Self::build_initial_impl(
+            collection,
+            internal_dns_version,
+            policy,
+            creator,
+            rng,
+        )
+    }
+
+    fn build_initial_impl(
+        collection: &'a Collection,
+        internal_dns_version: Generation,
+        policy: &'a Policy,
+        creator: &str,
+        mut rng: BlueprintBuilderRng,
     ) -> Result<Blueprint, Error> {
         let omicron_zones = policy
             .sleds
@@ -177,7 +220,7 @@ impl<'a> BlueprintBuilder<'a> {
         let zones_in_service =
             collection.all_omicron_zones().map(|z| z.id).collect();
         Ok(Blueprint {
-            id: Uuid::new_v4(),
+            id: rng.blueprint_rng.next_uuid(),
             omicron_zones,
             zones_in_service,
             parent_blueprint_id: None,
@@ -311,16 +354,17 @@ impl<'a> BlueprintBuilder<'a> {
             nexus_v6_ips,
             available_external_ips,
             available_system_macs,
+            rng: BlueprintBuilderRng::new(),
         })
     }
 
     /// Assemble a final [`Blueprint`] based on the contents of the builder
-    pub fn build(self) -> Blueprint {
+    pub fn build(mut self) -> Blueprint {
         // Collect the Omicron zones config for each in-service sled.
         let omicron_zones =
             self.zones.into_omicron_zones(self.policy.sleds.keys().copied());
         Blueprint {
-            id: Uuid::new_v4(),
+            id: self.rng.blueprint_rng.next_uuid(),
             omicron_zones,
             zones_in_service: self.zones_in_service,
             parent_blueprint_id: Some(self.parent_blueprint.id),
@@ -331,6 +375,15 @@ impl<'a> BlueprintBuilder<'a> {
         }
     }
 
+    /// Within tests, set a seeded RNG for deterministic results.
+    ///
+    /// This will ensure that tests that use this builder will produce the same
+    /// results each time they are run.
+    pub fn set_rng_seed<H: Hash>(&mut self, seed: H) -> &mut Self {
+        self.rng.set_seed(seed);
+        self
+    }
+
     /// Sets the blueprints "comment"
     ///
     /// This is a short human-readable string summarizing the changes reflected
@@ -388,7 +441,7 @@ impl<'a> BlueprintBuilder<'a> {
             .collect();
 
         let zone = OmicronZoneConfig {
-            id: Uuid::new_v4(),
+            id: self.rng.zone_rng.next_uuid(),
             underlay_address: ip,
             zone_type: OmicronZoneType::InternalNtp {
                 address: ntp_address.to_string(),
@@ -434,7 +487,7 @@ impl<'a> BlueprintBuilder<'a> {
         let port = omicron_common::address::CRUCIBLE_PORT;
         let address = SocketAddrV6::new(ip, port, 0, 0).to_string();
         let zone = OmicronZoneConfig {
-            id: Uuid::new_v4(),
+            id: self.rng.zone_rng.next_uuid(),
             underlay_address: ip,
             zone_type: OmicronZoneType::Crucible {
                 address,
@@ -519,7 +572,7 @@ impl<'a> BlueprintBuilder<'a> {
         };
 
         for _ in 0..num_nexus_to_add {
-            let nexus_id = Uuid::new_v4();
+            let nexus_id = self.rng.zone_rng.next_uuid();
             let external_ip = self
                 .available_external_ips
                 .next()
@@ -547,7 +600,7 @@ impl<'a> BlueprintBuilder<'a> {
                     .next()
                     .ok_or(Error::NoSystemMacAddressAvailable)?;
                 NetworkInterface {
-                    id: Uuid::new_v4(),
+                    id: self.rng.network_interface_rng.next_uuid(),
                     kind: NetworkInterfaceKind::Service { id: nexus_id },
                     name: format!("nexus-{nexus_id}").parse().unwrap(),
                     ip,
@@ -649,6 +702,75 @@ impl<'a> BlueprintBuilder<'a> {
     }
 }
 
+#[derive(Debug)]
+struct BlueprintBuilderRng {
+    // Have separate RNGs for the different kinds of UUIDs we might add,
+    // generated from the main RNG. This is so that e.g. adding a new network
+    // interface doesn't alter the blueprint or sled UUID.
+    //
+    // In the future, when we switch to typed UUIDs, each of these will be
+    // associated with a specific `TypedUuidKind`.
+    blueprint_rng: UuidRng,
+    zone_rng: UuidRng,
+    network_interface_rng: UuidRng,
+}
+
+impl BlueprintBuilderRng {
+    fn new() -> Self {
+        Self::new_from_rng(StdRng::from_entropy())
+    }
+
+    fn new_from_rng(mut root_rng: StdRng) -> Self {
+        let blueprint_rng = UuidRng::from_root_rng(&mut root_rng, "blueprint");
+        let zone_rng = UuidRng::from_root_rng(&mut root_rng, "zone");
+        let network_interface_rng =
+            UuidRng::from_root_rng(&mut root_rng, "network_interface");
+
+        BlueprintBuilderRng { blueprint_rng, zone_rng, network_interface_rng }
+    }
+
+    fn set_seed<H: Hash>(&mut self, seed: H) {
+        // Important to add some more bytes here, so that builders with the
+        // same seed but different purposes don't end up with the same UUIDs.
+        const SEED_EXTRA: &str = "blueprint-builder";
+        let mut seeder = rand_seeder::Seeder::from((seed, SEED_EXTRA));
+        *self = Self::new_from_rng(seeder.make_rng::<StdRng>());
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct UuidRng {
+    rng: StdRng,
+}
+
+impl UuidRng {
+    /// Returns a new `UuidRng` generated from the root RNG.
+    ///
+    /// `extra` is a string that should be unique to the purpose of the UUIDs.
+    fn from_root_rng(root_rng: &mut StdRng, extra: &'static str) -> Self {
+        let seed = root_rng.next_u64();
+        let mut seeder = rand_seeder::Seeder::from((seed, extra));
+        Self { rng: seeder.make_rng::<StdRng>() }
+    }
+
+    /// `extra` is a string that should be unique to the purpose of the UUIDs.
+    #[cfg(test)]
+    fn from_seed<H: Hash>(seed: H, extra: &'static str) -> Self {
+        let mut seeder = rand_seeder::Seeder::from((seed, extra));
+        Self { rng: seeder.make_rng::<StdRng>() }
+    }
+
+    /// Returns a new UUIDv4 generated from the RNG.
+    pub(crate) fn next_uuid(&mut self) -> Uuid {
+        let mut bytes = [0; 16];
+        self.rng.fill_bytes(&mut bytes);
+        // Builder::from_random_bytes will turn the random bytes into a valid
+        // UUIDv4. (Parts of the system depend on the UUID actually being valid
+        // v4, so it's important that we don't just use `uuid::from_bytes`.)
+        uuid::Builder::from_random_bytes(bytes).into_uuid()
+    }
+}
+
 /// Helper for working with sets of zones on each sled
 ///
 /// Tracking the set of zones is slightly non-trivial because we need to bump
@@ -765,13 +887,23 @@ pub mod test {
         pub policy: Policy,
         pub collection: Collection,
         pub blueprint: Blueprint,
+        // If we add more types of RNGs than just sleds here, we'll need to
+        // expand this to be similar to BlueprintBuilderRng where a root RNG
+        // creates sub-RNGs.
+        pub(crate) sled_rng: UuidRng,
     }
 
     impl ExampleSystem {
-        pub fn new(log: &slog::Logger, nsleds: usize) -> ExampleSystem {
+        pub fn new(
+            log: &slog::Logger,
+            test_name: &str,
+            nsleds: usize,
+        ) -> ExampleSystem {
             let mut system = SystemDescription::new();
+            let mut sled_rng = UuidRng::from_seed(test_name, "ExampleSystem");
+
             let sled_ids: Vec<_> =
-                (0..nsleds).map(|_| Uuid::new_v4()).collect();
+                (0..nsleds).map(|_| sled_rng.next_uuid()).collect();
             for sled_id in &sled_ids {
                 let _ = system.sled(SledBuilder::new().id(*sled_id)).unwrap();
             }
@@ -799,11 +931,12 @@ pub mod test {
 
             let empty_zone_inventory = inventory_builder.build();
             let initial_blueprint =
-                BlueprintBuilder::build_initial_from_collection(
+                BlueprintBuilder::build_initial_from_collection_seeded(
                     &empty_zone_inventory,
                     Generation::new(),
                     &policy,
                     "test suite",
+                    (test_name, "ExampleSystem initial"),
                 )
                 .unwrap();
 
@@ -816,6 +949,7 @@ pub mod test {
                 "test suite",
             )
             .unwrap();
+            builder.set_rng_seed((test_name, "ExampleSystem make_zones"));
             for (sled_id, sled_resources) in &policy.sleds {
                 let _ = builder.sled_ensure_zone_ntp(*sled_id).unwrap();
                 let _ = builder
@@ -856,17 +990,24 @@ pub mod test {
                 policy,
                 collection: builder.build(),
                 blueprint,
+                sled_rng,
             }
         }
     }
 
     /// Returns a collection and policy describing a pretty simple system.
     ///
+    /// The test name is used as the RNG seed.
+    ///
     /// `n_sleds` is the number of sleds supported. Currently, this value can
     /// be anywhere between 0 and 5. (More can be added in the future if
     /// necessary.)
-    pub fn example(log: &slog::Logger, nsleds: usize) -> (Collection, Policy) {
-        let example = ExampleSystem::new(log, nsleds);
+    pub fn example(
+        log: &slog::Logger,
+        test_name: &str,
+        nsleds: usize,
+    ) -> (Collection, Policy) {
+        let example = ExampleSystem::new(log, test_name, nsleds);
         (example.collection, example.policy)
     }
 
@@ -893,14 +1034,17 @@ pub mod test {
     fn test_initial() {
         // Test creating a blueprint from a collection and verifying that it
         // describes no changes.
-        let logctx = test_setup_log("blueprint_builder_test_initial");
-        let (collection, policy) = example(&logctx.log, DEFAULT_N_SLEDS);
+        static TEST_NAME: &str = "blueprint_builder_test_initial";
+        let logctx = test_setup_log(TEST_NAME);
+        let (collection, policy) =
+            example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS);
         let blueprint_initial =
-            BlueprintBuilder::build_initial_from_collection(
+            BlueprintBuilder::build_initial_from_collection_seeded(
                 &collection,
                 Generation::new(),
                 &policy,
                 "the_test",
+                TEST_NAME,
             )
             .expect("failed to create initial blueprint");
         verify_blueprint(&blueprint_initial);
@@ -912,7 +1056,7 @@ pub mod test {
             .diff_sleds_from_collection(&collection, &zones_in_service);
         println!(
             "collection -> initial blueprint (expected no changes):\n{}",
-            diff
+            diff.display()
         );
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
@@ -932,7 +1076,7 @@ pub mod test {
         let diff = blueprint_initial.diff_sleds(&blueprint);
         println!(
             "initial blueprint -> next blueprint (expected no changes):\n{}",
-            diff
+            diff.display()
         );
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
@@ -943,8 +1087,10 @@ pub mod test {
 
     #[test]
     fn test_basic() {
-        let logctx = test_setup_log("blueprint_builder_test_basic");
-        let mut example = ExampleSystem::new(&logctx.log, DEFAULT_N_SLEDS);
+        static TEST_NAME: &str = "blueprint_builder_test_basic";
+        let logctx = test_setup_log(TEST_NAME);
+        let mut example =
+            ExampleSystem::new(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS);
         let blueprint1 = &example.blueprint;
         verify_blueprint(blueprint1);
 
@@ -974,14 +1120,14 @@ pub mod test {
         let diff = blueprint1.diff_sleds(&blueprint2);
         println!(
             "initial blueprint -> next blueprint (expected no changes):\n{}",
-            diff
+            diff.display()
         );
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
         assert_eq!(diff.sleds_changed().count(), 0);
 
         // The next step is adding these zones to a new sled.
-        let new_sled_id = Uuid::new_v4();
+        let new_sled_id = example.sled_rng.next_uuid();
         let _ =
             example.system.sled(SledBuilder::new().id(new_sled_id)).unwrap();
         let policy = example.system.to_policy().unwrap();
@@ -1004,7 +1150,7 @@ pub mod test {
         let blueprint3 = builder.build();
         verify_blueprint(&blueprint3);
         let diff = blueprint2.diff_sleds(&blueprint3);
-        println!("expecting new NTP and Crucible zones:\n{}", diff);
+        println!("expecting new NTP and Crucible zones:\n{}", diff.display());
 
         // No sleds were changed or removed.
         assert_eq!(diff.sleds_changed().count(), 0);
@@ -1063,10 +1209,11 @@ pub mod test {
 
     #[test]
     fn test_add_nexus_with_no_existing_nexus_zones() {
-        let logctx = test_setup_log(
-            "blueprint_builder_test_add_nexus_with_no_existing_nexus_zones",
-        );
-        let (mut collection, policy) = example(&logctx.log, DEFAULT_N_SLEDS);
+        static TEST_NAME: &str =
+            "blueprint_builder_test_add_nexus_with_no_existing_nexus_zones";
+        let logctx = test_setup_log(TEST_NAME);
+        let (mut collection, policy) =
+            example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS);
 
         // We don't care about the internal DNS version here.
         let internal_dns_version = Generation::new();
@@ -1081,11 +1228,12 @@ pub mod test {
             });
         }
 
-        let parent = BlueprintBuilder::build_initial_from_collection(
+        let parent = BlueprintBuilder::build_initial_from_collection_seeded(
             &collection,
             internal_dns_version,
             &policy,
             "test",
+            TEST_NAME,
         )
         .expect("failed to create initial blueprint");
 
@@ -1120,9 +1268,10 @@ pub mod test {
 
     #[test]
     fn test_add_nexus_error_cases() {
-        let logctx =
-            test_setup_log("blueprint_builder_test_add_nexus_error_cases");
-        let (mut collection, policy) = example(&logctx.log, DEFAULT_N_SLEDS);
+        static TEST_NAME: &str = "blueprint_builder_test_add_nexus_error_cases";
+        let logctx = test_setup_log(TEST_NAME);
+        let (mut collection, policy) =
+            example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS);
 
         // We don't care about the internal DNS version here.
         let internal_dns_version = Generation::new();
@@ -1145,11 +1294,12 @@ pub mod test {
             selected_sled_id.expect("found no sleds with Nexus zone")
         };
 
-        let parent = BlueprintBuilder::build_initial_from_collection(
+        let parent = BlueprintBuilder::build_initial_from_collection_seeded(
             &collection,
             Generation::new(),
             &policy,
             "test",
+            TEST_NAME,
         )
         .expect("failed to create initial blueprint");
 
@@ -1239,11 +1389,12 @@ pub mod test {
 
     #[test]
     fn test_invalid_parent_blueprint_two_zones_with_same_external_ip() {
-        let logctx = test_setup_log(
+        static TEST_NAME: &str =
             "blueprint_builder_test_invalid_parent_blueprint_\
-             two_zones_with_same_external_ip",
-        );
-        let (mut collection, policy) = example(&logctx.log, DEFAULT_N_SLEDS);
+             two_zones_with_same_external_ip";
+        let logctx = test_setup_log(TEST_NAME);
+        let (mut collection, policy) =
+            example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS);
 
         // We should fail if the parent blueprint claims to contain two
         // zones with the same external IP. Skim through the zones, copy the
@@ -1270,11 +1421,12 @@ pub mod test {
         }
         assert!(found_second_nexus_zone, "only one Nexus zone present?");
 
-        let parent = BlueprintBuilder::build_initial_from_collection(
+        let parent = BlueprintBuilder::build_initial_from_collection_seeded(
             &collection,
             Generation::new(),
             &policy,
             "test",
+            TEST_NAME,
         )
         .unwrap();
 
@@ -1297,11 +1449,12 @@ pub mod test {
 
     #[test]
     fn test_invalid_parent_blueprint_two_nexus_zones_with_same_nic_ip() {
-        let logctx = test_setup_log(
+        static TEST_NAME: &str =
             "blueprint_builder_test_invalid_parent_blueprint_\
-             two_nexus_zones_with_same_nic_ip",
-        );
-        let (mut collection, policy) = example(&logctx.log, DEFAULT_N_SLEDS);
+             two_nexus_zones_with_same_nic_ip";
+        let logctx = test_setup_log(TEST_NAME);
+        let (mut collection, policy) =
+            example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS);
 
         // We should fail if the parent blueprint claims to contain two
         // Nexus zones with the same NIC IP. Skim through the zones, copy
@@ -1326,11 +1479,12 @@ pub mod test {
         }
         assert!(found_second_nexus_zone, "only one Nexus zone present?");
 
-        let parent = BlueprintBuilder::build_initial_from_collection(
+        let parent = BlueprintBuilder::build_initial_from_collection_seeded(
             &collection,
             Generation::new(),
             &policy,
             "test",
+            TEST_NAME,
         )
         .unwrap();
 
@@ -1353,11 +1507,12 @@ pub mod test {
 
     #[test]
     fn test_invalid_parent_blueprint_two_zones_with_same_vnic_mac() {
-        let logctx = test_setup_log(
+        static TEST_NAME: &str =
             "blueprint_builder_test_invalid_parent_blueprint_\
-             two_zones_with_same_vnic_mac",
-        );
-        let (mut collection, policy) = example(&logctx.log, DEFAULT_N_SLEDS);
+             two_zones_with_same_vnic_mac";
+        let logctx = test_setup_log(TEST_NAME);
+        let (mut collection, policy) =
+            example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS);
 
         // We should fail if the parent blueprint claims to contain two
         // zones with the same service vNIC MAC address. Skim through the
@@ -1382,11 +1537,12 @@ pub mod test {
         }
         assert!(found_second_nexus_zone, "only one Nexus zone present?");
 
-        let parent = BlueprintBuilder::build_initial_from_collection(
+        let parent = BlueprintBuilder::build_initial_from_collection_seeded(
             &collection,
             Generation::new(),
             &policy,
             "test",
+            TEST_NAME,
         )
         .unwrap();
 
diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs
index bae3dfd68a..1882d62e22 100644
--- a/nexus/reconfigurator/planning/src/planner.rs
+++ b/nexus/reconfigurator/planning/src/planner.rs
@@ -18,6 +18,7 @@ use omicron_common::api::external::Generation;
 use slog::{info, warn, Logger};
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
+use std::hash::Hash;
 use uuid::Uuid;
 
 pub struct Planner<'a> {
@@ -57,6 +58,17 @@ impl<'a> Planner<'a> {
         Ok(Planner { log, policy, blueprint, inventory })
     }
 
+    /// Within tests, set a seeded RNG for deterministic results.
+    ///
+    /// This will ensure that tests that use this builder will produce the same
+    /// results each time they are run.
+    pub fn with_rng_seed<H: Hash>(mut self, seed: H) -> Self {
+        // This is an owned builder because it is almost never going to be
+        // conditional.
+        self.blueprint.set_rng_seed(seed);
+        self
+    }
+
     pub fn plan(mut self) -> Result<Blueprint, Error> {
         self.do_plan()?;
         Ok(self.blueprint.build())
@@ -324,6 +336,7 @@ mod test {
     use crate::blueprint_builder::test::DEFAULT_N_SLEDS;
     use crate::blueprint_builder::BlueprintBuilder;
     use crate::system::SledBuilder;
+    use expectorate::assert_contents;
     use nexus_inventory::now_db_precision;
     use nexus_types::external_api::views::SledPolicy;
     use nexus_types::external_api::views::SledProvisionPolicy;
@@ -336,23 +349,27 @@ mod test {
     /// Runs through a basic sequence of blueprints for adding a sled
     #[test]
     fn test_basic_add_sled() {
-        let logctx = test_setup_log("planner_basic_add_sled");
+        static TEST_NAME: &str = "planner_basic_add_sled";
+        let logctx = test_setup_log(TEST_NAME);
 
         // For our purposes, we don't care about the internal DNS generation.
         let internal_dns_version = Generation::new();
 
         // Use our example inventory collection.
-        let mut example = ExampleSystem::new(&logctx.log, DEFAULT_N_SLEDS);
+        let mut example =
+            ExampleSystem::new(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS);
 
         // Build the initial blueprint.  We don't bother verifying it here
         // because there's a separate test for that.
-        let blueprint1 = BlueprintBuilder::build_initial_from_collection(
-            &example.collection,
-            internal_dns_version,
-            &example.policy,
-            "the_test",
-        )
-        .expect("failed to create initial blueprint");
+        let blueprint1 =
+            BlueprintBuilder::build_initial_from_collection_seeded(
+                &example.collection,
+                internal_dns_version,
+                &example.policy,
+                "the_test",
+                (TEST_NAME, "bp1"),
+            )
+            .expect("failed to create initial blueprint");
         verify_blueprint(&blueprint1);
 
         // Now run the planner.  It should do nothing because our initial
@@ -367,19 +384,19 @@ mod test {
             &example.collection,
         )
         .expect("failed to create planner")
+        .with_rng_seed((TEST_NAME, "bp2"))
         .plan()
         .expect("failed to plan");
 
         let diff = blueprint1.diff_sleds(&blueprint2);
-        println!("1 -> 2 (expected no changes):\n{}", diff);
+        println!("1 -> 2 (expected no changes):\n{}", diff.display());
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
         assert_eq!(diff.sleds_changed().count(), 0);
         verify_blueprint(&blueprint2);
 
         // Now add a new sled.
-        let new_sled_id =
-            "7097f5b3-5896-4fff-bd97-63a9a69563a9".parse().unwrap();
+        let new_sled_id = example.sled_rng.next_uuid();
         let _ =
             example.system.sled(SledBuilder::new().id(new_sled_id)).unwrap();
         let policy = example.system.to_policy().unwrap();
@@ -394,11 +411,19 @@ mod test {
             &example.collection,
         )
         .expect("failed to create planner")
+        .with_rng_seed((TEST_NAME, "bp3"))
         .plan()
         .expect("failed to plan");
 
         let diff = blueprint2.diff_sleds(&blueprint3);
-        println!("2 -> 3 (expect new NTP zone on new sled):\n{}", diff);
+        println!(
+            "2 -> 3 (expect new NTP zone on new sled):\n{}",
+            diff.display()
+        );
+        assert_contents(
+            "tests/output/planner_basic_add_sled_2_3.txt",
+            &diff.display().to_string(),
+        );
         let sleds = diff.sleds_added().collect::<Vec<_>>();
         let (sled_id, sled_zones) = sleds[0];
         // We have defined elsewhere that the first generation contains no
@@ -427,10 +452,11 @@ mod test {
             &example.collection,
         )
         .expect("failed to create planner")
+        .with_rng_seed((TEST_NAME, "bp4"))
         .plan()
         .expect("failed to plan");
         let diff = blueprint3.diff_sleds(&blueprint4);
-        println!("3 -> 4 (expected no changes):\n{}", diff);
+        println!("3 -> 4 (expected no changes):\n{}", diff.display());
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
         assert_eq!(diff.sleds_changed().count(), 0);
@@ -465,11 +491,16 @@ mod test {
             &collection,
         )
         .expect("failed to create planner")
+        .with_rng_seed((TEST_NAME, "bp5"))
         .plan()
         .expect("failed to plan");
 
         let diff = blueprint3.diff_sleds(&blueprint5);
-        println!("3 -> 5 (expect Crucible zones):\n{}", diff);
+        println!("3 -> 5 (expect Crucible zones):\n{}", diff.display());
+        assert_contents(
+            "tests/output/planner_basic_add_sled_3_5.txt",
+            &diff.display().to_string(),
+        );
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
         let sleds = diff.sleds_changed().collect::<Vec<_>>();
@@ -501,11 +532,12 @@ mod test {
             &collection,
         )
         .expect("failed to create planner")
+        .with_rng_seed((TEST_NAME, "bp6"))
         .plan()
         .expect("failed to plan");
 
         let diff = blueprint5.diff_sleds(&blueprint6);
-        println!("5 -> 6 (expect no changes):\n{}", diff);
+        println!("5 -> 6 (expect no changes):\n{}", diff.display());
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
         assert_eq!(diff.sleds_changed().count(), 0);
@@ -518,7 +550,8 @@ mod test {
     /// needed
     #[test]
     fn test_add_multiple_nexus_to_one_sled() {
-        let logctx = test_setup_log("planner_add_multiple_nexus_to_one_sled");
+        static TEST_NAME: &str = "planner_add_multiple_nexus_to_one_sled";
+        let logctx = test_setup_log(TEST_NAME);
 
         // For our purposes, we don't care about the internal DNS generation.
         let internal_dns_version = Generation::new();
@@ -527,7 +560,7 @@ mod test {
         // it down to just one sled.
         let (sled_id, collection, mut policy) = {
             let (mut collection, mut policy) =
-                example(&logctx.log, DEFAULT_N_SLEDS);
+                example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS);
 
             // Pick one sled ID to keep and remove the rest.
             let keep_sled_id =
@@ -543,13 +576,15 @@ mod test {
         };
 
         // Build the initial blueprint.
-        let blueprint1 = BlueprintBuilder::build_initial_from_collection(
-            &collection,
-            internal_dns_version,
-            &policy,
-            "the_test",
-        )
-        .expect("failed to create initial blueprint");
+        let blueprint1 =
+            BlueprintBuilder::build_initial_from_collection_seeded(
+                &collection,
+                internal_dns_version,
+                &policy,
+                "the_test",
+                (TEST_NAME, "bp1"),
+            )
+            .expect("failed to create initial blueprint");
 
         // This blueprint should only have 1 Nexus instance on the one sled we
         // kept.
@@ -578,11 +613,12 @@ mod test {
             &collection,
         )
         .expect("failed to create planner")
+        .with_rng_seed((TEST_NAME, "bp2"))
         .plan()
         .expect("failed to plan");
 
         let diff = blueprint1.diff_sleds(&blueprint2);
-        println!("1 -> 2 (added additional Nexus zones):\n{}", diff);
+        println!("1 -> 2 (added additional Nexus zones):\n{}", diff.display());
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
         let mut sleds = diff.sleds_changed().collect::<Vec<_>>();
@@ -606,21 +642,24 @@ mod test {
     /// sleds as it adds them
     #[test]
     fn test_spread_additional_nexus_zones_across_sleds() {
-        let logctx = test_setup_log(
-            "planner_spread_additional_nexus_zones_across_sleds",
-        );
+        static TEST_NAME: &str =
+            "planner_spread_additional_nexus_zones_across_sleds";
+        let logctx = test_setup_log(TEST_NAME);
 
         // Use our example inventory collection as a starting point.
-        let (collection, mut policy) = example(&logctx.log, DEFAULT_N_SLEDS);
+        let (collection, mut policy) =
+            example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS);
 
         // Build the initial blueprint.
-        let blueprint1 = BlueprintBuilder::build_initial_from_collection(
-            &collection,
-            Generation::new(),
-            &policy,
-            "the_test",
-        )
-        .expect("failed to create initial blueprint");
+        let blueprint1 =
+            BlueprintBuilder::build_initial_from_collection_seeded(
+                &collection,
+                Generation::new(),
+                &policy,
+                "the_test",
+                (TEST_NAME, "bp1"),
+            )
+            .expect("failed to create initial blueprint");
 
         // This blueprint should only have 3 Nexus zones: one on each sled.
         assert_eq!(blueprint1.omicron_zones.len(), 3);
@@ -646,11 +685,12 @@ mod test {
             &collection,
         )
         .expect("failed to create planner")
+        .with_rng_seed((TEST_NAME, "bp2"))
         .plan()
         .expect("failed to plan");
 
         let diff = blueprint1.diff_sleds(&blueprint2);
-        println!("1 -> 2 (added additional Nexus zones):\n{}", diff);
+        println!("1 -> 2 (added additional Nexus zones):\n{}", diff.display());
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
         let sleds = diff.sleds_changed().collect::<Vec<_>>();
@@ -687,9 +727,9 @@ mod test {
     /// extra Nexus zones
     #[test]
     fn test_nexus_allocation_skips_nonprovisionable_sleds() {
-        let logctx = test_setup_log(
-            "planner_nexus_allocation_skips_nonprovisionable_sleds",
-        );
+        static TEST_NAME: &str =
+            "planner_nexus_allocation_skips_nonprovisionable_sleds";
+        let logctx = test_setup_log(TEST_NAME);
 
         // Use our example inventory collection as a starting point.
         //
@@ -697,16 +737,18 @@ mod test {
         // and decommissioned sleds. (When we add more kinds of
         // non-provisionable states in the future, we'll have to add more
         // sleds.)
-        let (collection, mut policy) = example(&logctx.log, 5);
+        let (collection, mut policy) = example(&logctx.log, TEST_NAME, 5);
 
         // Build the initial blueprint.
-        let blueprint1 = BlueprintBuilder::build_initial_from_collection(
-            &collection,
-            Generation::new(),
-            &policy,
-            "the_test",
-        )
-        .expect("failed to create initial blueprint");
+        let blueprint1 =
+            BlueprintBuilder::build_initial_from_collection_seeded(
+                &collection,
+                Generation::new(),
+                &policy,
+                "the_test",
+                (TEST_NAME, "bp1"),
+            )
+            .expect("failed to create initial blueprint");
 
         // This blueprint should only have 5 Nexus zones: one on each sled.
         assert_eq!(blueprint1.omicron_zones.len(), 5);
@@ -763,11 +805,16 @@ mod test {
             &collection,
         )
         .expect("failed to create planner")
+        .with_rng_seed((TEST_NAME, "bp2"))
         .plan()
         .expect("failed to plan");
 
         let diff = blueprint1.diff_sleds(&blueprint2);
-        println!("1 -> 2 (added additional Nexus zones):\n{}", diff);
+        println!("1 -> 2 (added additional Nexus zones):\n{}", diff.display());
+        assert_contents(
+            "tests/output/planner_nonprovisionable_1_2.txt",
+            &diff.display().to_string(),
+        );
         assert_eq!(diff.sleds_added().count(), 0);
         assert_eq!(diff.sleds_removed().count(), 0);
         let sleds = diff.sleds_changed().collect::<Vec<_>>();
diff --git a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt
new file mode 100644
index 0000000000..edad186e0c
--- /dev/null
+++ b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt
@@ -0,0 +1,48 @@
+diff blueprint 979ef428-0bdd-4622-8a72-0719e942b415 blueprint 4171ad05-89dd-474b-846b-b007e4346366
+--- blueprint 979ef428-0bdd-4622-8a72-0719e942b415
++++ blueprint 4171ad05-89dd-474b-846b-b007e4346366
+  sled 41f45d9f-766e-4ca6-a881-61ee45c80f57
+      zone config generation 2
+         zone 267ed614-92af-4b9d-bdba-c2881c2e43a2 type internal_ntp underlay IP fd00:1122:3344:103::21 (unchanged)
+         zone 322ee9f1-8903-4542-a0a8-a54cefabdeca type crucible underlay IP fd00:1122:3344:103::24 (unchanged)
+         zone 4ab1650f-32c5-447f-939d-64b8103a7645 type crucible underlay IP fd00:1122:3344:103::2a (unchanged)
+         zone 64aa65f8-1ccb-4cd6-9953-027aebdac8ff type crucible underlay IP fd00:1122:3344:103::27 (unchanged)
+         zone 6e811d86-8aa7-4660-935b-84b4b7721b10 type crucible underlay IP fd00:1122:3344:103::2b (unchanged)
+         zone 747d2426-68bf-4c22-8806-41d290b5d5f5 type crucible underlay IP fd00:1122:3344:103::25 (unchanged)
+         zone 7fbd2c38-5dc3-48c4-b061-558a2041d70f type crucible underlay IP fd00:1122:3344:103::2c (unchanged)
+         zone 8e9e923e-62b1-4cbc-9f59-d6397e338b6b type crucible underlay IP fd00:1122:3344:103::29 (unchanged)
+         zone b14d5478-1a0e-4b90-b526-36b06339dfc4 type crucible underlay IP fd00:1122:3344:103::28 (unchanged)
+         zone b40f7c7b-526c-46c8-ae33-67280c280eb7 type crucible underlay IP fd00:1122:3344:103::23 (unchanged)
+         zone be97b92b-38d6-422a-8c76-d37060f75bd2 type crucible underlay IP fd00:1122:3344:103::26 (unchanged)
+         zone cc816cfe-3869-4dde-b596-397d41198628 type nexus underlay IP fd00:1122:3344:103::22 (unchanged)
+  sled 43677374-8d2f-4deb-8a41-eeea506db8e0
+      zone config generation 2
+         zone 02acbe6a-1c88-47e3-94c3-94084cbde098 type crucible underlay IP fd00:1122:3344:101::27 (unchanged)
+         zone 07c3c805-8888-4fe5-9543-3d2479dbe6f3 type crucible underlay IP fd00:1122:3344:101::26 (unchanged)
+         zone 08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb type internal_ntp underlay IP fd00:1122:3344:101::21 (unchanged)
+         zone 10d98a73-ec88-4aff-a7e8-7db6a87880e6 type crucible underlay IP fd00:1122:3344:101::24 (unchanged)
+         zone 2a455c35-eb3c-4c73-ab6c-d0a706e25316 type crucible underlay IP fd00:1122:3344:101::29 (unchanged)
+         zone 3eda924f-22a9-4f3e-9a1b-91d1c47601ab type crucible underlay IP fd00:1122:3344:101::23 (unchanged)
+         zone 587be699-a320-4c79-b320-128d9ecddc0b type crucible underlay IP fd00:1122:3344:101::2b (unchanged)
+         zone 6fa06115-4959-4913-8e7b-dd70d7651f07 type crucible underlay IP fd00:1122:3344:101::2c (unchanged)
+         zone 8f3a1cc5-9195-4a30-ad02-b804278fe639 type crucible underlay IP fd00:1122:3344:101::28 (unchanged)
+         zone a1696cd4-588c-484a-b95b-66e824c0ce05 type crucible underlay IP fd00:1122:3344:101::25 (unchanged)
+         zone a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 type crucible underlay IP fd00:1122:3344:101::2a (unchanged)
+         zone c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 type nexus underlay IP fd00:1122:3344:101::22 (unchanged)
+  sled 590e3034-d946-4166-b0e5-2d0034197a07
+      zone config generation 2
+         zone 18f8fe40-646e-4962-b17a-20e201f3a6e5 type crucible underlay IP fd00:1122:3344:102::2a (unchanged)
+         zone 47199d48-534c-4267-a654-d2d90e64b498 type internal_ntp underlay IP fd00:1122:3344:102::21 (unchanged)
+         zone 56d5d7cf-db2c-40a3-a775-003241ad4820 type crucible underlay IP fd00:1122:3344:102::29 (unchanged)
+         zone 6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 type crucible underlay IP fd00:1122:3344:102::2b (unchanged)
+         zone 704e1fed-f8d6-4cfa-a470-bad27fdc06d1 type nexus underlay IP fd00:1122:3344:102::22 (unchanged)
+         zone 7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 type crucible underlay IP fd00:1122:3344:102::26 (unchanged)
+         zone 93f2f40c-5616-4d8d-8519-ec6debdcede0 type crucible underlay IP fd00:1122:3344:102::2c (unchanged)
+         zone ab7ba6df-d401-40bd-940e-faf57c57aa2a type crucible underlay IP fd00:1122:3344:102::28 (unchanged)
+         zone af322036-371f-437c-8c08-7f40f3f1403b type crucible underlay IP fd00:1122:3344:102::23 (unchanged)
+         zone d637264f-6f40-44c2-8b7e-a179430210d2 type crucible underlay IP fd00:1122:3344:102::25 (unchanged)
+         zone dce226c9-7373-4bfa-8a94-79dc472857a6 type crucible underlay IP fd00:1122:3344:102::27 (unchanged)
+         zone edabedf3-839c-488d-ad6f-508ffa864674 type crucible underlay IP fd00:1122:3344:102::24 (unchanged)
++ sled b59ec570-2abb-4017-80ce-129d94e7a025 (added)
++     zone config generation 2
++         zone 2d73d30e-ca47-46a8-9c12-917d4ab824b6 type internal_ntp underlay IP fd00:1122:3344:104::21 (added)
diff --git a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt
new file mode 100644
index 0000000000..6ff8080165
--- /dev/null
+++ b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt
@@ -0,0 +1,59 @@
+diff blueprint 4171ad05-89dd-474b-846b-b007e4346366 blueprint f432fcd5-1284-4058-8b4a-9286a3de6163
+--- blueprint 4171ad05-89dd-474b-846b-b007e4346366
++++ blueprint f432fcd5-1284-4058-8b4a-9286a3de6163
+  sled 41f45d9f-766e-4ca6-a881-61ee45c80f57
+      zone config generation 2
+         zone 267ed614-92af-4b9d-bdba-c2881c2e43a2 type internal_ntp underlay IP fd00:1122:3344:103::21 (unchanged)
+         zone 322ee9f1-8903-4542-a0a8-a54cefabdeca type crucible underlay IP fd00:1122:3344:103::24 (unchanged)
+         zone 4ab1650f-32c5-447f-939d-64b8103a7645 type crucible underlay IP fd00:1122:3344:103::2a (unchanged)
+         zone 64aa65f8-1ccb-4cd6-9953-027aebdac8ff type crucible underlay IP fd00:1122:3344:103::27 (unchanged)
+         zone 6e811d86-8aa7-4660-935b-84b4b7721b10 type crucible underlay IP fd00:1122:3344:103::2b (unchanged)
+         zone 747d2426-68bf-4c22-8806-41d290b5d5f5 type crucible underlay IP fd00:1122:3344:103::25 (unchanged)
+         zone 7fbd2c38-5dc3-48c4-b061-558a2041d70f type crucible underlay IP fd00:1122:3344:103::2c (unchanged)
+         zone 8e9e923e-62b1-4cbc-9f59-d6397e338b6b type crucible underlay IP fd00:1122:3344:103::29 (unchanged)
+         zone b14d5478-1a0e-4b90-b526-36b06339dfc4 type crucible underlay IP fd00:1122:3344:103::28 (unchanged)
+         zone b40f7c7b-526c-46c8-ae33-67280c280eb7 type crucible underlay IP fd00:1122:3344:103::23 (unchanged)
+         zone be97b92b-38d6-422a-8c76-d37060f75bd2 type crucible underlay IP fd00:1122:3344:103::26 (unchanged)
+         zone cc816cfe-3869-4dde-b596-397d41198628 type nexus underlay IP fd00:1122:3344:103::22 (unchanged)
+  sled 43677374-8d2f-4deb-8a41-eeea506db8e0
+      zone config generation 2
+         zone 02acbe6a-1c88-47e3-94c3-94084cbde098 type crucible underlay IP fd00:1122:3344:101::27 (unchanged)
+         zone 07c3c805-8888-4fe5-9543-3d2479dbe6f3 type crucible underlay IP fd00:1122:3344:101::26 (unchanged)
+         zone 08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb type internal_ntp underlay IP fd00:1122:3344:101::21 (unchanged)
+         zone 10d98a73-ec88-4aff-a7e8-7db6a87880e6 type crucible underlay IP fd00:1122:3344:101::24 (unchanged)
+         zone 2a455c35-eb3c-4c73-ab6c-d0a706e25316 type crucible underlay IP fd00:1122:3344:101::29 (unchanged)
+         zone 3eda924f-22a9-4f3e-9a1b-91d1c47601ab type crucible underlay IP fd00:1122:3344:101::23 (unchanged)
+         zone 587be699-a320-4c79-b320-128d9ecddc0b type crucible underlay IP fd00:1122:3344:101::2b (unchanged)
+         zone 6fa06115-4959-4913-8e7b-dd70d7651f07 type crucible underlay IP fd00:1122:3344:101::2c (unchanged)
+         zone 8f3a1cc5-9195-4a30-ad02-b804278fe639 type crucible underlay IP fd00:1122:3344:101::28 (unchanged)
+         zone a1696cd4-588c-484a-b95b-66e824c0ce05 type crucible underlay IP fd00:1122:3344:101::25 (unchanged)
+         zone a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 type crucible underlay IP fd00:1122:3344:101::2a (unchanged)
+         zone c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 type nexus underlay IP fd00:1122:3344:101::22 (unchanged)
+  sled 590e3034-d946-4166-b0e5-2d0034197a07
+      zone config generation 2
+         zone 18f8fe40-646e-4962-b17a-20e201f3a6e5 type crucible underlay IP fd00:1122:3344:102::2a (unchanged)
+         zone 47199d48-534c-4267-a654-d2d90e64b498 type internal_ntp underlay IP fd00:1122:3344:102::21 (unchanged)
+         zone 56d5d7cf-db2c-40a3-a775-003241ad4820 type crucible underlay IP fd00:1122:3344:102::29 (unchanged)
+         zone 6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 type crucible underlay IP fd00:1122:3344:102::2b (unchanged)
+         zone 704e1fed-f8d6-4cfa-a470-bad27fdc06d1 type nexus underlay IP fd00:1122:3344:102::22 (unchanged)
+         zone 7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 type crucible underlay IP fd00:1122:3344:102::26 (unchanged)
+         zone 93f2f40c-5616-4d8d-8519-ec6debdcede0 type crucible underlay IP fd00:1122:3344:102::2c (unchanged)
+         zone ab7ba6df-d401-40bd-940e-faf57c57aa2a type crucible underlay IP fd00:1122:3344:102::28 (unchanged)
+         zone af322036-371f-437c-8c08-7f40f3f1403b type crucible underlay IP fd00:1122:3344:102::23 (unchanged)
+         zone d637264f-6f40-44c2-8b7e-a179430210d2 type crucible underlay IP fd00:1122:3344:102::25 (unchanged)
+         zone dce226c9-7373-4bfa-8a94-79dc472857a6 type crucible underlay IP fd00:1122:3344:102::27 (unchanged)
+         zone edabedf3-839c-488d-ad6f-508ffa864674 type crucible underlay IP fd00:1122:3344:102::24 (unchanged)
+  sled b59ec570-2abb-4017-80ce-129d94e7a025
+-     zone config generation 2
++     zone config generation 3
+         zone 2d73d30e-ca47-46a8-9c12-917d4ab824b6 type internal_ntp underlay IP fd00:1122:3344:104::21 (unchanged)
++        zone 1a20ee3c-f66e-4fca-ab85-2a248aa3d79d type crucible underlay IP fd00:1122:3344:104::2b (added)
++        zone 28852beb-d0e5-4cba-9adb-e7f0cd4bb864 type crucible underlay IP fd00:1122:3344:104::29 (added)
++        zone 45556184-7092-4a3d-873f-637976bb133b type crucible underlay IP fd00:1122:3344:104::22 (added)
++        zone 8215bf7a-10d6-4f40-aeb7-27a196307c37 type crucible underlay IP fd00:1122:3344:104::25 (added)
++        zone 9d75abfe-47ab-434a-93dd-af50dc0dddde type crucible underlay IP fd00:1122:3344:104::23 (added)
++        zone a36d291c-7f68-462f-830e-bc29e5841ce2 type crucible underlay IP fd00:1122:3344:104::27 (added)
++        zone b3a4d434-aaee-4752-8c99-69d88fbcb8c5 type crucible underlay IP fd00:1122:3344:104::2a (added)
++        zone cf5b636b-a505-4db6-bc32-baf9f53f4371 type crucible underlay IP fd00:1122:3344:104::28 (added)
++        zone f6125d45-b9cc-4721-ba60-ed4dbb177e41 type crucible underlay IP fd00:1122:3344:104::26 (added)
++        zone f86e19d2-9145-41cf-be89-6aaa34a73873 type crucible underlay IP fd00:1122:3344:104::24 (added)
diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt
new file mode 100644
index 0000000000..b63a92ad94
--- /dev/null
+++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt
@@ -0,0 +1,86 @@
+diff blueprint 55502b1b-e255-438b-a16a-2680a4b5f962 blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6
+--- blueprint 55502b1b-e255-438b-a16a-2680a4b5f962
++++ blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6
+  sled 2d1cb4f2-cf44-40fc-b118-85036eb732a9
+      zone config generation 2
+         zone 19fbc4f8-a683-4f22-8f5a-e74782b935be type crucible underlay IP fd00:1122:3344:105::26 (unchanged)
+         zone 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 type crucible underlay IP fd00:1122:3344:105::2c (unchanged)
+         zone 6b53ab2e-d98c-485f-87a3-4d5df595390f type crucible underlay IP fd00:1122:3344:105::27 (unchanged)
+         zone 6dff7633-66bb-4924-a6ff-2c896e66964b type nexus underlay IP fd00:1122:3344:105::22 (unchanged)
+         zone 7f4e9f9f-08f8-4d14-885d-e977c05525ad type internal_ntp underlay IP fd00:1122:3344:105::21 (unchanged)
+         zone 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 type crucible underlay IP fd00:1122:3344:105::23 (unchanged)
+         zone 9f0abbad-dbd3-4d43-9675-78092217ffd9 type crucible underlay IP fd00:1122:3344:105::25 (unchanged)
+         zone b0c63f48-01ea-4aae-bb26-fb0dd59d1662 type crucible underlay IP fd00:1122:3344:105::28 (unchanged)
+         zone c406da50-34b9-4bb4-a460-8f49875d2a6a type crucible underlay IP fd00:1122:3344:105::24 (unchanged)
+         zone d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 type crucible underlay IP fd00:1122:3344:105::2a (unchanged)
+         zone e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 type crucible underlay IP fd00:1122:3344:105::2b (unchanged)
+         zone f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 type crucible underlay IP fd00:1122:3344:105::29 (unchanged)
+  sled 48d95fef-bc9f-4f50-9a53-1e075836291d
+      zone config generation 2
+         zone 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 type crucible underlay IP fd00:1122:3344:103::2c (unchanged)
+         zone 0dcfdfc5-481e-4153-b97c-11cf02b648ea type crucible underlay IP fd00:1122:3344:103::25 (unchanged)
+         zone 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb type nexus underlay IP fd00:1122:3344:103::22 (unchanged)
+         zone 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f type crucible underlay IP fd00:1122:3344:103::27 (unchanged)
+         zone 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 type crucible underlay IP fd00:1122:3344:103::28 (unchanged)
+         zone 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb type crucible underlay IP fd00:1122:3344:103::24 (unchanged)
+         zone 67622d61-2df4-414d-aa0e-d1277265f405 type crucible underlay IP fd00:1122:3344:103::23 (unchanged)
+         zone 67d913e0-0005-4599-9b28-0abbf6cc2916 type internal_ntp underlay IP fd00:1122:3344:103::21 (unchanged)
+         zone b91b271d-8d80-4f49-99a0-34006ae86063 type crucible underlay IP fd00:1122:3344:103::2a (unchanged)
+         zone d6ee1338-3127-43ec-9aaa-b973ccf05496 type crucible underlay IP fd00:1122:3344:103::26 (unchanged)
+         zone e39d7c9e-182b-48af-af87-58079d723583 type crucible underlay IP fd00:1122:3344:103::29 (unchanged)
+         zone f69f92a1-5007-4bb0-a85b-604dc217154b type crucible underlay IP fd00:1122:3344:103::2b (unchanged)
+  sled 68d24ac5-f341-49ea-a92a-0381b52ab387
+      zone config generation 2
+         zone 01d58626-e1b0-480f-96be-ac784863c7dc type nexus underlay IP fd00:1122:3344:102::22 (unchanged)
+         zone 3b3c14b6-a8e2-4054-a577-8d96cb576230 type crucible underlay IP fd00:1122:3344:102::2c (unchanged)
+         zone 47a87c6e-ef45-4d52-9a3e-69cdd96737cc type crucible underlay IP fd00:1122:3344:102::23 (unchanged)
+         zone 6464d025-4652-4948-919e-740bec5699b1 type crucible underlay IP fd00:1122:3344:102::24 (unchanged)
+         zone 6939ce48-b17c-4616-b176-8a419a7697be type crucible underlay IP fd00:1122:3344:102::29 (unchanged)
+         zone 878dfddd-3113-4197-a3ea-e0d4dbe9b476 type crucible underlay IP fd00:1122:3344:102::25 (unchanged)
+         zone 8d4d2b28-82bb-4e36-80da-1408d8c35d82 type crucible underlay IP fd00:1122:3344:102::2b (unchanged)
+         zone 9fd52961-426f-4e62-a644-b70871103fca type crucible underlay IP fd00:1122:3344:102::26 (unchanged)
+         zone b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 type crucible underlay IP fd00:1122:3344:102::27 (unchanged)
+         zone b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 type crucible underlay IP fd00:1122:3344:102::28 (unchanged)
+         zone c407795c-6c8b-428e-8ab8-b962913c447f type crucible underlay IP fd00:1122:3344:102::2a (unchanged)
+         zone f3f2e4f3-0985-4ef6-8336-ce479382d05d type internal_ntp underlay IP fd00:1122:3344:102::21 (unchanged)
+  sled 75bc286f-2b4b-482c-9431-59272af529da
+-     zone config generation 2
++     zone config generation 3
+         zone 15bb9def-69b8-4d2e-b04f-9fee1143387c type crucible underlay IP fd00:1122:3344:104::25 (unchanged)
+         zone 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c type crucible underlay IP fd00:1122:3344:104::2c (unchanged)
+         zone 57b96d5c-b71e-43e4-8869-7d514003d00d type internal_ntp underlay IP fd00:1122:3344:104::21 (unchanged)
+         zone 621509d6-3772-4009-aca1-35eefd1098fb type crucible underlay IP fd00:1122:3344:104::28 (unchanged)
+         zone 85b8c68a-160d-461d-94dd-1baf175fa75c type crucible underlay IP fd00:1122:3344:104::2a (unchanged)
+         zone 996d7570-b0df-46d5-aaa4-0c97697cf484 type crucible underlay IP fd00:1122:3344:104::26 (unchanged)
+         zone a732c489-d29a-4f75-b900-5966385943af type crucible underlay IP fd00:1122:3344:104::29 (unchanged)
+         zone b1783e95-9598-451d-b6ba-c50b52b428c3 type crucible underlay IP fd00:1122:3344:104::24 (unchanged)
+         zone b4947d31-f70e-4ee0-8817-0ca6cea9b16b type nexus underlay IP fd00:1122:3344:104::22 (unchanged)
+         zone c6dd531e-2d1d-423b-acc8-358533dab78c type crucible underlay IP fd00:1122:3344:104::27 (unchanged)
+         zone e4b3e159-3dbe-48cb-8497-e3da92a90e5a type crucible underlay IP fd00:1122:3344:104::23 (unchanged)
+         zone f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 type crucible underlay IP fd00:1122:3344:104::2b (unchanged)
++        zone 2ec75441-3d7d-4b4b-9614-af03de5a3666 type nexus underlay IP fd00:1122:3344:104::2d (added)
++        zone 3ca5292f-8a59-4475-bb72-0f43714d0fff type nexus underlay IP fd00:1122:3344:104::31 (added)
++        zone 508abd03-cbfe-4654-9a6d-7f15a1ad32e5 type nexus underlay IP fd00:1122:3344:104::2e (added)
++        zone 59950bc8-1497-44dd-8cbf-b6502ba921b2 type nexus underlay IP fd00:1122:3344:104::2f (added)
++        zone 99f6d544-8599-4e2b-a55a-82d9e0034662 type nexus underlay IP fd00:1122:3344:104::30 (added)
++        zone c26b3bda-5561-44a1-a69f-22103fe209a1 type nexus underlay IP fd00:1122:3344:104::32 (added)
+  sled affab35f-600a-4109-8ea0-34a067a4e0bc
+-     zone config generation 2
++     zone config generation 3
+         zone 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 type crucible underlay IP fd00:1122:3344:101::27 (unchanged)
+         zone 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 type nexus underlay IP fd00:1122:3344:101::22 (unchanged)
+         zone 3aa07966-5899-4789-ace5-f8eeb375c6c3 type crucible underlay IP fd00:1122:3344:101::24 (unchanged)
+         zone 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf type crucible underlay IP fd00:1122:3344:101::29 (unchanged)
+         zone 72c5a909-077d-4ec1-a9d5-ae64ef9d716e type crucible underlay IP fd00:1122:3344:101::26 (unchanged)
+         zone 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 type crucible underlay IP fd00:1122:3344:101::23 (unchanged)
+         zone a1c03689-fc62-4ea5-bb72-4d01f5138614 type crucible underlay IP fd00:1122:3344:101::2a (unchanged)
+         zone a568e92e-4fbd-4b69-acd8-f16277073031 type crucible underlay IP fd00:1122:3344:101::2c (unchanged)
+         zone bf79a56a-97af-4cc4-94a5-8b20d64c2cda type crucible underlay IP fd00:1122:3344:101::28 (unchanged)
+         zone c60379ba-4e30-4628-a79a-0ae509aef4c5 type crucible underlay IP fd00:1122:3344:101::25 (unchanged)
+         zone d47f4996-fac0-4657-bcea-01b1fee6404d type crucible underlay IP fd00:1122:3344:101::2b (unchanged)
+         zone f1a7b9a7-fc6a-4b23-b829-045ff33117ff type internal_ntp underlay IP fd00:1122:3344:101::21 (unchanged)
++        zone 6f86d5cb-17d7-424b-9d4c-39f670532cbe type nexus underlay IP fd00:1122:3344:101::2e (added)
++        zone 87c299eb-470e-4b6d-b8c7-6759694e66b6 type nexus underlay IP fd00:1122:3344:101::30 (added)
++        zone c72b7930-0580-4f00-93b9-8cba2c8d344e type nexus underlay IP fd00:1122:3344:101::2d (added)
++        zone d0095508-bdb8-4faf-b091-964276a20b15 type nexus underlay IP fd00:1122:3344:101::31 (added)
++        zone ff422442-4b31-4ade-a11a-9e5a25f0404c type nexus underlay IP fd00:1122:3344:101::2f (added)
diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs
index 7dcb843b7f..7054a3ea15 100644
--- a/nexus/types/src/deployment.rs
+++ b/nexus/types/src/deployment.rs
@@ -133,7 +133,7 @@ impl SledResources {
 // zones deployed on each host and some supporting configuration (e.g., DNS).
 // This is aimed at supporting add/remove sleds.  The plan is to grow this to
 // include more of the system as we support more use cases.
-#[derive(Clone, Eq, PartialEq, JsonSchema, Deserialize, Serialize)]
+#[derive(Clone, Debug, Eq, PartialEq, JsonSchema, Deserialize, Serialize)]
 pub struct Blueprint {
     /// unique identifier for this blueprint
     pub id: Uuid,
@@ -222,23 +222,38 @@ impl Blueprint {
             after_zones_in_service: &self.zones_in_service,
         }
     }
+
+    /// Return a struct that can be displayed.
+    pub fn display(&self) -> BlueprintDisplay<'_> {
+        BlueprintDisplay { blueprint: self }
+    }
 }
 
-impl fmt::Debug for Blueprint {
+/// Wrapper to allow a [`Blueprint`] to be displayed with information.
+///
+/// Returned by [`Blueprint::display()`].
+#[derive(Clone, Debug)]
+pub struct BlueprintDisplay<'a> {
+    blueprint: &'a Blueprint,
+    // TODO: add colorization with a stylesheet
+}
+
+impl<'a> fmt::Display for BlueprintDisplay<'a> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        writeln!(f, "blueprint  {}", self.id)?;
+        let b = self.blueprint;
+        writeln!(f, "blueprint  {}", b.id)?;
         writeln!(
             f,
             "parent:    {}",
-            self.parent_blueprint_id
+            b.parent_blueprint_id
                 .map(|u| u.to_string())
                 .unwrap_or_else(|| String::from("<none>"))
         )?;
         writeln!(
             f,
             "created by {}{}",
-            self.creator,
-            if self.creator.parse::<Uuid>().is_ok() {
+            b.creator,
+            if b.creator.parse::<Uuid>().is_ok() {
                 " (likely a Nexus instance)"
             } else {
                 ""
@@ -247,12 +262,12 @@ impl fmt::Debug for Blueprint {
         writeln!(
             f,
             "created at {}",
-            humantime::format_rfc3339_millis(self.time_created.into(),)
+            humantime::format_rfc3339_millis(b.time_created.into(),)
         )?;
-        writeln!(f, "internal DNS version: {}", self.internal_dns_version)?;
-        writeln!(f, "comment: {}", self.comment)?;
+        writeln!(f, "internal DNS version: {}", b.internal_dns_version)?;
+        writeln!(f, "comment: {}", b.comment)?;
         writeln!(f, "zones:\n")?;
-        for (sled_id, sled_zones) in &self.omicron_zones {
+        for (sled_id, sled_zones) in &b.omicron_zones {
             writeln!(
                 f,
                 "  sled {}: Omicron zones at generation {}",
@@ -263,7 +278,7 @@ impl fmt::Debug for Blueprint {
                     f,
                     "    {} {} {}",
                     z.id,
-                    if self.zones_in_service.contains(&z.id) {
+                    if b.zones_in_service.contains(&z.id) {
                         "in service    "
                     } else {
                         "not in service"
@@ -322,6 +337,7 @@ pub struct BlueprintTargetSet {
 }
 
 /// Summarizes the differences between two blueprints
+#[derive(Debug)]
 pub struct OmicronZonesDiff<'a> {
     before_label: String,
     // We store an owned copy of "before_zones" to make it easier to support
@@ -544,6 +560,25 @@ impl<'a> OmicronZonesDiff<'a> {
         })
     }
 
+    /// Return a struct that can be used to display the diff in a
+    /// unified `diff(1)`-like format.
+    pub fn display(&self) -> OmicronZonesDiffDisplay<'_, 'a> {
+        OmicronZonesDiffDisplay::new(self)
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct OmicronZonesDiffDisplay<'diff, 'a> {
+    diff: &'diff OmicronZonesDiff<'a>,
+    // TODO: add colorization with a stylesheet
+}
+
+impl<'diff, 'a> OmicronZonesDiffDisplay<'diff, 'a> {
+    #[inline]
+    fn new(diff: &'diff OmicronZonesDiff<'a>) -> Self {
+        Self { diff }
+    }
+
     fn print_whole_sled(
         &self,
         f: &mut fmt::Formatter<'_>,
@@ -574,18 +609,18 @@ impl<'a> OmicronZonesDiff<'a> {
     }
 }
 
-/// Implements diff(1)-like output for diff'ing two blueprints
-impl<'a> fmt::Display for OmicronZonesDiff<'a> {
+impl<'diff, 'a> fmt::Display for OmicronZonesDiffDisplay<'diff, 'a> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        writeln!(f, "diff {} {}", self.before_label, self.after_label)?;
-        writeln!(f, "--- {}", self.before_label)?;
-        writeln!(f, "+++ {}", self.after_label)?;
+        let diff = self.diff;
+        writeln!(f, "diff {} {}", diff.before_label, diff.after_label)?;
+        writeln!(f, "--- {}", diff.before_label)?;
+        writeln!(f, "+++ {}", diff.after_label)?;
 
-        for (sled_id, sled_zones) in self.sleds_removed() {
+        for (sled_id, sled_zones) in diff.sleds_removed() {
             self.print_whole_sled(f, '-', "removed", sled_zones, sled_id)?;
         }
 
-        for (sled_id, sled_changes) in self.sleds_in_common() {
+        for (sled_id, sled_changes) in diff.sleds_in_common() {
             // Print a line about the sled itself and zone config generation,
             // regardless of whether anything has changed.
             writeln!(f, "  sled {}", sled_id)?;
@@ -700,7 +735,7 @@ impl<'a> fmt::Display for OmicronZonesDiff<'a> {
             }
         }
 
-        for (sled_id, sled_zones) in self.sleds_added() {
+        for (sled_id, sled_zones) in diff.sleds_added() {
             self.print_whole_sled(f, '+', "added", sled_zones, sled_id)?;
         }
 

From b818c5d3a2d4433ff944c9fb96cfd2bbbc6e91de Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Wed, 13 Mar 2024 16:55:58 -0400
Subject: [PATCH 108/157] Revert "fix(deps): update rust crate vsss-rs to 3.4.0
 (#5115)" (#5255)

This reverts commit 6d44cf529c8821e7fc107fe3b15bc2377ba96872.

Hopefully (?) fixes #5254 (@andrewjstone noticed the version bump when I
pinged him on the issue). Cc @sunshowers
---
 Cargo.lock           | 8 ++++----
 bootstore/Cargo.toml | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 9acd070208..98b24ff3ac 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1454,9 +1454,9 @@ dependencies = [
 
 [[package]]
 name = "curve25519-dalek"
-version = "4.1.2"
+version = "4.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348"
+checksum = "e89b8c6a2e4b1f45971ad09761aafb85514a84744b67a95e32c3cc1352d1f65c"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -10322,9 +10322,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "vsss-rs"
-version = "3.4.0"
+version = "3.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14d9b5a415d1bbbf1a9c5982babc0d8054fa10effe9c5b7086cd2fdd36be79d9"
+checksum = "196bbee60607a195bc850e94f0e040bd090e45794ad8df0e9c5a422b9975a00f"
 dependencies = [
  "curve25519-dalek",
  "elliptic-curve",
diff --git a/bootstore/Cargo.toml b/bootstore/Cargo.toml
index 97db1d52b7..5e9bcd1ef4 100644
--- a/bootstore/Cargo.toml
+++ b/bootstore/Cargo.toml
@@ -27,7 +27,7 @@ slog.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
 uuid.workspace = true
-vsss-rs = { version = "3.4.0", features = ["std", "curve25519"] }
+vsss-rs = { version = "3.3.4", features = ["std", "curve25519"] }
 zeroize.workspace = true
 
 # See omicron-rpaths for more about the "pq-sys" dependency.

From cf422c4211dda8f21cb41907d5b3bea3bfe79e25 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Wed, 13 Mar 2024 14:05:55 -0700
Subject: [PATCH 109/157] Report physical disks through the inventory system
 (#5145)

- In Sled Agent, send info about physical storage up to Nexus
- In Nexus, process physical disks and add them to inventory in a new
`inv_physical_disk` table
- This PR also adjusts inventory tests to account for this new "physical
disk" information being added alongside sleds, and adds some disk to the
"representative inventory data"

Fixes https://github.com/oxidecomputer/omicron/issues/5150
---
 clients/sled-agent-client/src/lib.rs          |    10 +
 common/src/disk.rs                            |    16 +-
 .../tests/input/complex.json                  | 12851 ++++++----------
 .../tests/output/cmd-complex-stdout           |   552 +-
 nexus/db-model/src/inventory.rs               |    52 +-
 nexus/db-model/src/schema.rs                  |    16 +-
 .../db-queries/src/db/datastore/inventory.rs  |    90 +
 nexus/inventory/src/builder.rs                |     6 +
 nexus/inventory/src/examples.rs               |    48 +
 nexus/reconfigurator/planning/src/system.rs   |     4 +-
 nexus/types/src/external_api/params.rs        |    10 +
 nexus/types/src/inventory.rs                  |    24 +
 openapi/sled-agent.json                       |    55 +
 schema/crdb/41.0.0/up01.sql                   |    14 +
 schema/crdb/dbinit.sql                        |    26 +-
 sled-agent/src/http_entrypoints.rs            |     2 +-
 sled-agent/src/long_running_tasks.rs          |     5 +-
 sled-agent/src/params.rs                      |     9 +
 sled-agent/src/services.rs                    |     4 +-
 sled-agent/src/sim/http_entrypoints.rs        |     1 +
 sled-agent/src/sim/sled_agent.rs              |    20 +-
 sled-agent/src/sim/storage.rs                 |    37 +-
 sled-agent/src/sled_agent.rs                  |    15 +-
 sled-agent/src/zone_bundle.rs                 |     4 +-
 sled-hardware/src/disk.rs                     |     7 +-
 sled-storage/src/disk.rs                      |    22 +-
 sled-storage/src/manager.rs                   |    28 +-
 27 files changed, 5691 insertions(+), 8237 deletions(-)
 create mode 100644 schema/crdb/41.0.0/up01.sql

diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs
index b2bc232ef5..5303a7bafd 100644
--- a/clients/sled-agent-client/src/lib.rs
+++ b/clients/sled-agent-client/src/lib.rs
@@ -167,6 +167,16 @@ impl omicron_common::api::external::ClientError for types::Error {
     }
 }
 
+impl From<types::DiskIdentity> for omicron_common::disk::DiskIdentity {
+    fn from(identity: types::DiskIdentity) -> Self {
+        Self {
+            vendor: identity.vendor,
+            serial: identity.serial,
+            model: identity.model,
+        }
+    }
+}
+
 impl From<omicron_common::api::internal::nexus::InstanceRuntimeState>
     for types::InstanceRuntimeState
 {
diff --git a/common/src/disk.rs b/common/src/disk.rs
index 3ae9c31e01..0cf9b6e073 100644
--- a/common/src/disk.rs
+++ b/common/src/disk.rs
@@ -4,8 +4,22 @@
 
 //! Disk related types shared among crates
 
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
 /// Uniquely identifies a disk.
-#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)]
+#[derive(
+    Debug,
+    Clone,
+    PartialEq,
+    Eq,
+    Hash,
+    Ord,
+    PartialOrd,
+    Serialize,
+    Deserialize,
+    JsonSchema,
+)]
 pub struct DiskIdentity {
     pub vendor: String,
     pub serial: String,
diff --git a/dev-tools/reconfigurator-cli/tests/input/complex.json b/dev-tools/reconfigurator-cli/tests/input/complex.json
index c168e153c1..5bdba202d3 100644
--- a/dev-tools/reconfigurator-cli/tests/input/complex.json
+++ b/dev-tools/reconfigurator-cli/tests/input/complex.json
@@ -1,380 +1,270 @@
 {
   "policy": {
     "sleds": {
-      "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
+      "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
         "policy": {
           "kind": "in_service",
           "provision_policy": "provisionable"
         },
         "state": "active",
         "zpools": [
-          "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0",
-          "oxp_1d864940-c723-4f58-82e8-b03772b42356",
-          "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698",
-          "oxp_466351db-2ced-45fa-9431-22de1c4bd480",
-          "oxp_88d39560-b091-4fad-bad0-b0b1c514033b",
-          "oxp_88ddd98f-3181-4797-864a-7e8d5b028657",
-          "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d",
-          "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c",
-          "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+          "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03",
+          "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03",
+          "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b",
+          "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03",
+          "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
         ],
         "subnet": {
-          "net": "fd00:1122:3344:101::/64"
+          "net": "fd00:1122:3344:103::/64"
         }
       },
-      "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
+      "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
         "policy": {
           "kind": "in_service",
           "provision_policy": "provisionable"
         },
         "state": "active",
         "zpools": [
-          "oxp_0622b2cf-297e-4356-b94c-d74309f443ba",
-          "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37",
-          "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1",
-          "oxp_574728ca-1559-48f3-a46c-1e608966e6a0",
-          "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945",
-          "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c",
-          "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3",
-          "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
+          "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03",
+          "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03",
+          "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b",
+          "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
+          "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
         ],
         "subnet": {
-          "net": "fd00:1122:3344:121::/64"
+          "net": "fd00:1122:3344:104::/64"
         }
       },
-      "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
+      "77851655-10a5-4e80-88b0-a001473cab7e": {
         "policy": {
           "kind": "in_service",
           "provision_policy": "provisionable"
         },
         "state": "active",
         "zpools": [
-          "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd",
-          "oxp_0c07bcc1-7666-4754-881d-4135992ff561",
-          "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9",
-          "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae",
-          "oxp_6a275151-d85f-488b-81a6-e558fdede658",
-          "oxp_742699b5-4ded-406d-9a08-24648d3b2efb",
-          "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9",
-          "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42",
-          "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c",
-          "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
+          "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03",
+          "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03",
+          "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b",
+          "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03",
+          "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
         ],
         "subnet": {
           "net": "fd00:1122:3344:102::/64"
         }
       },
-      "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
+      "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
         "policy": {
           "kind": "in_service",
           "provision_policy": "provisionable"
         },
         "state": "active",
         "zpools": [
-          "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98",
-          "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960",
-          "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea",
-          "oxp_6aa59516-2018-4494-af6b-46704b20f6dc",
-          "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3",
-          "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3",
-          "oxp_c8feed09-6556-4409-af8e-563c97d556eb",
-          "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a",
-          "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62",
-          "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+          "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03",
+          "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03",
+          "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b",
+          "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03",
+          "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
         ],
         "subnet": {
-          "net": "fd00:1122:3344:103::/64"
+          "net": "fd00:1122:3344:101::/64"
         }
       }
     },
     "service_ip_pool_ranges": [
       {
-        "first": "172.20.28.1",
-        "last": "172.20.28.10"
+        "first": "198.51.100.20",
+        "last": "198.51.100.29"
       }
     ],
     "target_nexus_zone_count": 3
   },
   "collections": [
     {
-      "id": "83ed3949-d221-44f3-a901-02e9ff16d2a5",
-      "errors": [
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d7328126-995e-437c-b92e-c6c220d508b2\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"d7328126-995e-437c-b92e-c6c220d508b2\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a94be62-a0b1-4c63-b54c-7083266de0ab\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"6a94be62-a0b1-4c63-b54c-7083266de0ab\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"858a0aad-c3df-4cdf-9c20-be6b1effdb20\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"858a0aad-c3df-4cdf-9c20-be6b1effdb20\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"40a5f1ae-5e03-41ec-8c45-6b8a83ce7ad4\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"40a5f1ae-5e03-41ec-8c45-6b8a83ce7ad4\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fe9383e3-5b7c-45f2-bec6-c1f5df8caaa4\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"fe9383e3-5b7c-45f2-bec6-c1f5df8caaa4\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9c467503-9155-4d79-b048-aea0c4153f9e\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"9c467503-9155-4d79-b048-aea0c4153f9e\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d397fe11-ec68-4a1f-9739-e42313158ae4\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"d397fe11-ec68-4a1f-9739-e42313158ae4\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"db83ac82-ec73-4914-adb7-6a243409702c\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"db83ac82-ec73-4914-adb7-6a243409702c\" }"
-      ],
-      "time_started": "2024-03-01T19:17:06.797656Z",
-      "time_done": "2024-03-01T19:17:10.594988Z",
-      "collector": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
+      "id": "3f61b723-cfe7-40ec-b22c-e3e1f35325f9",
+      "errors": [],
+      "time_started": "2024-03-12T18:59:34.196825Z",
+      "time_done": "2024-03-12T18:59:34.397868Z",
+      "collector": "0d459323-e414-4f32-9944-76c7331da622",
       "baseboards": [
         {
-          "part_number": "913-0000006",
-          "serial_number": "BRM44220005"
-        },
-        {
-          "part_number": "913-0000006",
-          "serial_number": "BRM44220011"
+          "part_number": "FAKE_SIM_SIDECAR",
+          "serial_number": "SimSidecar0"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220004"
+          "part_number": "i86pc",
+          "serial_number": "g0"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220046"
+          "part_number": "i86pc",
+          "serial_number": "g1"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220081"
+          "part_number": "i86pc",
+          "serial_number": "g2"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM44220001"
+          "part_number": "i86pc",
+          "serial_number": "g3"
         }
       ],
       "cabooses": [
         {
-          "board": "gimlet-c",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "gimlet-c",
-          "version": "1.0.6"
-        },
-        {
-          "board": "gimlet-c",
-          "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
-          "name": "gimlet-c",
-          "version": "1.0.5"
-        },
-        {
-          "board": "gimlet-c",
-          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-          "name": "gimlet-c",
-          "version": "1.0.8"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "oxide-rot-1",
-          "version": "1.0.5"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
-          "name": "oxide-rot-1",
-          "version": "1.0.4"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
-          "name": "oxide-rot-1",
-          "version": "0.0.0-git"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-          "name": "oxide-rot-1",
-          "version": "1.0.6"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-          "name": "oxide-rot-1",
-          "version": "1.0.0"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-          "name": "oxide-rot-1-dev",
-          "version": "0.0.0-git"
+          "board": "SimGimletSp",
+          "git_commit": "ffffffff",
+          "name": "SimGimlet",
+          "version": "0.0.1"
         },
         {
-          "board": "sidecar-b",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "sidecar-b",
-          "version": "1.0.6"
+          "board": "SimRot",
+          "git_commit": "eeeeeeee",
+          "name": "SimGimlet",
+          "version": "0.0.1"
         },
         {
-          "board": "sidecar-b",
-          "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
-          "name": "sidecar-b",
-          "version": "0.0.0-git"
+          "board": "SimRot",
+          "git_commit": "eeeeeeee",
+          "name": "SimSidecar",
+          "version": "0.0.1"
         },
         {
-          "board": "sidecar-b",
-          "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
-          "name": "sidecar-b",
-          "version": "0.0.0-git"
+          "board": "SimSidecarSp",
+          "git_commit": "ffffffff",
+          "name": "SimSidecar",
+          "version": "0.0.1"
         }
       ],
       "rot_pages": [
         {
-          "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+          "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+        },
+        {
+          "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+          "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+          "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+          "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+          "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+          "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+          "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         }
       ],
       "sps": [
         [
           {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220005"
-          },
-          {
-            "time_collected": "2024-03-01T19:17:09.327993Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "sp_type": "switch",
-            "sp_slot": 0,
-            "baseboard_revision": 4,
-            "hubris_archive": "9184be93590e655e",
-            "power_state": "A0"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220011"
+            "part_number": "FAKE_SIM_SIDECAR",
+            "serial_number": "SimSidecar0"
           },
           {
-            "time_collected": "2024-03-01T19:17:09.913395Z",
-            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "time_collected": "2024-03-12T18:59:34.203615Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "switch",
             "sp_slot": 1,
-            "baseboard_revision": 4,
-            "hubris_archive": "9184be93590e655e",
-            "power_state": "A0"
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
+            "part_number": "i86pc",
+            "serial_number": "g0"
           },
           {
-            "time_collected": "2024-03-01T19:17:08.168996Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T18:59:34.208589Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 17,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
+            "sp_slot": 0,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
+            "part_number": "i86pc",
+            "serial_number": "g1"
           },
           {
-            "time_collected": "2024-03-01T19:17:08.746433Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T18:59:34.214982Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 15,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
+            "sp_slot": 1,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
+            "part_number": "i86pc",
+            "serial_number": "g2"
           },
           {
-            "time_collected": "2024-03-01T19:17:07.575660Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T18:59:34.220269Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 14,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
+            "sp_slot": 2,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
+            "part_number": "i86pc",
+            "serial_number": "g3"
           },
           {
-            "time_collected": "2024-03-01T19:17:06.810258Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T18:59:34.198632Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 16,
-            "baseboard_revision": 6,
-            "hubris_archive": "3c670b44fdeef4af",
-            "power_state": "A0"
+            "sp_slot": 3,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ]
       ],
       "rots": [
         [
           {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220005"
+            "part_number": "FAKE_SIM_SIDECAR",
+            "serial_number": "SimSidecar0"
           },
           {
-            "time_collected": "2024-03-01T19:17:09.327993Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T18:59:34.203615Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
             "persistent_boot_preference": {
-              "slot": "b"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "4599e4d7bc9b2164e8c0f0f7ce4bcae2bd35d635c8450854ea9a94ce49a2ca0f",
-            "slot_b_sha3_256_digest": "1e42a0761e5bd1d9f9a5e2c44acc34e7b7e007792977cdb0523d271db2e80c8c"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220011"
-          },
-          {
-            "time_collected": "2024-03-01T19:17:09.913395Z",
-            "source": "http://[fd00:1122:3344:103::2]:12225",
-            "active_slot": {
-              "slot": "b"
-            },
-            "persistent_boot_preference": {
-              "slot": "b"
+              "slot": "a"
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "92e07077972c83af6872ec817fb88d74a36f6956276505e9771302c5bb7ec2a8",
-            "slot_b_sha3_256_digest": "fb25825eaf97f8f2307afe3caebf0b70865499f6d116db3aff382c505b4d8dd7"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
+            "part_number": "i86pc",
+            "serial_number": "g0"
           },
           {
-            "time_collected": "2024-03-01T19:17:08.168996Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T18:59:34.208589Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
@@ -383,18 +273,18 @@
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
-            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
+            "part_number": "i86pc",
+            "serial_number": "g1"
           },
           {
-            "time_collected": "2024-03-01T19:17:08.746433Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T18:59:34.214982Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
@@ -403,38 +293,38 @@
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
-            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
+            "part_number": "i86pc",
+            "serial_number": "g2"
           },
           {
-            "time_collected": "2024-03-01T19:17:07.575660Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T18:59:34.220269Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
-              "slot": "b"
+              "slot": "a"
             },
             "persistent_boot_preference": {
-              "slot": "b"
+              "slot": "a"
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
-            "slot_b_sha3_256_digest": "d416d5ba9d0d3bd495f0f5fcd2aecc28cd83b53d1a8f94a28241d85b30761451"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
+            "part_number": "i86pc",
+            "serial_number": "g3"
           },
           {
-            "time_collected": "2024-03-01T19:17:06.810258Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T18:59:34.198632Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
@@ -443,8 +333,8 @@
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
-            "slot_b_sha3_256_digest": "4ec328195a23e86a1545d5ee576e73834aae81fb79830a54d0503475875f1760"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ]
       ],
@@ -452,97 +342,81 @@
         "SpSlot0": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:17:09.329382Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "sidecar-b",
-                "version": "1.0.6"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.915072Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.204198Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "sidecar-b",
-                "version": "1.0.6"
+                "board": "SimSidecarSp",
+                "git_commit": "ffffffff",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.170364Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.209189Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.751437Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.215708Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.577731Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.220965Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:06.811674Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.199229Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ]
@@ -550,97 +424,81 @@
         "SpSlot1": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:17:09.330756Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
-                "name": "sidecar-b",
-                "version": "0.0.0-git"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.916864Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.204765Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
-                "name": "sidecar-b",
-                "version": "0.0.0-git"
+                "board": "SimSidecarSp",
+                "git_commit": "ffffffff",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.172087Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.209887Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.752817Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.216379Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.579882Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.221612Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:06.813085Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.199808Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
-                "name": "gimlet-c",
-                "version": "1.0.5"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ]
@@ -648,97 +506,81 @@
         "RotSlotA": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:17:09.537712Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-                "name": "oxide-rot-1-dev",
-                "version": "0.0.0-git"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:17:10.123027Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.205525Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
-                "name": "oxide-rot-1",
-                "version": "0.0.0-git"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.423630Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.210499Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.004153Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.217080Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.831036Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.222264Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.074158Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.200511Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ]
@@ -746,153 +588,150 @@
         "RotSlotB": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.744584Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.206173Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:10.331006Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.211127Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-                "name": "oxide-rot-1",
-                "version": "1.0.0"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.675629Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.217758Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.256102Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.222982Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.090096Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.201220Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
-          ],
+          ]
+        ]
+      },
+      "rot_pages_found": {
+        "Cmpa": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.327673Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
-                "name": "oxide-rot-1",
-                "version": "1.0.4"
+              "time_collected": "2024-03-12T18:59:34.206630Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
-          ]
-        ]
-      },
-      "rot_pages_found": {
-        "Cmpa": [
+          ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.691258Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.211561Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.272635Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.218199Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.107097Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.223472Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.344150Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.201705Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
@@ -900,53 +739,66 @@
         "CfpaActive": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
+            },
+            {
+              "time_collected": "2024-03-12T18:59:34.207099Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.707253Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.212036Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.288646Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.218655Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.123462Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.223978Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.416190Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.202177Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
@@ -954,53 +806,66 @@
         "CfpaInactive": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.723232Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.207582Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+                "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.304669Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.212533Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.139558Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.219102Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.487114Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.224471Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "i86pc",
+              "serial_number": "g3"
+            },
+            {
+              "time_collected": "2024-03-12T18:59:34.202661Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
@@ -1008,910 +873,1010 @@
         "CfpaScratch": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
+            },
+            {
+              "time_collected": "2024-03-12T18:59:34.208087Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.739167Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.213042Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.320638Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.219676Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.162584Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.224951Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.568207Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T18:59:34.203171Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
         ]
       },
       "sled_agents": {
-        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
-          "time_collected": "2024-03-01T19:17:10.485528Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
-          },
-          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
-        },
-        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
-          "time_collected": "2024-03-01T19:17:10.522140Z",
-          "source": "http://[fd00:1122:3344:121::1]:12345",
-          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
-          },
-          "sled_agent_address": "[fd00:1122:3344:121::1]:12345",
+        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
+          "time_collected": "2024-03-12T18:59:34.262487Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
+          "baseboard_id": null,
+          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
           "sled_role": "gimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a452a7f5-b528-40fe-80ff-4e4189e2d52b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b452a7f5-b528-40fe-80ff-4e4189e2d52b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d452a7f5-b528-40fe-80ff-4e4189e2d52b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
+        },
+        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
+          "time_collected": "2024-03-12T18:59:34.306155Z",
+          "source": "http://[fd00:1122:3344:104::1]:12345",
+          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
+          "baseboard_id": null,
+          "sled_agent_address": "[fd00:1122:3344:104::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
         },
-        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
-          "time_collected": "2024-03-01T19:17:10.557759Z",
+        "77851655-10a5-4e80-88b0-a001473cab7e": {
+          "time_collected": "2024-03-12T18:59:34.349258Z",
           "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
-          },
+          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
+          "baseboard_id": null,
           "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
           "sled_role": "gimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a462a7f6-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b462a7f6-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d462a7f6-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
         },
-        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
-          "time_collected": "2024-03-01T19:17:10.593424Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
-          },
-          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
+        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
+          "time_collected": "2024-03-12T18:59:34.392838Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
+          "baseboard_id": null,
+          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
           "sled_role": "scrimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a462a4f4-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b462a4f4-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d462a4f4-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
         }
       },
       "omicron_zones": {
-        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
-          "time_collected": "2024-03-01T19:17:10.486656Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
+        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
+          "time_collected": "2024-03-12T18:59:34.266337Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
           "zones": {
             "generation": 5,
             "zones": [
               {
-                "id": "0706860a-3801-42a7-8000-fff741f7665b",
-                "underlay_address": "fd00:1122:3344:101::c",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::c]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
-                  }
-                }
-              },
-              {
-                "id": "34a279a2-a025-41ff-a392-1addac400f38",
-                "underlay_address": "fd00:1122:3344:101::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                  }
-                }
-              },
-              {
-                "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
-                "underlay_address": "fd00:1122:3344:1::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                  },
-                  "dns_address": "[fd00:1122:3344:1::1]:53",
-                  "gz_address": "fd00:1122:3344:1::2",
-                  "gz_address_index": 0,
-                  "http_address": "[fd00:1122:3344:1::1]:5353"
-                }
-              },
-              {
-                "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-                "underlay_address": "fd00:1122:3344:101::5",
+                "id": "0db88baf-911d-47bb-af71-3aebd7c96934",
+                "underlay_address": "fd00:1122:3344:103::4",
                 "zone_type": {
                   "type": "nexus",
                   "external_dns_servers": [
                     "1.1.1.1",
                     "9.9.9.9"
                   ],
-                  "external_ip": "172.20.28.2",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:101::5]:12221",
+                  "external_ip": "198.51.100.22",
+                  "external_tls": false,
+                  "internal_address": "[fd00:1122:3344:103::4]:12221",
                   "nic": {
-                    "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
-                    "ip": "172.30.2.5",
+                    "id": "8847a09c-cabc-429b-bf46-66d5e2e1140c",
                     "kind": {
                       "type": "service",
-                      "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
+                      "id": "0db88baf-911d-47bb-af71-3aebd7c96934"
                     },
-                    "mac": "A8:40:25:FF:E1:17",
-                    "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-                    "primary": true,
-                    "slot": 0,
+                    "name": "nexus-0db88baf-911d-47bb-af71-3aebd7c96934",
+                    "ip": "172.30.2.5",
+                    "mac": "A8:40:25:FF:CC:DA",
                     "subnet": "172.30.2.0/24",
-                    "vni": 100
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
-                "underlay_address": "fd00:1122:3344:101::10",
+                "id": "213d7dd3-95a4-45a7-99ca-e4899991dab4",
+                "underlay_address": "fd00:1122:3344:103::6",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::10]:32345",
+                  "address": "[fd00:1122:3344:103::6]:32345",
                   "dataset": {
-                    "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
-                "underlay_address": "fd00:1122:3344:101::f",
+                "id": "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
+                "underlay_address": "fd00:1122:3344:3::1",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::f]:32345",
+                  "type": "internal_dns",
                   "dataset": {
-                    "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
-                  }
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
+                  },
+                  "dns_address": "[fd00:1122:3344:3::1]:53",
+                  "gz_address": "fd00:1122:3344:3::2",
+                  "gz_address_index": 2,
+                  "http_address": "[fd00:1122:3344:3::1]:5353"
                 }
               },
               {
-                "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
-                "underlay_address": "fd00:1122:3344:101::9",
+                "id": "40c47ead-3be8-4724-a8ef-b66233fc5c65",
+                "underlay_address": "fd00:1122:3344:103::7",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::9]:32345",
+                  "address": "[fd00:1122:3344:103::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                    "pool_name": "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
-                "underlay_address": "fd00:1122:3344:101::8",
+                "id": "425d9d2f-72ea-4f53-80b6-04e002771cdb",
+                "underlay_address": "fd00:1122:3344:103::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                  }
-                }
-              },
-              {
-                "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
-                "underlay_address": "fd00:1122:3344:101::4",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::4]:32221",
+                  "address": "[fd00:1122:3344:103::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                    "pool_name": "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
-                "underlay_address": "fd00:1122:3344:101::d",
+                "id": "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
+                "underlay_address": "fd00:1122:3344:103::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::d]:32345",
+                  "address": "[fd00:1122:3344:103::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
+                    "pool_name": "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b"
                   }
                 }
               },
               {
-                "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
-                "underlay_address": "fd00:1122:3344:101::a",
+                "id": "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
+                "underlay_address": "fd00:1122:3344:103::5",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::a]:32345",
+                  "type": "clickhouse",
+                  "address": "[fd00:1122:3344:103::5]:8123",
                   "dataset": {
-                    "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
-                "underlay_address": "fd00:1122:3344:101::6",
-                "zone_type": {
-                  "type": "oximeter",
-                  "address": "[fd00:1122:3344:101::6]:12223"
-                }
-              },
-              {
-                "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
-                "underlay_address": "fd00:1122:3344:101::11",
+                "id": "7611a549-effc-41b6-b6cb-c03e0c70a31f",
+                "underlay_address": "fd00:1122:3344:103::b",
                 "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:101::11]:123",
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:103::b]:123",
                   "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
                   ],
-                  "nic": {
-                    "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
-                    "ip": "172.30.3.5",
-                    "kind": {
-                      "type": "service",
-                      "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
-                    },
-                    "mac": "A8:40:25:FF:E7:A8",
-                    "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100
-                  },
                   "ntp_servers": [
-                    "ntp.eng.oxide.computer"
-                  ],
-                  "snat_cfg": {
-                    "ip": "172.20.28.5",
-                    "first_port": 0,
-                    "last_port": 16383
-                  }
+                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
+                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
+                  ]
                 }
               },
               {
-                "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
-                "underlay_address": "fd00:1122:3344:101::e",
+                "id": "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
+                "underlay_address": "fd00:1122:3344:103::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::e]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:103::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
-                "underlay_address": "fd00:1122:3344:101::b",
+                "id": "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
+                "underlay_address": "fd00:1122:3344:103::a",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::b]:32345",
+                  "address": "[fd00:1122:3344:103::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
+                    "pool_name": "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
-              },
-              {
-                "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
-                "underlay_address": "fd00:1122:3344:101::7",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:101::7]:17000"
-                }
               }
             ]
           }
         },
-        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
-          "time_collected": "2024-03-01T19:17:10.522942Z",
-          "source": "http://[fd00:1122:3344:121::1]:12345",
-          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
+        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
+          "time_collected": "2024-03-12T18:59:34.311965Z",
+          "source": "http://[fd00:1122:3344:104::1]:12345",
+          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
           "zones": {
-            "generation": 3,
+            "generation": 5,
             "zones": [
               {
-                "id": "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
-                "underlay_address": "fd00:1122:3344:121::28",
+                "id": "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
+                "underlay_address": "fd00:1122:3344:104::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:104::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
+                  }
+                }
+              },
+              {
+                "id": "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
+                "underlay_address": "fd00:1122:3344:104::4",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:104::4]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
+                  }
+                }
+              },
+              {
+                "id": "57c712a6-6429-44b7-a9b2-9957cb9054d4",
+                "underlay_address": "fd00:1122:3344:104::a",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::28]:32345",
+                  "address": "[fd00:1122:3344:104::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3"
+                    "pool_name": "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "12431d3e-1516-4822-98b3-3711ff912e69",
-                "underlay_address": "fd00:1122:3344:121::22",
+                "id": "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
+                "underlay_address": "fd00:1122:3344:104::7",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::22]:32345",
+                  "address": "[fd00:1122:3344:104::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_0622b2cf-297e-4356-b94c-d74309f443ba"
+                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
-                "underlay_address": "fd00:1122:3344:121::21",
+                "id": "73916098-788f-48d8-b745-e13fdcfb771b",
+                "underlay_address": "fd00:1122:3344:104::c",
                 "zone_type": {
                   "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:121::21]:123",
+                  "address": "[fd00:1122:3344:104::c]:123",
                   "dns_servers": [
                     "fd00:1122:3344:1::1",
                     "fd00:1122:3344:2::1",
                     "fd00:1122:3344:3::1"
                   ],
                   "ntp_servers": [
-                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
+                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
                   ]
                 }
               },
               {
-                "id": "56414d76-9b8c-4e15-acf1-cd269c2d8638",
-                "underlay_address": "fd00:1122:3344:121::25",
+                "id": "9bb513e3-7505-416c-aa70-b26f1152bbdc",
+                "underlay_address": "fd00:1122:3344:104::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::25]:32345",
+                  "address": "[fd00:1122:3344:104::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_574728ca-1559-48f3-a46c-1e608966e6a0"
+                    "pool_name": "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b"
                   }
                 }
               },
               {
-                "id": "891cb9da-301d-40cc-b93d-5fcb350fa968",
-                "underlay_address": "fd00:1122:3344:121::29",
+                "id": "9c14c879-137b-4f5b-8068-1292bffc83ed",
+                "underlay_address": "fd00:1122:3344:104::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::29]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
-                  }
-                }
-              },
-              {
-                "id": "d9014907-24c0-4c33-a475-db15942f4f8f",
-                "underlay_address": "fd00:1122:3344:121::23",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::23]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:104::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37"
+                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "e950de25-7306-4cf7-a6f4-6db11adaa310",
-                "underlay_address": "fd00:1122:3344:121::27",
+                "id": "9cf90589-5da1-4b42-b09b-d784e4134637",
+                "underlay_address": "fd00:1122:3344:104::5",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::27]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c"
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "198.51.100.23",
+                  "external_tls": false,
+                  "internal_address": "[fd00:1122:3344:104::5]:12221",
+                  "nic": {
+                    "id": "fa76e7e1-dee6-4574-b71f-e07ea1cffd08",
+                    "kind": {
+                      "type": "service",
+                      "id": "9cf90589-5da1-4b42-b09b-d784e4134637"
+                    },
+                    "name": "nexus-9cf90589-5da1-4b42-b09b-d784e4134637",
+                    "ip": "172.30.2.6",
+                    "mac": "A8:40:25:FF:C4:6F",
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
-                "underlay_address": "fd00:1122:3344:121::26",
+                "id": "d48b2dfa-c749-47dd-98b6-3f478cd63934",
+                "underlay_address": "fd00:1122:3344:104::6",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::26]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945"
-                  }
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:104::6]:17000"
                 }
               },
               {
-                "id": "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00",
-                "underlay_address": "fd00:1122:3344:121::24",
+                "id": "fd003bee-818a-4c95-aae5-e378ac601522",
+                "underlay_address": "fd00:1122:3344:104::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::24]:32345",
+                  "address": "[fd00:1122:3344:104::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1"
+                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               }
             ]
           }
         },
-        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
-          "time_collected": "2024-03-01T19:17:10.558583Z",
+        "77851655-10a5-4e80-88b0-a001473cab7e": {
+          "time_collected": "2024-03-12T18:59:34.356678Z",
           "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
           "zones": {
             "generation": 5,
             "zones": [
               {
-                "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
-                "underlay_address": "fd00:1122:3344:102::11",
+                "id": "084cac6e-44b5-4333-a2ff-b0993b534898",
+                "underlay_address": "fd00:1122:3344:102::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::11]:32345",
+                  "address": "[fd00:1122:3344:102::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                    "pool_name": "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
-                "underlay_address": "fd00:1122:3344:102::e",
+                "id": "11c527c5-0598-428e-9270-be5b74d3461b",
+                "underlay_address": "fd00:1122:3344:102::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::e]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "479811e7-6409-45d4-b294-5333534e47ed",
-                "underlay_address": "fd00:1122:3344:102::9",
+                "id": "20be9299-495d-428d-b315-694c2c4b64af",
+                "underlay_address": "fd00:1122:3344:102::6",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:102::6]:17000"
+                }
+              },
+              {
+                "id": "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
+                "underlay_address": "fd00:1122:3344:102::b",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::9]:32345",
+                  "address": "[fd00:1122:3344:102::b]:32345",
                   "dataset": {
-                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                    "pool_name": "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
+                "id": "8d452bdd-e8b8-4108-a690-46fc59197db0",
                 "underlay_address": "fd00:1122:3344:102::a",
                 "zone_type": {
                   "type": "crucible",
                   "address": "[fd00:1122:3344:102::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                    "pool_name": "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
-                "underlay_address": "fd00:1122:3344:2::1",
+                "id": "953019c0-8e85-4417-b231-d42820470125",
+                "underlay_address": "fd00:1122:3344:102::5",
                 "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                  },
-                  "dns_address": "[fd00:1122:3344:2::1]:53",
-                  "gz_address": "fd00:1122:3344:2::2",
-                  "gz_address_index": 1,
-                  "http_address": "[fd00:1122:3344:2::1]:5353"
+                  "type": "oximeter",
+                  "address": "[fd00:1122:3344:102::5]:12223"
                 }
               },
               {
-                "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+                "id": "99c635b9-8e97-414a-8316-8261d2d767d0",
                 "underlay_address": "fd00:1122:3344:102::7",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:102::7]:17000"
-                }
-              },
-              {
-                "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
-                "underlay_address": "fd00:1122:3344:102::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::8]:32345",
+                  "address": "[fd00:1122:3344:102::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "658076ed-fb43-49e7-a813-df3d23497848",
-                "underlay_address": "fd00:1122:3344:102::10",
+                "id": "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
+                "underlay_address": "fd00:1122:3344:2::1",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::10]:32345",
+                  "type": "internal_dns",
                   "dataset": {
-                    "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
-                  }
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
+                  },
+                  "dns_address": "[fd00:1122:3344:2::1]:53",
+                  "gz_address": "fd00:1122:3344:2::2",
+                  "gz_address_index": 1,
+                  "http_address": "[fd00:1122:3344:2::1]:5353"
                 }
               },
               {
-                "id": "6edb836e-b100-4080-bb03-738d6441d571",
+                "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
                 "underlay_address": "fd00:1122:3344:102::c",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::c]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
-                  }
-                }
-              },
-              {
-                "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
-                "underlay_address": "fd00:1122:3344:102::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:102::c]:123",
+                  "dns_servers": [
                     "1.1.1.1",
                     "9.9.9.9"
                   ],
-                  "external_ip": "172.20.28.3",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:102::5]:12221",
                   "nic": {
-                    "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
-                    "ip": "172.30.2.6",
+                    "id": "659f33f7-e7bd-4875-9140-bf8a8b2d964a",
                     "kind": {
                       "type": "service",
-                      "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
+                      "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded"
                     },
-                    "mac": "A8:40:25:FF:B2:AF",
-                    "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                    "name": "ntp-d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
+                    "ip": "172.30.3.6",
+                    "mac": "A8:40:25:FF:B1:99",
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100,
                     "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100
+                    "slot": 0
+                  },
+                  "ntp_servers": [
+                    "time.cloudflare.com"
+                  ],
+                  "snat_cfg": {
+                    "ip": "198.51.100.26",
+                    "first_port": 16384,
+                    "last_port": 32767
                   }
                 }
               },
               {
-                "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
-                "underlay_address": "fd00:1122:3344:102::d",
+                "id": "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
+                "underlay_address": "fd00:1122:3344:102::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::d]:32345",
+                  "address": "[fd00:1122:3344:102::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                    "pool_name": "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b"
                   }
                 }
               },
               {
-                "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+                "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
                 "underlay_address": "fd00:1122:3344:102::4",
                 "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::4]:32221",
+                  "type": "external_dns",
                   "dataset": {
-                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
+                  },
+                  "dns_address": "198.51.100.21:53",
+                  "http_address": "[fd00:1122:3344:102::4]:5353",
+                  "nic": {
+                    "id": "d4dd2f7a-8a3b-405e-8444-62f99a754a8a",
+                    "kind": {
+                      "type": "service",
+                      "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802"
+                    },
+                    "name": "external-dns-f0ed9838-6f1b-4646-a478-9f74f2e6e802",
+                    "ip": "172.30.1.6",
+                    "mac": "A8:40:25:FF:F4:5C",
+                    "subnet": "172.30.1.0/24",
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
-              },
+              }
+            ]
+          }
+        },
+        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
+          "time_collected": "2024-03-12T18:59:34.396440Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
+          "zones": {
+            "generation": 5,
+            "zones": [
               {
-                "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
-                "underlay_address": "fd00:1122:3344:102::6",
+                "id": "0d459323-e414-4f32-9944-76c7331da622",
+                "underlay_address": "fd00:1122:3344:101::5",
                 "zone_type": {
-                  "type": "clickhouse",
-                  "address": "[fd00:1122:3344:102::6]:8123",
-                  "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "198.51.100.24",
+                  "external_tls": false,
+                  "internal_address": "[fd00:1122:3344:101::5]:12221",
+                  "nic": {
+                    "id": "514902b8-7a1a-42a3-95a6-d00861833b5c",
+                    "kind": {
+                      "type": "service",
+                      "id": "0d459323-e414-4f32-9944-76c7331da622"
+                    },
+                    "name": "nexus-0d459323-e414-4f32-9944-76c7331da622",
+                    "ip": "172.30.2.7",
+                    "mac": "A8:40:25:FF:DE:AC",
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
-                "underlay_address": "fd00:1122:3344:102::f",
+                "id": "2902972b-3ae4-489d-b937-6480352e214a",
+                "underlay_address": "fd00:1122:3344:101::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::f]:32345",
+                  "address": "[fd00:1122:3344:101::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
+                    "pool_name": "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
-                "underlay_address": "fd00:1122:3344:102::3",
+                "id": "2ee832de-af75-47ac-abac-f91304857ca4",
+                "underlay_address": "fd00:1122:3344:101::4",
                 "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::3]:32221",
+                  "type": "external_dns",
                   "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
+                  },
+                  "dns_address": "198.51.100.20:53",
+                  "http_address": "[fd00:1122:3344:101::4]:5353",
+                  "nic": {
+                    "id": "fadff4ab-62d7-466a-9203-04080e4078ed",
+                    "kind": {
+                      "type": "service",
+                      "id": "2ee832de-af75-47ac-abac-f91304857ca4"
+                    },
+                    "name": "external-dns-2ee832de-af75-47ac-abac-f91304857ca4",
+                    "ip": "172.30.1.5",
+                    "mac": "A8:40:25:FF:B4:93",
+                    "subnet": "172.30.1.0/24",
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-                "underlay_address": "fd00:1122:3344:102::12",
+                "id": "30050a7e-7fc3-4965-b07b-1635212393eb",
+                "underlay_address": "fd00:1122:3344:101::6",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:101::6]:17000"
+                }
+              },
+              {
+                "id": "8494befe-6df6-42d1-9baa-599ba4228430",
+                "underlay_address": "fd00:1122:3344:101::c",
                 "zone_type": {
                   "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:102::12]:123",
+                  "address": "[fd00:1122:3344:101::c]:123",
                   "dns_servers": [
                     "1.1.1.1",
                     "9.9.9.9"
                   ],
                   "nic": {
-                    "id": "9f03c82c-3192-498d-978d-a175049ac389",
-                    "ip": "172.30.3.6",
+                    "id": "23041748-e3a6-4846-930d-5a348c316e91",
                     "kind": {
                       "type": "service",
-                      "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
+                      "id": "8494befe-6df6-42d1-9baa-599ba4228430"
                     },
-                    "mac": "A8:40:25:FF:E4:11",
-                    "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-                    "primary": true,
-                    "slot": 0,
+                    "name": "ntp-8494befe-6df6-42d1-9baa-599ba4228430",
+                    "ip": "172.30.3.5",
+                    "mac": "A8:40:25:FF:D1:C4",
                     "subnet": "172.30.3.0/24",
-                    "vni": 100
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   },
                   "ntp_servers": [
-                    "ntp.eng.oxide.computer"
+                    "time.cloudflare.com"
                   ],
                   "snat_cfg": {
-                    "ip": "172.20.28.6",
-                    "first_port": 16384,
-                    "last_port": 32767
+                    "ip": "198.51.100.25",
+                    "first_port": 0,
+                    "last_port": 16383
                   }
                 }
               },
               {
-                "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
-                "underlay_address": "fd00:1122:3344:102::b",
+                "id": "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
+                "underlay_address": "fd00:1122:3344:101::7",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::b]:32345",
+                  "address": "[fd00:1122:3344:101::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
-          "time_collected": "2024-03-01T19:17:10.594368Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
-                "underlay_address": "fd00:1122:3344:103::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
-                "underlay_address": "fd00:1122:3344:103::10",
+                "id": "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
+                "underlay_address": "fd00:1122:3344:101::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::10]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
-                "underlay_address": "fd00:1122:3344:3::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  },
-                  "dns_address": "[fd00:1122:3344:3::1]:53",
-                  "gz_address": "fd00:1122:3344:3::2",
-                  "gz_address_index": 2,
-                  "http_address": "[fd00:1122:3344:3::1]:5353"
-                }
-              },
-              {
-                "id": "4b74c418-0a08-4489-b652-e97267c1c220",
-                "underlay_address": "fd00:1122:3344:103::7",
+                "id": "e668e3f5-5457-4689-97da-b2af537787ac",
+                "underlay_address": "fd00:1122:3344:101::b",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::7]:32345",
+                  "address": "[fd00:1122:3344:101::b]:32345",
                   "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  }
-                }
-              },
-              {
-                "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
-                "underlay_address": "fd00:1122:3344:103::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "172.20.28.4",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:103::5]:12221",
-                  "nic": {
-                    "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
-                    "ip": "172.30.2.7",
-                    "kind": {
-                      "type": "service",
-                      "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
-                    },
-                    "mac": "A8:40:25:FF:96:05",
-                    "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100
+                    "pool_name": "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
-                "underlay_address": "fd00:1122:3344:103::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:103::6]:17000"
-                }
-              },
-              {
-                "id": "921958c0-7761-4294-945d-2a078327bf2c",
-                "underlay_address": "fd00:1122:3344:103::8",
+                "id": "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
+                "underlay_address": "fd00:1122:3344:101::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::8]:32345",
+                  "address": "[fd00:1122:3344:101::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
+                    "pool_name": "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b"
                   }
                 }
               },
               {
-                "id": "9df22034-59f6-431f-82e7-258fe983879b",
-                "underlay_address": "fd00:1122:3344:103::4",
+                "id": "f8c79ba4-194a-4cfe-8090-5315e297b780",
+                "underlay_address": "fd00:1122:3344:1::1",
                 "zone_type": {
-                  "type": "external_dns",
+                  "type": "internal_dns",
                   "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   },
-                  "dns_address": "172.20.28.1:53",
-                  "http_address": "[fd00:1122:3344:103::4]:5353",
-                  "nic": {
-                    "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
-                    "ip": "172.30.1.5",
-                    "kind": {
-                      "type": "service",
-                      "id": "9df22034-59f6-431f-82e7-258fe983879b"
-                    },
-                    "mac": "A8:40:25:FF:90:12",
-                    "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.1.0/24",
-                    "vni": 100
-                  }
-                }
-              },
-              {
-                "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
-                "underlay_address": "fd00:1122:3344:103::11",
-                "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:103::11]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
-                  ]
-                }
-              },
-              {
-                "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
-                "underlay_address": "fd00:1122:3344:103::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
-                  }
-                }
-              },
-              {
-                "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
-                "underlay_address": "fd00:1122:3344:103::d",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::d]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
-                  }
-                }
-              },
-              {
-                "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
-                "underlay_address": "fd00:1122:3344:103::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
-                  }
-                }
-              },
-              {
-                "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
-                "underlay_address": "fd00:1122:3344:103::e",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::e]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
-                  }
-                }
-              },
-              {
-                "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
-                "underlay_address": "fd00:1122:3344:103::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:103::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  }
-                }
-              },
-              {
-                "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
-                "underlay_address": "fd00:1122:3344:103::f",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::f]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
-                  }
+                  "dns_address": "[fd00:1122:3344:1::1]:53",
+                  "gz_address": "fd00:1122:3344:1::2",
+                  "gz_address_index": 0,
+                  "http_address": "[fd00:1122:3344:1::1]:5353"
                 }
               },
               {
-                "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
-                "underlay_address": "fd00:1122:3344:103::c",
+                "id": "fa5e0176-b4a8-4598-aa31-40fd562d246c",
+                "underlay_address": "fd00:1122:3344:101::a",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::c]:32345",
+                  "address": "[fd00:1122:3344:101::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                    "pool_name": "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               }
@@ -1921,284 +1886,191 @@
       }
     },
     {
-      "id": "0a8dc8ff-9013-43aa-9f06-9540b24ea902",
-      "errors": [
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bb340ba7-cf9e-48c9-9e87-4b26fcb9b2aa\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"bb340ba7-cf9e-48c9-9e87-4b26fcb9b2aa\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"86914fc3-f142-4dbf-8ac5-a94c03c38a93\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"86914fc3-f142-4dbf-8ac5-a94c03c38a93\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e29b10e1-aea9-48a9-8054-951715c06bdb\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:09 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"e29b10e1-aea9-48a9-8054-951715c06bdb\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"719a3865-65bc-4228-810f-20b93a5199d0\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:10 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"719a3865-65bc-4228-810f-20b93a5199d0\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"70ee4f97-6ffe-46fe-810d-6d0f602e8749\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:10 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"70ee4f97-6ffe-46fe-810d-6d0f602e8749\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6d81dff5-4390-43dc-a976-489980a574f3\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:10 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"6d81dff5-4390-43dc-a976-489980a574f3\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f83589b8-03d7-47f6-8e49-4090810c6485\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:10 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"f83589b8-03d7-47f6-8e49-4090810c6485\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"73e37767-229b-48ca-9fcd-d98988c1a131\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:17:10 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"73e37767-229b-48ca-9fcd-d98988c1a131\" }"
-      ],
-      "time_started": "2024-03-01T19:17:06.813002Z",
-      "time_done": "2024-03-01T19:17:11.153859Z",
-      "collector": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+      "id": "37c212f1-bfdd-4cfd-a17c-e65bd93bea5f",
+      "errors": [],
+      "time_started": "2024-03-12T19:07:28.237276Z",
+      "time_done": "2024-03-12T19:07:28.472922Z",
+      "collector": "0db88baf-911d-47bb-af71-3aebd7c96934",
       "baseboards": [
         {
-          "part_number": "913-0000006",
-          "serial_number": "BRM44220005"
-        },
-        {
-          "part_number": "913-0000006",
-          "serial_number": "BRM44220011"
+          "part_number": "FAKE_SIM_SIDECAR",
+          "serial_number": "SimSidecar0"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220004"
+          "part_number": "i86pc",
+          "serial_number": "g0"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220046"
+          "part_number": "i86pc",
+          "serial_number": "g1"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220081"
+          "part_number": "i86pc",
+          "serial_number": "g2"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM44220001"
+          "part_number": "i86pc",
+          "serial_number": "g3"
         }
       ],
       "cabooses": [
         {
-          "board": "gimlet-c",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "gimlet-c",
-          "version": "1.0.6"
-        },
-        {
-          "board": "gimlet-c",
-          "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
-          "name": "gimlet-c",
-          "version": "1.0.5"
-        },
-        {
-          "board": "gimlet-c",
-          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-          "name": "gimlet-c",
-          "version": "1.0.8"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "oxide-rot-1",
-          "version": "1.0.5"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
-          "name": "oxide-rot-1",
-          "version": "1.0.4"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
-          "name": "oxide-rot-1",
-          "version": "0.0.0-git"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-          "name": "oxide-rot-1",
-          "version": "1.0.6"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-          "name": "oxide-rot-1",
-          "version": "1.0.0"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-          "name": "oxide-rot-1-dev",
-          "version": "0.0.0-git"
+          "board": "SimGimletSp",
+          "git_commit": "ffffffff",
+          "name": "SimGimlet",
+          "version": "0.0.1"
         },
         {
-          "board": "sidecar-b",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "sidecar-b",
-          "version": "1.0.6"
+          "board": "SimRot",
+          "git_commit": "eeeeeeee",
+          "name": "SimGimlet",
+          "version": "0.0.1"
         },
         {
-          "board": "sidecar-b",
-          "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
-          "name": "sidecar-b",
-          "version": "0.0.0-git"
+          "board": "SimRot",
+          "git_commit": "eeeeeeee",
+          "name": "SimSidecar",
+          "version": "0.0.1"
         },
         {
-          "board": "sidecar-b",
-          "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
-          "name": "sidecar-b",
-          "version": "0.0.0-git"
+          "board": "SimSidecarSp",
+          "git_commit": "ffffffff",
+          "name": "SimSidecar",
+          "version": "0.0.1"
         }
       ],
       "rot_pages": [
         {
-          "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+          "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+        },
+        {
+          "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+          "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+          "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+          "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+          "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+          "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+          "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         }
       ],
       "sps": [
         [
           {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220005"
-          },
-          {
-            "time_collected": "2024-03-01T19:17:09.906135Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "sp_type": "switch",
-            "sp_slot": 0,
-            "baseboard_revision": 4,
-            "hubris_archive": "9184be93590e655e",
-            "power_state": "A0"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220011"
+            "part_number": "FAKE_SIM_SIDECAR",
+            "serial_number": "SimSidecar0"
           },
           {
-            "time_collected": "2024-03-01T19:17:10.475436Z",
-            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.244597Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "switch",
             "sp_slot": 1,
-            "baseboard_revision": 4,
-            "hubris_archive": "9184be93590e655e",
-            "power_state": "A0"
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
+            "part_number": "i86pc",
+            "serial_number": "g0"
           },
           {
-            "time_collected": "2024-03-01T19:17:08.746072Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.251644Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 17,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
+            "sp_slot": 0,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
+            "part_number": "i86pc",
+            "serial_number": "g1"
           },
           {
-            "time_collected": "2024-03-01T19:17:09.327592Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.257993Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 15,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
+            "sp_slot": 1,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
+            "part_number": "i86pc",
+            "serial_number": "g2"
           },
           {
-            "time_collected": "2024-03-01T19:17:08.014626Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.263543Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 14,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
+            "sp_slot": 2,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
+            "part_number": "i86pc",
+            "serial_number": "g3"
           },
           {
-            "time_collected": "2024-03-01T19:17:06.876242Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.239320Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 16,
-            "baseboard_revision": 6,
-            "hubris_archive": "3c670b44fdeef4af",
-            "power_state": "A0"
+            "sp_slot": 3,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ]
       ],
       "rots": [
         [
           {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220005"
+            "part_number": "FAKE_SIM_SIDECAR",
+            "serial_number": "SimSidecar0"
           },
           {
-            "time_collected": "2024-03-01T19:17:09.906135Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.244597Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
             "persistent_boot_preference": {
-              "slot": "b"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "4599e4d7bc9b2164e8c0f0f7ce4bcae2bd35d635c8450854ea9a94ce49a2ca0f",
-            "slot_b_sha3_256_digest": "1e42a0761e5bd1d9f9a5e2c44acc34e7b7e007792977cdb0523d271db2e80c8c"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220011"
-          },
-          {
-            "time_collected": "2024-03-01T19:17:10.475436Z",
-            "source": "http://[fd00:1122:3344:103::2]:12225",
-            "active_slot": {
-              "slot": "b"
-            },
-            "persistent_boot_preference": {
-              "slot": "b"
+              "slot": "a"
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "92e07077972c83af6872ec817fb88d74a36f6956276505e9771302c5bb7ec2a8",
-            "slot_b_sha3_256_digest": "fb25825eaf97f8f2307afe3caebf0b70865499f6d116db3aff382c505b4d8dd7"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
+            "part_number": "i86pc",
+            "serial_number": "g0"
           },
           {
-            "time_collected": "2024-03-01T19:17:08.746072Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.251644Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
@@ -2207,18 +2079,18 @@
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
-            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
+            "part_number": "i86pc",
+            "serial_number": "g1"
           },
           {
-            "time_collected": "2024-03-01T19:17:09.327592Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.257993Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
@@ -2227,38 +2099,38 @@
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
-            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
+            "part_number": "i86pc",
+            "serial_number": "g2"
           },
           {
-            "time_collected": "2024-03-01T19:17:08.014626Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.263543Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
-              "slot": "b"
+              "slot": "a"
             },
             "persistent_boot_preference": {
-              "slot": "b"
+              "slot": "a"
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
-            "slot_b_sha3_256_digest": "d416d5ba9d0d3bd495f0f5fcd2aecc28cd83b53d1a8f94a28241d85b30761451"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
+            "part_number": "i86pc",
+            "serial_number": "g3"
           },
           {
-            "time_collected": "2024-03-01T19:17:06.876242Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.239320Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
@@ -2267,8 +2139,8 @@
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
-            "slot_b_sha3_256_digest": "4ec328195a23e86a1545d5ee576e73834aae81fb79830a54d0503475875f1760"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ]
       ],
@@ -2276,97 +2148,81 @@
         "SpSlot0": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:17:09.907517Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "sidecar-b",
-                "version": "1.0.6"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:17:10.476830Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.245320Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "sidecar-b",
-                "version": "1.0.6"
+                "board": "SimSidecarSp",
+                "git_commit": "ffffffff",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.751041Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.252625Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.329018Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.258749Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.123957Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.273088Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.130790Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.240046Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ]
@@ -2374,97 +2230,81 @@
         "SpSlot1": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:17:09.909025Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
-                "name": "sidecar-b",
-                "version": "0.0.0-git"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:17:10.478263Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.245980Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
-                "name": "sidecar-b",
-                "version": "0.0.0-git"
+                "board": "SimSidecarSp",
+                "git_commit": "ffffffff",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.752437Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.253364Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.330438Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.259496Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.163634Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.273807Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.328321Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.240736Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
-                "name": "gimlet-c",
-                "version": "1.0.5"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ]
@@ -2472,97 +2312,81 @@
         "RotSlotA": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:17:10.115796Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-                "name": "oxide-rot-1-dev",
-                "version": "0.0.0-git"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:17:10.685118Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.246605Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
-                "name": "oxide-rot-1",
-                "version": "0.0.0-git"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.003752Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.254069Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.582176Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.260195Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.415126Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.274568Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.643681Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.241356Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ]
@@ -2570,153 +2394,150 @@
         "RotSlotB": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:17:10.322717Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.247577Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:10.892030Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.254910Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-                "name": "oxide-rot-1",
-                "version": "1.0.0"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.255745Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.260882Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.834209Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.275221Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.667106Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.241947Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
-          ],
+          ]
+        ]
+      },
+      "rot_pages_found": {
+        "Cmpa": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.894743Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
-                "name": "oxide-rot-1",
-                "version": "1.0.4"
+              "time_collected": "2024-03-12T19:07:28.248572Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
-          ]
-        ]
-      },
-      "rot_pages_found": {
-        "Cmpa": [
+          ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.272691Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.255711Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.850755Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.261401Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.682710Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.275673Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.911203Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.242430Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
@@ -2724,53 +2545,66 @@
         "CfpaActive": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
+            },
+            {
+              "time_collected": "2024-03-12T19:07:28.249231Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.288403Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.256314Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.866661Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.261975Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.699070Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.276189Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.927224Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.242948Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
@@ -2778,53 +2612,66 @@
         "CfpaInactive": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
+            },
+            {
+              "time_collected": "2024-03-12T19:07:28.249902Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.304293Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.256959Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.882683Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.262485Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.714707Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.276907Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.943244Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.243514Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
@@ -2832,910 +2679,1010 @@
         "CfpaScratch": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
+            },
+            {
+              "time_collected": "2024-03-12T19:07:28.250611Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.320329Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.257483Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:17:09.898662Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.262983Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:17:08.730680Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.277386Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:17:07.959254Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.244032Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
         ]
       },
       "sled_agents": {
-        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
-          "time_collected": "2024-03-01T19:17:11.045570Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
-          },
-          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
-        },
-        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
-          "time_collected": "2024-03-01T19:17:11.081825Z",
-          "source": "http://[fd00:1122:3344:121::1]:12345",
-          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
-          },
-          "sled_agent_address": "[fd00:1122:3344:121::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
-        },
-        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
-          "time_collected": "2024-03-01T19:17:11.117162Z",
-          "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
-          },
-          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
-        },
-        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
-          "time_collected": "2024-03-01T19:17:11.152394Z",
+        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
+          "time_collected": "2024-03-12T19:07:28.330250Z",
           "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
-          },
+          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
+          "baseboard_id": null,
           "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a452a7f5-b528-40fe-80ff-4e4189e2d52b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b452a7f5-b528-40fe-80ff-4e4189e2d52b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d452a7f5-b528-40fe-80ff-4e4189e2d52b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
+        },
+        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
+          "time_collected": "2024-03-12T19:07:28.371508Z",
+          "source": "http://[fd00:1122:3344:104::1]:12345",
+          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
+          "baseboard_id": null,
+          "sled_agent_address": "[fd00:1122:3344:104::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
+        },
+        "77851655-10a5-4e80-88b0-a001473cab7e": {
+          "time_collected": "2024-03-12T19:07:28.409958Z",
+          "source": "http://[fd00:1122:3344:102::1]:12345",
+          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
+          "baseboard_id": null,
+          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a462a7f6-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b462a7f6-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d462a7f6-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
+        },
+        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
+          "time_collected": "2024-03-12T19:07:28.461671Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
+          "baseboard_id": null,
+          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
           "sled_role": "scrimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a462a4f4-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b462a4f4-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d462a4f4-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
         }
       },
       "omicron_zones": {
-        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
-          "time_collected": "2024-03-01T19:17:11.046682Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
+        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
+          "time_collected": "2024-03-12T19:07:28.340118Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
           "zones": {
             "generation": 5,
             "zones": [
               {
-                "id": "0706860a-3801-42a7-8000-fff741f7665b",
-                "underlay_address": "fd00:1122:3344:101::c",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::c]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
-                  }
-                }
-              },
-              {
-                "id": "34a279a2-a025-41ff-a392-1addac400f38",
-                "underlay_address": "fd00:1122:3344:101::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                  }
-                }
-              },
-              {
-                "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
-                "underlay_address": "fd00:1122:3344:1::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                  },
-                  "dns_address": "[fd00:1122:3344:1::1]:53",
-                  "gz_address": "fd00:1122:3344:1::2",
-                  "gz_address_index": 0,
-                  "http_address": "[fd00:1122:3344:1::1]:5353"
-                }
-              },
-              {
-                "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-                "underlay_address": "fd00:1122:3344:101::5",
+                "id": "0db88baf-911d-47bb-af71-3aebd7c96934",
+                "underlay_address": "fd00:1122:3344:103::4",
                 "zone_type": {
                   "type": "nexus",
                   "external_dns_servers": [
                     "1.1.1.1",
                     "9.9.9.9"
                   ],
-                  "external_ip": "172.20.28.2",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:101::5]:12221",
+                  "external_ip": "198.51.100.22",
+                  "external_tls": false,
+                  "internal_address": "[fd00:1122:3344:103::4]:12221",
                   "nic": {
-                    "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
-                    "ip": "172.30.2.5",
+                    "id": "8847a09c-cabc-429b-bf46-66d5e2e1140c",
                     "kind": {
                       "type": "service",
-                      "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
+                      "id": "0db88baf-911d-47bb-af71-3aebd7c96934"
                     },
-                    "mac": "A8:40:25:FF:E1:17",
-                    "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-                    "primary": true,
-                    "slot": 0,
+                    "name": "nexus-0db88baf-911d-47bb-af71-3aebd7c96934",
+                    "ip": "172.30.2.5",
+                    "mac": "A8:40:25:FF:CC:DA",
                     "subnet": "172.30.2.0/24",
-                    "vni": 100
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
-                "underlay_address": "fd00:1122:3344:101::10",
+                "id": "213d7dd3-95a4-45a7-99ca-e4899991dab4",
+                "underlay_address": "fd00:1122:3344:103::6",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::10]:32345",
+                  "address": "[fd00:1122:3344:103::6]:32345",
                   "dataset": {
-                    "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
-                "underlay_address": "fd00:1122:3344:101::f",
+                "id": "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
+                "underlay_address": "fd00:1122:3344:3::1",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::f]:32345",
+                  "type": "internal_dns",
                   "dataset": {
-                    "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
-                  }
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
+                  },
+                  "dns_address": "[fd00:1122:3344:3::1]:53",
+                  "gz_address": "fd00:1122:3344:3::2",
+                  "gz_address_index": 2,
+                  "http_address": "[fd00:1122:3344:3::1]:5353"
                 }
               },
               {
-                "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
-                "underlay_address": "fd00:1122:3344:101::9",
+                "id": "40c47ead-3be8-4724-a8ef-b66233fc5c65",
+                "underlay_address": "fd00:1122:3344:103::7",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::9]:32345",
+                  "address": "[fd00:1122:3344:103::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                    "pool_name": "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
-                "underlay_address": "fd00:1122:3344:101::8",
+                "id": "425d9d2f-72ea-4f53-80b6-04e002771cdb",
+                "underlay_address": "fd00:1122:3344:103::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                  }
-                }
-              },
-              {
-                "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
-                "underlay_address": "fd00:1122:3344:101::4",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::4]:32221",
+                  "address": "[fd00:1122:3344:103::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                    "pool_name": "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
-                "underlay_address": "fd00:1122:3344:101::d",
+                "id": "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
+                "underlay_address": "fd00:1122:3344:103::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::d]:32345",
+                  "address": "[fd00:1122:3344:103::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
+                    "pool_name": "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b"
                   }
                 }
               },
               {
-                "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
-                "underlay_address": "fd00:1122:3344:101::a",
+                "id": "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
+                "underlay_address": "fd00:1122:3344:103::5",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::a]:32345",
+                  "type": "clickhouse",
+                  "address": "[fd00:1122:3344:103::5]:8123",
                   "dataset": {
-                    "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
-                "underlay_address": "fd00:1122:3344:101::6",
-                "zone_type": {
-                  "type": "oximeter",
-                  "address": "[fd00:1122:3344:101::6]:12223"
-                }
-              },
-              {
-                "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
-                "underlay_address": "fd00:1122:3344:101::11",
+                "id": "7611a549-effc-41b6-b6cb-c03e0c70a31f",
+                "underlay_address": "fd00:1122:3344:103::b",
                 "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:101::11]:123",
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:103::b]:123",
                   "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
                   ],
-                  "nic": {
-                    "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
-                    "ip": "172.30.3.5",
-                    "kind": {
-                      "type": "service",
-                      "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
-                    },
-                    "mac": "A8:40:25:FF:E7:A8",
-                    "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100
-                  },
                   "ntp_servers": [
-                    "ntp.eng.oxide.computer"
-                  ],
-                  "snat_cfg": {
-                    "ip": "172.20.28.5",
-                    "first_port": 0,
-                    "last_port": 16383
-                  }
+                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
+                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
+                  ]
                 }
               },
               {
-                "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
-                "underlay_address": "fd00:1122:3344:101::e",
+                "id": "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
+                "underlay_address": "fd00:1122:3344:103::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::e]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:103::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
-                "underlay_address": "fd00:1122:3344:101::b",
+                "id": "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
+                "underlay_address": "fd00:1122:3344:103::a",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::b]:32345",
+                  "address": "[fd00:1122:3344:103::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
+                    "pool_name": "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
-              },
-              {
-                "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
-                "underlay_address": "fd00:1122:3344:101::7",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:101::7]:17000"
-                }
               }
             ]
           }
         },
-        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
-          "time_collected": "2024-03-01T19:17:11.082540Z",
-          "source": "http://[fd00:1122:3344:121::1]:12345",
-          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
+        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
+          "time_collected": "2024-03-12T19:07:28.381330Z",
+          "source": "http://[fd00:1122:3344:104::1]:12345",
+          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
           "zones": {
-            "generation": 3,
+            "generation": 5,
             "zones": [
               {
-                "id": "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
-                "underlay_address": "fd00:1122:3344:121::28",
+                "id": "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
+                "underlay_address": "fd00:1122:3344:104::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:104::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
+                  }
+                }
+              },
+              {
+                "id": "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
+                "underlay_address": "fd00:1122:3344:104::4",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:104::4]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
+                  }
+                }
+              },
+              {
+                "id": "57c712a6-6429-44b7-a9b2-9957cb9054d4",
+                "underlay_address": "fd00:1122:3344:104::a",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::28]:32345",
+                  "address": "[fd00:1122:3344:104::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3"
+                    "pool_name": "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "12431d3e-1516-4822-98b3-3711ff912e69",
-                "underlay_address": "fd00:1122:3344:121::22",
+                "id": "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
+                "underlay_address": "fd00:1122:3344:104::7",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::22]:32345",
+                  "address": "[fd00:1122:3344:104::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_0622b2cf-297e-4356-b94c-d74309f443ba"
+                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
-                "underlay_address": "fd00:1122:3344:121::21",
+                "id": "73916098-788f-48d8-b745-e13fdcfb771b",
+                "underlay_address": "fd00:1122:3344:104::c",
                 "zone_type": {
                   "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:121::21]:123",
+                  "address": "[fd00:1122:3344:104::c]:123",
                   "dns_servers": [
                     "fd00:1122:3344:1::1",
                     "fd00:1122:3344:2::1",
                     "fd00:1122:3344:3::1"
                   ],
                   "ntp_servers": [
-                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
+                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
                   ]
                 }
               },
               {
-                "id": "56414d76-9b8c-4e15-acf1-cd269c2d8638",
-                "underlay_address": "fd00:1122:3344:121::25",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::25]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_574728ca-1559-48f3-a46c-1e608966e6a0"
-                  }
-                }
-              },
-              {
-                "id": "891cb9da-301d-40cc-b93d-5fcb350fa968",
-                "underlay_address": "fd00:1122:3344:121::29",
+                "id": "9bb513e3-7505-416c-aa70-b26f1152bbdc",
+                "underlay_address": "fd00:1122:3344:104::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::29]:32345",
+                  "address": "[fd00:1122:3344:104::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
+                    "pool_name": "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b"
                   }
                 }
               },
               {
-                "id": "d9014907-24c0-4c33-a475-db15942f4f8f",
-                "underlay_address": "fd00:1122:3344:121::23",
+                "id": "9c14c879-137b-4f5b-8068-1292bffc83ed",
+                "underlay_address": "fd00:1122:3344:104::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::23]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:104::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37"
+                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "e950de25-7306-4cf7-a6f4-6db11adaa310",
-                "underlay_address": "fd00:1122:3344:121::27",
+                "id": "9cf90589-5da1-4b42-b09b-d784e4134637",
+                "underlay_address": "fd00:1122:3344:104::5",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::27]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c"
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "198.51.100.23",
+                  "external_tls": false,
+                  "internal_address": "[fd00:1122:3344:104::5]:12221",
+                  "nic": {
+                    "id": "fa76e7e1-dee6-4574-b71f-e07ea1cffd08",
+                    "kind": {
+                      "type": "service",
+                      "id": "9cf90589-5da1-4b42-b09b-d784e4134637"
+                    },
+                    "name": "nexus-9cf90589-5da1-4b42-b09b-d784e4134637",
+                    "ip": "172.30.2.6",
+                    "mac": "A8:40:25:FF:C4:6F",
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
-                "underlay_address": "fd00:1122:3344:121::26",
+                "id": "d48b2dfa-c749-47dd-98b6-3f478cd63934",
+                "underlay_address": "fd00:1122:3344:104::6",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::26]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945"
-                  }
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:104::6]:17000"
                 }
               },
               {
-                "id": "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00",
-                "underlay_address": "fd00:1122:3344:121::24",
+                "id": "fd003bee-818a-4c95-aae5-e378ac601522",
+                "underlay_address": "fd00:1122:3344:104::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::24]:32345",
+                  "address": "[fd00:1122:3344:104::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1"
+                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               }
             ]
           }
         },
-        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
-          "time_collected": "2024-03-01T19:17:11.118025Z",
+        "77851655-10a5-4e80-88b0-a001473cab7e": {
+          "time_collected": "2024-03-12T19:07:28.426275Z",
           "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
           "zones": {
             "generation": 5,
             "zones": [
               {
-                "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
-                "underlay_address": "fd00:1122:3344:102::11",
+                "id": "084cac6e-44b5-4333-a2ff-b0993b534898",
+                "underlay_address": "fd00:1122:3344:102::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::11]:32345",
+                  "address": "[fd00:1122:3344:102::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                    "pool_name": "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
-                "underlay_address": "fd00:1122:3344:102::e",
+                "id": "11c527c5-0598-428e-9270-be5b74d3461b",
+                "underlay_address": "fd00:1122:3344:102::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::e]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "479811e7-6409-45d4-b294-5333534e47ed",
-                "underlay_address": "fd00:1122:3344:102::9",
+                "id": "20be9299-495d-428d-b315-694c2c4b64af",
+                "underlay_address": "fd00:1122:3344:102::6",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:102::6]:17000"
+                }
+              },
+              {
+                "id": "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
+                "underlay_address": "fd00:1122:3344:102::b",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::9]:32345",
+                  "address": "[fd00:1122:3344:102::b]:32345",
                   "dataset": {
-                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                    "pool_name": "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
+                "id": "8d452bdd-e8b8-4108-a690-46fc59197db0",
                 "underlay_address": "fd00:1122:3344:102::a",
                 "zone_type": {
                   "type": "crucible",
                   "address": "[fd00:1122:3344:102::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                    "pool_name": "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
-                "underlay_address": "fd00:1122:3344:2::1",
+                "id": "953019c0-8e85-4417-b231-d42820470125",
+                "underlay_address": "fd00:1122:3344:102::5",
                 "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                  },
-                  "dns_address": "[fd00:1122:3344:2::1]:53",
-                  "gz_address": "fd00:1122:3344:2::2",
-                  "gz_address_index": 1,
-                  "http_address": "[fd00:1122:3344:2::1]:5353"
+                  "type": "oximeter",
+                  "address": "[fd00:1122:3344:102::5]:12223"
                 }
               },
               {
-                "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+                "id": "99c635b9-8e97-414a-8316-8261d2d767d0",
                 "underlay_address": "fd00:1122:3344:102::7",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:102::7]:17000"
-                }
-              },
-              {
-                "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
-                "underlay_address": "fd00:1122:3344:102::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::8]:32345",
+                  "address": "[fd00:1122:3344:102::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "658076ed-fb43-49e7-a813-df3d23497848",
-                "underlay_address": "fd00:1122:3344:102::10",
+                "id": "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
+                "underlay_address": "fd00:1122:3344:2::1",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::10]:32345",
+                  "type": "internal_dns",
                   "dataset": {
-                    "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
-                  }
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
+                  },
+                  "dns_address": "[fd00:1122:3344:2::1]:53",
+                  "gz_address": "fd00:1122:3344:2::2",
+                  "gz_address_index": 1,
+                  "http_address": "[fd00:1122:3344:2::1]:5353"
                 }
               },
               {
-                "id": "6edb836e-b100-4080-bb03-738d6441d571",
+                "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
                 "underlay_address": "fd00:1122:3344:102::c",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::c]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
-                  }
-                }
-              },
-              {
-                "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
-                "underlay_address": "fd00:1122:3344:102::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:102::c]:123",
+                  "dns_servers": [
                     "1.1.1.1",
                     "9.9.9.9"
                   ],
-                  "external_ip": "172.20.28.3",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:102::5]:12221",
                   "nic": {
-                    "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
-                    "ip": "172.30.2.6",
+                    "id": "659f33f7-e7bd-4875-9140-bf8a8b2d964a",
                     "kind": {
                       "type": "service",
-                      "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
+                      "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded"
                     },
-                    "mac": "A8:40:25:FF:B2:AF",
-                    "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                    "name": "ntp-d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
+                    "ip": "172.30.3.6",
+                    "mac": "A8:40:25:FF:B1:99",
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100,
                     "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100
+                    "slot": 0
+                  },
+                  "ntp_servers": [
+                    "time.cloudflare.com"
+                  ],
+                  "snat_cfg": {
+                    "ip": "198.51.100.26",
+                    "first_port": 16384,
+                    "last_port": 32767
                   }
                 }
               },
               {
-                "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
-                "underlay_address": "fd00:1122:3344:102::d",
+                "id": "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
+                "underlay_address": "fd00:1122:3344:102::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::d]:32345",
+                  "address": "[fd00:1122:3344:102::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                    "pool_name": "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b"
                   }
                 }
               },
               {
-                "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+                "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
                 "underlay_address": "fd00:1122:3344:102::4",
                 "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::4]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
-                  }
-                }
-              },
-              {
-                "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
-                "underlay_address": "fd00:1122:3344:102::6",
-                "zone_type": {
-                  "type": "clickhouse",
-                  "address": "[fd00:1122:3344:102::6]:8123",
-                  "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                  }
-                }
-              },
-              {
-                "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
-                "underlay_address": "fd00:1122:3344:102::f",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::f]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
-                  }
-                }
-              },
-              {
-                "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
-                "underlay_address": "fd00:1122:3344:102::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::3]:32221",
+                  "type": "external_dns",
                   "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                  }
-                }
-              },
-              {
-                "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-                "underlay_address": "fd00:1122:3344:102::12",
-                "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:102::12]:123",
-                  "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
+                  },
+                  "dns_address": "198.51.100.21:53",
+                  "http_address": "[fd00:1122:3344:102::4]:5353",
                   "nic": {
-                    "id": "9f03c82c-3192-498d-978d-a175049ac389",
-                    "ip": "172.30.3.6",
+                    "id": "d4dd2f7a-8a3b-405e-8444-62f99a754a8a",
                     "kind": {
                       "type": "service",
-                      "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
+                      "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802"
                     },
-                    "mac": "A8:40:25:FF:E4:11",
-                    "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                    "name": "external-dns-f0ed9838-6f1b-4646-a478-9f74f2e6e802",
+                    "ip": "172.30.1.6",
+                    "mac": "A8:40:25:FF:F4:5C",
+                    "subnet": "172.30.1.0/24",
+                    "vni": 100,
                     "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100
-                  },
-                  "ntp_servers": [
-                    "ntp.eng.oxide.computer"
-                  ],
-                  "snat_cfg": {
-                    "ip": "172.20.28.6",
-                    "first_port": 16384,
-                    "last_port": 32767
-                  }
-                }
-              },
-              {
-                "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
-                "underlay_address": "fd00:1122:3344:102::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
+                    "slot": 0
                   }
                 }
               }
             ]
           }
         },
-        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
-          "time_collected": "2024-03-01T19:17:11.153275Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
+        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
+          "time_collected": "2024-03-12T19:07:28.471504Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
           "zones": {
             "generation": 5,
             "zones": [
               {
-                "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
-                "underlay_address": "fd00:1122:3344:103::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
-                  }
-                }
-              },
-              {
-                "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
-                "underlay_address": "fd00:1122:3344:103::10",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::10]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
-                  }
-                }
-              },
-              {
-                "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
-                "underlay_address": "fd00:1122:3344:3::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  },
-                  "dns_address": "[fd00:1122:3344:3::1]:53",
-                  "gz_address": "fd00:1122:3344:3::2",
-                  "gz_address_index": 2,
-                  "http_address": "[fd00:1122:3344:3::1]:5353"
-                }
-              },
-              {
-                "id": "4b74c418-0a08-4489-b652-e97267c1c220",
-                "underlay_address": "fd00:1122:3344:103::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  }
-                }
-              },
-              {
-                "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
-                "underlay_address": "fd00:1122:3344:103::5",
+                "id": "0d459323-e414-4f32-9944-76c7331da622",
+                "underlay_address": "fd00:1122:3344:101::5",
                 "zone_type": {
                   "type": "nexus",
                   "external_dns_servers": [
                     "1.1.1.1",
                     "9.9.9.9"
                   ],
-                  "external_ip": "172.20.28.4",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:103::5]:12221",
+                  "external_ip": "198.51.100.24",
+                  "external_tls": false,
+                  "internal_address": "[fd00:1122:3344:101::5]:12221",
                   "nic": {
-                    "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
-                    "ip": "172.30.2.7",
+                    "id": "514902b8-7a1a-42a3-95a6-d00861833b5c",
                     "kind": {
                       "type": "service",
-                      "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
+                      "id": "0d459323-e414-4f32-9944-76c7331da622"
                     },
-                    "mac": "A8:40:25:FF:96:05",
-                    "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
-                    "primary": true,
-                    "slot": 0,
+                    "name": "nexus-0d459323-e414-4f32-9944-76c7331da622",
+                    "ip": "172.30.2.7",
+                    "mac": "A8:40:25:FF:DE:AC",
                     "subnet": "172.30.2.0/24",
-                    "vni": 100
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
-                "underlay_address": "fd00:1122:3344:103::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:103::6]:17000"
-                }
-              },
-              {
-                "id": "921958c0-7761-4294-945d-2a078327bf2c",
-                "underlay_address": "fd00:1122:3344:103::8",
+                "id": "2902972b-3ae4-489d-b937-6480352e214a",
+                "underlay_address": "fd00:1122:3344:101::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::8]:32345",
+                  "address": "[fd00:1122:3344:101::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
+                    "pool_name": "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "9df22034-59f6-431f-82e7-258fe983879b",
-                "underlay_address": "fd00:1122:3344:103::4",
+                "id": "2ee832de-af75-47ac-abac-f91304857ca4",
+                "underlay_address": "fd00:1122:3344:101::4",
                 "zone_type": {
                   "type": "external_dns",
                   "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   },
-                  "dns_address": "172.20.28.1:53",
-                  "http_address": "[fd00:1122:3344:103::4]:5353",
+                  "dns_address": "198.51.100.20:53",
+                  "http_address": "[fd00:1122:3344:101::4]:5353",
                   "nic": {
-                    "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
-                    "ip": "172.30.1.5",
+                    "id": "fadff4ab-62d7-466a-9203-04080e4078ed",
                     "kind": {
                       "type": "service",
-                      "id": "9df22034-59f6-431f-82e7-258fe983879b"
+                      "id": "2ee832de-af75-47ac-abac-f91304857ca4"
                     },
-                    "mac": "A8:40:25:FF:90:12",
-                    "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
-                    "primary": true,
-                    "slot": 0,
+                    "name": "external-dns-2ee832de-af75-47ac-abac-f91304857ca4",
+                    "ip": "172.30.1.5",
+                    "mac": "A8:40:25:FF:B4:93",
                     "subnet": "172.30.1.0/24",
-                    "vni": 100
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
-                "underlay_address": "fd00:1122:3344:103::11",
+                "id": "30050a7e-7fc3-4965-b07b-1635212393eb",
+                "underlay_address": "fd00:1122:3344:101::6",
                 "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:103::11]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
-                  ]
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:101::6]:17000"
                 }
               },
               {
-                "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
-                "underlay_address": "fd00:1122:3344:103::9",
+                "id": "8494befe-6df6-42d1-9baa-599ba4228430",
+                "underlay_address": "fd00:1122:3344:101::c",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:101::c]:123",
+                  "dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "nic": {
+                    "id": "23041748-e3a6-4846-930d-5a348c316e91",
+                    "kind": {
+                      "type": "service",
+                      "id": "8494befe-6df6-42d1-9baa-599ba4228430"
+                    },
+                    "name": "ntp-8494befe-6df6-42d1-9baa-599ba4228430",
+                    "ip": "172.30.3.5",
+                    "mac": "A8:40:25:FF:D1:C4",
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
+                  },
+                  "ntp_servers": [
+                    "time.cloudflare.com"
+                  ],
+                  "snat_cfg": {
+                    "ip": "198.51.100.25",
+                    "first_port": 0,
+                    "last_port": 16383
                   }
                 }
               },
               {
-                "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
-                "underlay_address": "fd00:1122:3344:103::d",
+                "id": "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
+                "underlay_address": "fd00:1122:3344:101::7",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::d]:32345",
+                  "address": "[fd00:1122:3344:101::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
-                "underlay_address": "fd00:1122:3344:103::b",
+                "id": "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
+                "underlay_address": "fd00:1122:3344:101::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::b]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
-                "underlay_address": "fd00:1122:3344:103::e",
+                "id": "e668e3f5-5457-4689-97da-b2af537787ac",
+                "underlay_address": "fd00:1122:3344:101::b",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::e]:32345",
+                  "address": "[fd00:1122:3344:101::b]:32345",
                   "dataset": {
-                    "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
+                    "pool_name": "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
-                "underlay_address": "fd00:1122:3344:103::3",
+                "id": "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
+                "underlay_address": "fd00:1122:3344:101::9",
                 "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:103::3]:32221",
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                    "pool_name": "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b"
                   }
                 }
               },
               {
-                "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
-                "underlay_address": "fd00:1122:3344:103::f",
+                "id": "f8c79ba4-194a-4cfe-8090-5315e297b780",
+                "underlay_address": "fd00:1122:3344:1::1",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::f]:32345",
+                  "type": "internal_dns",
                   "dataset": {
-                    "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
-                  }
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
+                  },
+                  "dns_address": "[fd00:1122:3344:1::1]:53",
+                  "gz_address": "fd00:1122:3344:1::2",
+                  "gz_address_index": 0,
+                  "http_address": "[fd00:1122:3344:1::1]:5353"
                 }
               },
               {
-                "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
-                "underlay_address": "fd00:1122:3344:103::c",
+                "id": "fa5e0176-b4a8-4598-aa31-40fd562d246c",
+                "underlay_address": "fd00:1122:3344:101::a",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::c]:32345",
+                  "address": "[fd00:1122:3344:101::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                    "pool_name": "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               }
@@ -3745,284 +3692,191 @@
       }
     },
     {
-      "id": "cd3547d7-f172-47d2-9bbf-9730e88b0559",
-      "errors": [
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a86ce874-ddce-4497-9e34-c7053d224035\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"a86ce874-ddce-4497-9e34-c7053d224035\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bf3dbdb6-cac9-43a1-a55e-973d2397ffe6\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"bf3dbdb6-cac9-43a1-a55e-973d2397ffe6\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fabd84d3-7c0c-4e60-99b4-f97b9288839c\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"fabd84d3-7c0c-4e60-99b4-f97b9288839c\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c0160967-c6b3-49b7-b801-0fcd6d4f5e31\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"c0160967-c6b3-49b7-b801-0fcd6d4f5e31\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"247b36be-d31b-460a-8722-58a21008eba3\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"247b36be-d31b-460a-8722-58a21008eba3\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"81998378-2641-4eab-8bcb-f335995e0af8\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"81998378-2641-4eab-8bcb-f335995e0af8\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8fcd5a26-f45a-47fc-9219-e57ac207fd2e\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"8fcd5a26-f45a-47fc-9219-e57ac207fd2e\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"20e66cd5-274f-487d-8186-c5e4cd25ae74\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:06 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"20e66cd5-274f-487d-8186-c5e4cd25ae74\" }"
-      ],
-      "time_started": "2024-03-01T19:18:03.580607Z",
-      "time_done": "2024-03-01T19:18:07.486108Z",
-      "collector": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+      "id": "3700dcd4-64bb-4997-bfdb-dcfc7cbbc998",
+      "errors": [],
+      "time_started": "2024-03-12T19:07:28.555090Z",
+      "time_done": "2024-03-12T19:07:28.728530Z",
+      "collector": "9cf90589-5da1-4b42-b09b-d784e4134637",
       "baseboards": [
         {
-          "part_number": "913-0000006",
-          "serial_number": "BRM44220005"
-        },
-        {
-          "part_number": "913-0000006",
-          "serial_number": "BRM44220011"
+          "part_number": "FAKE_SIM_SIDECAR",
+          "serial_number": "SimSidecar0"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220004"
+          "part_number": "i86pc",
+          "serial_number": "g0"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220046"
+          "part_number": "i86pc",
+          "serial_number": "g1"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220081"
+          "part_number": "i86pc",
+          "serial_number": "g2"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM44220001"
+          "part_number": "i86pc",
+          "serial_number": "g3"
         }
       ],
       "cabooses": [
         {
-          "board": "gimlet-c",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "gimlet-c",
-          "version": "1.0.6"
+          "board": "SimGimletSp",
+          "git_commit": "ffffffff",
+          "name": "SimGimlet",
+          "version": "0.0.1"
         },
         {
-          "board": "gimlet-c",
-          "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
-          "name": "gimlet-c",
-          "version": "1.0.5"
+          "board": "SimRot",
+          "git_commit": "eeeeeeee",
+          "name": "SimGimlet",
+          "version": "0.0.1"
         },
         {
-          "board": "gimlet-c",
-          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-          "name": "gimlet-c",
-          "version": "1.0.8"
+          "board": "SimRot",
+          "git_commit": "eeeeeeee",
+          "name": "SimSidecar",
+          "version": "0.0.1"
         },
         {
-          "board": "oxide-rot-1",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "oxide-rot-1",
-          "version": "1.0.5"
-        },
+          "board": "SimSidecarSp",
+          "git_commit": "ffffffff",
+          "name": "SimSidecar",
+          "version": "0.0.1"
+        }
+      ],
+      "rot_pages": [
         {
-          "board": "oxide-rot-1",
-          "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
-          "name": "oxide-rot-1",
-          "version": "1.0.4"
+          "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "board": "oxide-rot-1",
-          "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
-          "name": "oxide-rot-1",
-          "version": "0.0.0-git"
+          "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "board": "oxide-rot-1",
-          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-          "name": "oxide-rot-1",
-          "version": "1.0.6"
+          "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "board": "oxide-rot-1",
-          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-          "name": "oxide-rot-1",
-          "version": "1.0.0"
+          "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "board": "oxide-rot-1",
-          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-          "name": "oxide-rot-1-dev",
-          "version": "0.0.0-git"
+          "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "board": "sidecar-b",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "sidecar-b",
-          "version": "1.0.6"
+          "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "board": "sidecar-b",
-          "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
-          "name": "sidecar-b",
-          "version": "0.0.0-git"
+          "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "board": "sidecar-b",
-          "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
-          "name": "sidecar-b",
-          "version": "0.0.0-git"
-        }
-      ],
-      "rot_pages": [
-        {
-          "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
-        },
-        {
-          "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
-        },
-        {
-          "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
-        },
-        {
-          "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
-        },
-        {
-          "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
-        },
-        {
-          "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
-        },
-        {
-          "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+          "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         }
       ],
       "sps": [
         [
           {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220005"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:05.910026Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "sp_type": "switch",
-            "sp_slot": 0,
-            "baseboard_revision": 4,
-            "hubris_archive": "9184be93590e655e",
-            "power_state": "A0"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220011"
+            "part_number": "FAKE_SIM_SIDECAR",
+            "serial_number": "SimSidecar0"
           },
           {
-            "time_collected": "2024-03-01T19:18:06.485436Z",
-            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.560161Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "switch",
             "sp_slot": 1,
-            "baseboard_revision": 4,
-            "hubris_archive": "9184be93590e655e",
-            "power_state": "A0"
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
+            "part_number": "i86pc",
+            "serial_number": "g0"
           },
           {
-            "time_collected": "2024-03-01T19:18:04.749894Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.563500Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 17,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
+            "sp_slot": 0,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
+            "part_number": "i86pc",
+            "serial_number": "g1"
           },
           {
-            "time_collected": "2024-03-01T19:18:05.327348Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.566461Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 15,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
+            "sp_slot": 1,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
+            "part_number": "i86pc",
+            "serial_number": "g2"
           },
           {
-            "time_collected": "2024-03-01T19:18:04.172071Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.576218Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 14,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
+            "sp_slot": 2,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
+            "part_number": "i86pc",
+            "serial_number": "g3"
           },
           {
-            "time_collected": "2024-03-01T19:18:03.592833Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.556487Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 16,
-            "baseboard_revision": 6,
-            "hubris_archive": "3c670b44fdeef4af",
-            "power_state": "A0"
+            "sp_slot": 3,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ]
       ],
       "rots": [
         [
           {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220005"
+            "part_number": "FAKE_SIM_SIDECAR",
+            "serial_number": "SimSidecar0"
           },
           {
-            "time_collected": "2024-03-01T19:18:05.910026Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.560161Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
             "persistent_boot_preference": {
-              "slot": "b"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "4599e4d7bc9b2164e8c0f0f7ce4bcae2bd35d635c8450854ea9a94ce49a2ca0f",
-            "slot_b_sha3_256_digest": "1e42a0761e5bd1d9f9a5e2c44acc34e7b7e007792977cdb0523d271db2e80c8c"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220011"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:06.485436Z",
-            "source": "http://[fd00:1122:3344:103::2]:12225",
-            "active_slot": {
-              "slot": "b"
-            },
-            "persistent_boot_preference": {
-              "slot": "b"
+              "slot": "a"
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "92e07077972c83af6872ec817fb88d74a36f6956276505e9771302c5bb7ec2a8",
-            "slot_b_sha3_256_digest": "fb25825eaf97f8f2307afe3caebf0b70865499f6d116db3aff382c505b4d8dd7"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
+            "part_number": "i86pc",
+            "serial_number": "g0"
           },
           {
-            "time_collected": "2024-03-01T19:18:04.749894Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.563500Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
@@ -4031,18 +3885,18 @@
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
-            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
+            "part_number": "i86pc",
+            "serial_number": "g1"
           },
           {
-            "time_collected": "2024-03-01T19:18:05.327348Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.566461Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
@@ -4051,38 +3905,38 @@
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
-            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
+            "part_number": "i86pc",
+            "serial_number": "g2"
           },
           {
-            "time_collected": "2024-03-01T19:18:04.172071Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.576218Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
-              "slot": "b"
+              "slot": "a"
             },
             "persistent_boot_preference": {
-              "slot": "b"
+              "slot": "a"
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
-            "slot_b_sha3_256_digest": "d416d5ba9d0d3bd495f0f5fcd2aecc28cd83b53d1a8f94a28241d85b30761451"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
+            "part_number": "i86pc",
+            "serial_number": "g3"
           },
           {
-            "time_collected": "2024-03-01T19:18:03.592833Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:07:28.556487Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
@@ -4091,8 +3945,8 @@
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
-            "slot_b_sha3_256_digest": "4ec328195a23e86a1545d5ee576e73834aae81fb79830a54d0503475875f1760"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ]
       ],
@@ -4100,97 +3954,81 @@
         "SpSlot0": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:05.911536Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "sidecar-b",
-                "version": "1.0.6"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:18:06.486873Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.560637Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "sidecar-b",
-                "version": "1.0.6"
+                "board": "SimSidecarSp",
+                "git_commit": "ffffffff",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.751575Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.563951Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.332841Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.566869Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.173529Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.576918Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:03.594379Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.557119Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ]
@@ -4198,97 +4036,81 @@
         "SpSlot1": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:05.912952Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
-                "name": "sidecar-b",
-                "version": "0.0.0-git"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:18:06.488258Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.561066Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
-                "name": "sidecar-b",
-                "version": "0.0.0-git"
+                "board": "SimSidecarSp",
+                "git_commit": "ffffffff",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.753197Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.564309Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.334324Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.567309Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.174886Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.577611Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:03.596125Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.557613Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
-                "name": "gimlet-c",
-                "version": "1.0.5"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ]
@@ -4296,97 +4118,81 @@
         "RotSlotA": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:06.119739Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-                "name": "oxide-rot-1-dev",
-                "version": "0.0.0-git"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:18:06.695247Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.561535Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
-                "name": "oxide-rot-1",
-                "version": "0.0.0-git"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.004236Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.564659Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.586015Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.567749Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.426689Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.578148Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:03.847427Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.558097Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ]
@@ -4394,153 +4200,150 @@
         "RotSlotB": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:18:06.328695Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.561932Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:06.903153Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.565057Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-                "name": "oxide-rot-1",
-                "version": "1.0.0"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.256365Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.568166Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.838048Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.578556Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.678679Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.558565Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
-          ],
+          ]
+        ]
+      },
+      "rot_pages_found": {
+        "Cmpa": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.100460Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
-                "name": "oxide-rot-1",
-                "version": "1.0.4"
+              "time_collected": "2024-03-12T19:07:28.562164Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
-          ]
-        ]
-      },
-      "rot_pages_found": {
-        "Cmpa": [
+          ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.273195Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.565265Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.854538Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.574062Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.695195Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.578773Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.116988Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.558913Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
@@ -4548,53 +4351,66 @@
         "CfpaActive": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
+            },
+            {
+              "time_collected": "2024-03-12T19:07:28.562441Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.288821Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.565512Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.870545Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.574335Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.711196Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.579009Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.132904Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.559298Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
@@ -4602,53 +4418,66 @@
         "CfpaInactive": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
+            },
+            {
+              "time_collected": "2024-03-12T19:07:28.562788Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.304991Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.565754Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.886552Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.574960Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.727187Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.579245Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.148861Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.559529Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
@@ -4656,910 +4485,1010 @@
         "CfpaScratch": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
+            },
+            {
+              "time_collected": "2024-03-12T19:07:28.563108Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.320918Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.566172Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:05.902654Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.575618Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.743134Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.579487Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:04.164870Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:07:28.559780Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
         ]
       },
       "sled_agents": {
-        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
-          "time_collected": "2024-03-01T19:18:07.325811Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
-          },
-          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
-        },
-        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
-          "time_collected": "2024-03-01T19:18:07.365147Z",
-          "source": "http://[fd00:1122:3344:121::1]:12345",
-          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
-          },
-          "sled_agent_address": "[fd00:1122:3344:121::1]:12345",
+        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
+          "time_collected": "2024-03-12T19:07:28.622525Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
+          "baseboard_id": null,
+          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
           "sled_role": "gimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a452a7f5-b528-40fe-80ff-4e4189e2d52b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b452a7f5-b528-40fe-80ff-4e4189e2d52b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d452a7f5-b528-40fe-80ff-4e4189e2d52b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
+        },
+        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
+          "time_collected": "2024-03-12T19:07:28.654092Z",
+          "source": "http://[fd00:1122:3344:104::1]:12345",
+          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
+          "baseboard_id": null,
+          "sled_agent_address": "[fd00:1122:3344:104::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
         },
-        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
-          "time_collected": "2024-03-01T19:18:07.401947Z",
+        "77851655-10a5-4e80-88b0-a001473cab7e": {
+          "time_collected": "2024-03-12T19:07:28.685728Z",
           "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
-          },
+          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
+          "baseboard_id": null,
           "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
           "sled_role": "gimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a462a7f6-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b462a7f6-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d462a7f6-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
         },
-        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
-          "time_collected": "2024-03-01T19:18:07.440624Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
-          },
-          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
+        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
+          "time_collected": "2024-03-12T19:07:28.725605Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
+          "baseboard_id": null,
+          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
           "sled_role": "scrimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a462a4f4-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b462a4f4-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d462a4f4-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
         }
       },
       "omicron_zones": {
-        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
-          "time_collected": "2024-03-01T19:18:07.327154Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
+        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
+          "time_collected": "2024-03-12T19:07:28.623604Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
           "zones": {
             "generation": 5,
             "zones": [
               {
-                "id": "0706860a-3801-42a7-8000-fff741f7665b",
-                "underlay_address": "fd00:1122:3344:101::c",
+                "id": "0db88baf-911d-47bb-af71-3aebd7c96934",
+                "underlay_address": "fd00:1122:3344:103::4",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::c]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "198.51.100.22",
+                  "external_tls": false,
+                  "internal_address": "[fd00:1122:3344:103::4]:12221",
+                  "nic": {
+                    "id": "8847a09c-cabc-429b-bf46-66d5e2e1140c",
+                    "kind": {
+                      "type": "service",
+                      "id": "0db88baf-911d-47bb-af71-3aebd7c96934"
+                    },
+                    "name": "nexus-0db88baf-911d-47bb-af71-3aebd7c96934",
+                    "ip": "172.30.2.5",
+                    "mac": "A8:40:25:FF:CC:DA",
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "34a279a2-a025-41ff-a392-1addac400f38",
-                "underlay_address": "fd00:1122:3344:101::3",
+                "id": "213d7dd3-95a4-45a7-99ca-e4899991dab4",
+                "underlay_address": "fd00:1122:3344:103::6",
                 "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::3]:32221",
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::6]:32345",
                   "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
-                "underlay_address": "fd00:1122:3344:1::1",
+                "id": "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
+                "underlay_address": "fd00:1122:3344:3::1",
                 "zone_type": {
                   "type": "internal_dns",
                   "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   },
-                  "dns_address": "[fd00:1122:3344:1::1]:53",
-                  "gz_address": "fd00:1122:3344:1::2",
-                  "gz_address_index": 0,
-                  "http_address": "[fd00:1122:3344:1::1]:5353"
+                  "dns_address": "[fd00:1122:3344:3::1]:53",
+                  "gz_address": "fd00:1122:3344:3::2",
+                  "gz_address_index": 2,
+                  "http_address": "[fd00:1122:3344:3::1]:5353"
                 }
               },
               {
-                "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-                "underlay_address": "fd00:1122:3344:101::5",
+                "id": "40c47ead-3be8-4724-a8ef-b66233fc5c65",
+                "underlay_address": "fd00:1122:3344:103::7",
                 "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "172.20.28.2",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:101::5]:12221",
-                  "nic": {
-                    "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
-                    "ip": "172.30.2.5",
-                    "kind": {
-                      "type": "service",
-                      "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
-                    },
-                    "mac": "A8:40:25:FF:E1:17",
-                    "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:103::7]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
-                "underlay_address": "fd00:1122:3344:101::10",
+                "id": "425d9d2f-72ea-4f53-80b6-04e002771cdb",
+                "underlay_address": "fd00:1122:3344:103::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::10]:32345",
+                  "address": "[fd00:1122:3344:103::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+                    "pool_name": "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
-                "underlay_address": "fd00:1122:3344:101::f",
+                "id": "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
+                "underlay_address": "fd00:1122:3344:103::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::f]:32345",
+                  "address": "[fd00:1122:3344:103::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
+                    "pool_name": "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b"
                   }
                 }
               },
               {
-                "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
-                "underlay_address": "fd00:1122:3344:101::9",
+                "id": "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
+                "underlay_address": "fd00:1122:3344:103::5",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::9]:32345",
+                  "type": "clickhouse",
+                  "address": "[fd00:1122:3344:103::5]:8123",
                   "dataset": {
-                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
-                "underlay_address": "fd00:1122:3344:101::8",
+                "id": "7611a549-effc-41b6-b6cb-c03e0c70a31f",
+                "underlay_address": "fd00:1122:3344:103::b",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                  }
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:103::b]:123",
+                  "dns_servers": [
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
+                  ],
+                  "ntp_servers": [
+                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
+                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
+                  ]
                 }
               },
               {
-                "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
-                "underlay_address": "fd00:1122:3344:101::4",
+                "id": "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
+                "underlay_address": "fd00:1122:3344:103::3",
                 "zone_type": {
                   "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::4]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
-                  }
-                }
-              },
-              {
-                "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
-                "underlay_address": "fd00:1122:3344:101::d",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::d]:32345",
+                  "address": "[fd00:1122:3344:103::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
-                "underlay_address": "fd00:1122:3344:101::a",
+                "id": "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
+                "underlay_address": "fd00:1122:3344:103::a",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::a]:32345",
+                  "address": "[fd00:1122:3344:103::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
-                  }
-                }
-              },
-              {
-                "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
-                "underlay_address": "fd00:1122:3344:101::6",
-                "zone_type": {
-                  "type": "oximeter",
-                  "address": "[fd00:1122:3344:101::6]:12223"
-                }
-              },
-              {
-                "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
-                "underlay_address": "fd00:1122:3344:101::11",
-                "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:101::11]:123",
-                  "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "nic": {
-                    "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
-                    "ip": "172.30.3.5",
-                    "kind": {
-                      "type": "service",
-                      "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
-                    },
-                    "mac": "A8:40:25:FF:E7:A8",
-                    "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100
-                  },
-                  "ntp_servers": [
-                    "ntp.eng.oxide.computer"
-                  ],
-                  "snat_cfg": {
-                    "ip": "172.20.28.5",
-                    "first_port": 0,
-                    "last_port": 16383
+                    "pool_name": "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
-              },
+              }
+            ]
+          }
+        },
+        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
+          "time_collected": "2024-03-12T19:07:28.655005Z",
+          "source": "http://[fd00:1122:3344:104::1]:12345",
+          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
+          "zones": {
+            "generation": 5,
+            "zones": [
               {
-                "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
-                "underlay_address": "fd00:1122:3344:101::e",
+                "id": "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
+                "underlay_address": "fd00:1122:3344:104::b",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::e]:32345",
+                  "address": "[fd00:1122:3344:104::b]:32345",
                   "dataset": {
-                    "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                    "pool_name": "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
-                "underlay_address": "fd00:1122:3344:101::b",
+                "id": "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
+                "underlay_address": "fd00:1122:3344:104::4",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::b]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:104::4]:32221",
                   "dataset": {
-                    "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
+                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
-                "underlay_address": "fd00:1122:3344:101::7",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:101::7]:17000"
-                }
-              }
-            ]
-          }
-        },
-        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
-          "time_collected": "2024-03-01T19:18:07.365954Z",
-          "source": "http://[fd00:1122:3344:121::1]:12345",
-          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
-          "zones": {
-            "generation": 3,
-            "zones": [
-              {
-                "id": "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
-                "underlay_address": "fd00:1122:3344:121::28",
+                "id": "57c712a6-6429-44b7-a9b2-9957cb9054d4",
+                "underlay_address": "fd00:1122:3344:104::a",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::28]:32345",
+                  "address": "[fd00:1122:3344:104::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3"
+                    "pool_name": "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "12431d3e-1516-4822-98b3-3711ff912e69",
-                "underlay_address": "fd00:1122:3344:121::22",
+                "id": "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
+                "underlay_address": "fd00:1122:3344:104::7",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::22]:32345",
+                  "address": "[fd00:1122:3344:104::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_0622b2cf-297e-4356-b94c-d74309f443ba"
+                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
-                "underlay_address": "fd00:1122:3344:121::21",
+                "id": "73916098-788f-48d8-b745-e13fdcfb771b",
+                "underlay_address": "fd00:1122:3344:104::c",
                 "zone_type": {
                   "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:121::21]:123",
+                  "address": "[fd00:1122:3344:104::c]:123",
                   "dns_servers": [
                     "fd00:1122:3344:1::1",
                     "fd00:1122:3344:2::1",
                     "fd00:1122:3344:3::1"
                   ],
                   "ntp_servers": [
-                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
+                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
                   ]
                 }
               },
               {
-                "id": "56414d76-9b8c-4e15-acf1-cd269c2d8638",
-                "underlay_address": "fd00:1122:3344:121::25",
+                "id": "9bb513e3-7505-416c-aa70-b26f1152bbdc",
+                "underlay_address": "fd00:1122:3344:104::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::25]:32345",
+                  "address": "[fd00:1122:3344:104::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_574728ca-1559-48f3-a46c-1e608966e6a0"
+                    "pool_name": "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b"
                   }
                 }
               },
               {
-                "id": "891cb9da-301d-40cc-b93d-5fcb350fa968",
-                "underlay_address": "fd00:1122:3344:121::29",
+                "id": "9c14c879-137b-4f5b-8068-1292bffc83ed",
+                "underlay_address": "fd00:1122:3344:104::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::29]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
-                  }
-                }
-              },
-              {
-                "id": "d9014907-24c0-4c33-a475-db15942f4f8f",
-                "underlay_address": "fd00:1122:3344:121::23",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::23]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:104::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37"
+                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "e950de25-7306-4cf7-a6f4-6db11adaa310",
-                "underlay_address": "fd00:1122:3344:121::27",
+                "id": "9cf90589-5da1-4b42-b09b-d784e4134637",
+                "underlay_address": "fd00:1122:3344:104::5",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::27]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c"
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "198.51.100.23",
+                  "external_tls": false,
+                  "internal_address": "[fd00:1122:3344:104::5]:12221",
+                  "nic": {
+                    "id": "fa76e7e1-dee6-4574-b71f-e07ea1cffd08",
+                    "kind": {
+                      "type": "service",
+                      "id": "9cf90589-5da1-4b42-b09b-d784e4134637"
+                    },
+                    "name": "nexus-9cf90589-5da1-4b42-b09b-d784e4134637",
+                    "ip": "172.30.2.6",
+                    "mac": "A8:40:25:FF:C4:6F",
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
-                "underlay_address": "fd00:1122:3344:121::26",
+                "id": "d48b2dfa-c749-47dd-98b6-3f478cd63934",
+                "underlay_address": "fd00:1122:3344:104::6",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::26]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945"
-                  }
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:104::6]:17000"
                 }
               },
               {
-                "id": "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00",
-                "underlay_address": "fd00:1122:3344:121::24",
+                "id": "fd003bee-818a-4c95-aae5-e378ac601522",
+                "underlay_address": "fd00:1122:3344:104::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::24]:32345",
+                  "address": "[fd00:1122:3344:104::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1"
+                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               }
             ]
           }
         },
-        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
-          "time_collected": "2024-03-01T19:18:07.403197Z",
+        "77851655-10a5-4e80-88b0-a001473cab7e": {
+          "time_collected": "2024-03-12T19:07:28.687431Z",
           "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
           "zones": {
             "generation": 5,
             "zones": [
               {
-                "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
-                "underlay_address": "fd00:1122:3344:102::11",
+                "id": "084cac6e-44b5-4333-a2ff-b0993b534898",
+                "underlay_address": "fd00:1122:3344:102::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::11]:32345",
+                  "address": "[fd00:1122:3344:102::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                    "pool_name": "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
-                "underlay_address": "fd00:1122:3344:102::e",
+                "id": "11c527c5-0598-428e-9270-be5b74d3461b",
+                "underlay_address": "fd00:1122:3344:102::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::e]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "479811e7-6409-45d4-b294-5333534e47ed",
-                "underlay_address": "fd00:1122:3344:102::9",
+                "id": "20be9299-495d-428d-b315-694c2c4b64af",
+                "underlay_address": "fd00:1122:3344:102::6",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:102::6]:17000"
+                }
+              },
+              {
+                "id": "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
+                "underlay_address": "fd00:1122:3344:102::b",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::9]:32345",
+                  "address": "[fd00:1122:3344:102::b]:32345",
                   "dataset": {
-                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                    "pool_name": "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
+                "id": "8d452bdd-e8b8-4108-a690-46fc59197db0",
                 "underlay_address": "fd00:1122:3344:102::a",
                 "zone_type": {
                   "type": "crucible",
                   "address": "[fd00:1122:3344:102::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                    "pool_name": "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
-                "underlay_address": "fd00:1122:3344:2::1",
+                "id": "953019c0-8e85-4417-b231-d42820470125",
+                "underlay_address": "fd00:1122:3344:102::5",
                 "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                  },
-                  "dns_address": "[fd00:1122:3344:2::1]:53",
-                  "gz_address": "fd00:1122:3344:2::2",
-                  "gz_address_index": 1,
-                  "http_address": "[fd00:1122:3344:2::1]:5353"
+                  "type": "oximeter",
+                  "address": "[fd00:1122:3344:102::5]:12223"
                 }
               },
               {
-                "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+                "id": "99c635b9-8e97-414a-8316-8261d2d767d0",
                 "underlay_address": "fd00:1122:3344:102::7",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:102::7]:17000"
-                }
-              },
-              {
-                "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
-                "underlay_address": "fd00:1122:3344:102::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::8]:32345",
+                  "address": "[fd00:1122:3344:102::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "658076ed-fb43-49e7-a813-df3d23497848",
-                "underlay_address": "fd00:1122:3344:102::10",
+                "id": "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
+                "underlay_address": "fd00:1122:3344:2::1",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::10]:32345",
+                  "type": "internal_dns",
                   "dataset": {
-                    "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
-                  }
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
+                  },
+                  "dns_address": "[fd00:1122:3344:2::1]:53",
+                  "gz_address": "fd00:1122:3344:2::2",
+                  "gz_address_index": 1,
+                  "http_address": "[fd00:1122:3344:2::1]:5353"
                 }
               },
               {
-                "id": "6edb836e-b100-4080-bb03-738d6441d571",
+                "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
                 "underlay_address": "fd00:1122:3344:102::c",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::c]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
-                  }
-                }
-              },
-              {
-                "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
-                "underlay_address": "fd00:1122:3344:102::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:102::c]:123",
+                  "dns_servers": [
                     "1.1.1.1",
                     "9.9.9.9"
                   ],
-                  "external_ip": "172.20.28.3",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:102::5]:12221",
                   "nic": {
-                    "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
-                    "ip": "172.30.2.6",
+                    "id": "659f33f7-e7bd-4875-9140-bf8a8b2d964a",
                     "kind": {
                       "type": "service",
-                      "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
+                      "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded"
                     },
-                    "mac": "A8:40:25:FF:B2:AF",
-                    "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                    "name": "ntp-d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
+                    "ip": "172.30.3.6",
+                    "mac": "A8:40:25:FF:B1:99",
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100,
                     "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100
+                    "slot": 0
+                  },
+                  "ntp_servers": [
+                    "time.cloudflare.com"
+                  ],
+                  "snat_cfg": {
+                    "ip": "198.51.100.26",
+                    "first_port": 16384,
+                    "last_port": 32767
                   }
                 }
               },
               {
-                "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
-                "underlay_address": "fd00:1122:3344:102::d",
+                "id": "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
+                "underlay_address": "fd00:1122:3344:102::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::d]:32345",
+                  "address": "[fd00:1122:3344:102::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                    "pool_name": "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b"
                   }
                 }
               },
               {
-                "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+                "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
                 "underlay_address": "fd00:1122:3344:102::4",
                 "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::4]:32221",
+                  "type": "external_dns",
                   "dataset": {
-                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
+                  },
+                  "dns_address": "198.51.100.21:53",
+                  "http_address": "[fd00:1122:3344:102::4]:5353",
+                  "nic": {
+                    "id": "d4dd2f7a-8a3b-405e-8444-62f99a754a8a",
+                    "kind": {
+                      "type": "service",
+                      "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802"
+                    },
+                    "name": "external-dns-f0ed9838-6f1b-4646-a478-9f74f2e6e802",
+                    "ip": "172.30.1.6",
+                    "mac": "A8:40:25:FF:F4:5C",
+                    "subnet": "172.30.1.0/24",
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
-              },
+              }
+            ]
+          }
+        },
+        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
+          "time_collected": "2024-03-12T19:07:28.727281Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
+          "zones": {
+            "generation": 5,
+            "zones": [
               {
-                "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
-                "underlay_address": "fd00:1122:3344:102::6",
+                "id": "0d459323-e414-4f32-9944-76c7331da622",
+                "underlay_address": "fd00:1122:3344:101::5",
                 "zone_type": {
-                  "type": "clickhouse",
-                  "address": "[fd00:1122:3344:102::6]:8123",
-                  "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "198.51.100.24",
+                  "external_tls": false,
+                  "internal_address": "[fd00:1122:3344:101::5]:12221",
+                  "nic": {
+                    "id": "514902b8-7a1a-42a3-95a6-d00861833b5c",
+                    "kind": {
+                      "type": "service",
+                      "id": "0d459323-e414-4f32-9944-76c7331da622"
+                    },
+                    "name": "nexus-0d459323-e414-4f32-9944-76c7331da622",
+                    "ip": "172.30.2.7",
+                    "mac": "A8:40:25:FF:DE:AC",
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
-                "underlay_address": "fd00:1122:3344:102::f",
+                "id": "2902972b-3ae4-489d-b937-6480352e214a",
+                "underlay_address": "fd00:1122:3344:101::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::f]:32345",
+                  "address": "[fd00:1122:3344:101::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
+                    "pool_name": "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
-                "underlay_address": "fd00:1122:3344:102::3",
+                "id": "2ee832de-af75-47ac-abac-f91304857ca4",
+                "underlay_address": "fd00:1122:3344:101::4",
                 "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::3]:32221",
+                  "type": "external_dns",
                   "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
+                  },
+                  "dns_address": "198.51.100.20:53",
+                  "http_address": "[fd00:1122:3344:101::4]:5353",
+                  "nic": {
+                    "id": "fadff4ab-62d7-466a-9203-04080e4078ed",
+                    "kind": {
+                      "type": "service",
+                      "id": "2ee832de-af75-47ac-abac-f91304857ca4"
+                    },
+                    "name": "external-dns-2ee832de-af75-47ac-abac-f91304857ca4",
+                    "ip": "172.30.1.5",
+                    "mac": "A8:40:25:FF:B4:93",
+                    "subnet": "172.30.1.0/24",
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-                "underlay_address": "fd00:1122:3344:102::12",
+                "id": "30050a7e-7fc3-4965-b07b-1635212393eb",
+                "underlay_address": "fd00:1122:3344:101::6",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:101::6]:17000"
+                }
+              },
+              {
+                "id": "8494befe-6df6-42d1-9baa-599ba4228430",
+                "underlay_address": "fd00:1122:3344:101::c",
                 "zone_type": {
                   "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:102::12]:123",
+                  "address": "[fd00:1122:3344:101::c]:123",
                   "dns_servers": [
                     "1.1.1.1",
                     "9.9.9.9"
                   ],
                   "nic": {
-                    "id": "9f03c82c-3192-498d-978d-a175049ac389",
-                    "ip": "172.30.3.6",
+                    "id": "23041748-e3a6-4846-930d-5a348c316e91",
                     "kind": {
                       "type": "service",
-                      "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
+                      "id": "8494befe-6df6-42d1-9baa-599ba4228430"
                     },
-                    "mac": "A8:40:25:FF:E4:11",
-                    "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-                    "primary": true,
-                    "slot": 0,
+                    "name": "ntp-8494befe-6df6-42d1-9baa-599ba4228430",
+                    "ip": "172.30.3.5",
+                    "mac": "A8:40:25:FF:D1:C4",
                     "subnet": "172.30.3.0/24",
-                    "vni": 100
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   },
                   "ntp_servers": [
-                    "ntp.eng.oxide.computer"
+                    "time.cloudflare.com"
                   ],
                   "snat_cfg": {
-                    "ip": "172.20.28.6",
-                    "first_port": 16384,
-                    "last_port": 32767
+                    "ip": "198.51.100.25",
+                    "first_port": 0,
+                    "last_port": 16383
                   }
                 }
               },
               {
-                "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
-                "underlay_address": "fd00:1122:3344:102::b",
+                "id": "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
+                "underlay_address": "fd00:1122:3344:101::7",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::b]:32345",
+                  "address": "[fd00:1122:3344:101::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
-              }
-            ]
-          }
-        },
-        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
-          "time_collected": "2024-03-01T19:18:07.483569Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
-          "zones": {
-            "generation": 5,
-            "zones": [
+              },
               {
-                "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
-                "underlay_address": "fd00:1122:3344:103::a",
+                "id": "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
+                "underlay_address": "fd00:1122:3344:101::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::a]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
-                "underlay_address": "fd00:1122:3344:103::10",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::10]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
-                  }
-                }
-              },
-              {
-                "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
-                "underlay_address": "fd00:1122:3344:3::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  },
-                  "dns_address": "[fd00:1122:3344:3::1]:53",
-                  "gz_address": "fd00:1122:3344:3::2",
-                  "gz_address_index": 2,
-                  "http_address": "[fd00:1122:3344:3::1]:5353"
-                }
-              },
-              {
-                "id": "4b74c418-0a08-4489-b652-e97267c1c220",
-                "underlay_address": "fd00:1122:3344:103::7",
+                "id": "e668e3f5-5457-4689-97da-b2af537787ac",
+                "underlay_address": "fd00:1122:3344:101::b",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::7]:32345",
+                  "address": "[fd00:1122:3344:101::b]:32345",
                   "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  }
-                }
-              },
-              {
-                "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
-                "underlay_address": "fd00:1122:3344:103::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "172.20.28.4",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:103::5]:12221",
-                  "nic": {
-                    "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
-                    "ip": "172.30.2.7",
-                    "kind": {
-                      "type": "service",
-                      "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
-                    },
-                    "mac": "A8:40:25:FF:96:05",
-                    "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100
+                    "pool_name": "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
-                "underlay_address": "fd00:1122:3344:103::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:103::6]:17000"
-                }
-              },
-              {
-                "id": "921958c0-7761-4294-945d-2a078327bf2c",
-                "underlay_address": "fd00:1122:3344:103::8",
+                "id": "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
+                "underlay_address": "fd00:1122:3344:101::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::8]:32345",
+                  "address": "[fd00:1122:3344:101::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
+                    "pool_name": "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b"
                   }
                 }
               },
               {
-                "id": "9df22034-59f6-431f-82e7-258fe983879b",
-                "underlay_address": "fd00:1122:3344:103::4",
+                "id": "f8c79ba4-194a-4cfe-8090-5315e297b780",
+                "underlay_address": "fd00:1122:3344:1::1",
                 "zone_type": {
-                  "type": "external_dns",
+                  "type": "internal_dns",
                   "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   },
-                  "dns_address": "172.20.28.1:53",
-                  "http_address": "[fd00:1122:3344:103::4]:5353",
-                  "nic": {
-                    "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
-                    "ip": "172.30.1.5",
-                    "kind": {
-                      "type": "service",
-                      "id": "9df22034-59f6-431f-82e7-258fe983879b"
-                    },
-                    "mac": "A8:40:25:FF:90:12",
-                    "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.1.0/24",
-                    "vni": 100
-                  }
-                }
-              },
-              {
-                "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
-                "underlay_address": "fd00:1122:3344:103::11",
-                "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:103::11]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
-                  ]
-                }
-              },
-              {
-                "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
-                "underlay_address": "fd00:1122:3344:103::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
-                  }
-                }
-              },
-              {
-                "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
-                "underlay_address": "fd00:1122:3344:103::d",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::d]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
-                  }
-                }
-              },
-              {
-                "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
-                "underlay_address": "fd00:1122:3344:103::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
-                  }
-                }
-              },
-              {
-                "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
-                "underlay_address": "fd00:1122:3344:103::e",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::e]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
-                  }
-                }
-              },
-              {
-                "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
-                "underlay_address": "fd00:1122:3344:103::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:103::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  }
-                }
-              },
-              {
-                "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
-                "underlay_address": "fd00:1122:3344:103::f",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::f]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
-                  }
+                  "dns_address": "[fd00:1122:3344:1::1]:53",
+                  "gz_address": "fd00:1122:3344:1::2",
+                  "gz_address_index": 0,
+                  "http_address": "[fd00:1122:3344:1::1]:5353"
                 }
               },
               {
-                "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
-                "underlay_address": "fd00:1122:3344:103::c",
+                "id": "fa5e0176-b4a8-4598-aa31-40fd562d246c",
+                "underlay_address": "fd00:1122:3344:101::a",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::c]:32345",
+                  "address": "[fd00:1122:3344:101::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                    "pool_name": "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               }
@@ -5569,284 +5498,191 @@
       }
     },
     {
-      "id": "623f7e48-6c47-4e17-8aa3-cebe98ca4287",
-      "errors": [
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"431a5722-e328-450a-89f4-017274532055\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:10 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"431a5722-e328-450a-89f4-017274532055\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"38399802-a750-4c20-8243-f8df25e6c01b\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"38399802-a750-4c20-8243-f8df25e6c01b\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cbbbe7aa-4a9f-44d3-9713-6c12a1d2b00a\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"cbbbe7aa-4a9f-44d3-9713-6c12a1d2b00a\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a93a52ae-2837-4a80-a8d9-68f43bdaabe9\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"a93a52ae-2837-4a80-a8d9-68f43bdaabe9\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0016667c-69f3-4359-9ed8-176e8d03f270\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"0016667c-69f3-4359-9ed8-176e8d03f270\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0f8d60bc-31ee-460d-a8b5-9073728c2402\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"0f8d60bc-31ee-460d-a8b5-9073728c2402\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"595cf967-cade-4ae6-87f5-d04e5dbc55b9\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"595cf967-cade-4ae6-87f5-d04e5dbc55b9\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"83fe7422-15cd-48d3-b21d-967479fb090c\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"83fe7422-15cd-48d3-b21d-967479fb090c\" }"
-      ],
-      "time_started": "2024-03-01T19:18:07.755939Z",
-      "time_done": "2024-03-01T19:18:12.110340Z",
-      "collector": "54c947d2-6355-453c-80fc-8f49cc2129ee",
+      "id": "5e1d6ae0-c5b4-4884-ac41-680cd4f5762d",
+      "errors": [],
+      "time_started": "2024-03-12T19:09:36.927333Z",
+      "time_done": "2024-03-12T19:09:37.116573Z",
+      "collector": "0d459323-e414-4f32-9944-76c7331da622",
       "baseboards": [
         {
-          "part_number": "913-0000006",
-          "serial_number": "BRM44220005"
-        },
-        {
-          "part_number": "913-0000006",
-          "serial_number": "BRM44220011"
+          "part_number": "FAKE_SIM_SIDECAR",
+          "serial_number": "SimSidecar0"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220004"
+          "part_number": "i86pc",
+          "serial_number": "g0"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220046"
+          "part_number": "i86pc",
+          "serial_number": "g1"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220081"
+          "part_number": "i86pc",
+          "serial_number": "g2"
         },
         {
-          "part_number": "913-0000019",
-          "serial_number": "BRM44220001"
+          "part_number": "i86pc",
+          "serial_number": "g3"
         }
       ],
       "cabooses": [
         {
-          "board": "gimlet-c",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "gimlet-c",
-          "version": "1.0.6"
-        },
-        {
-          "board": "gimlet-c",
-          "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
-          "name": "gimlet-c",
-          "version": "1.0.5"
-        },
-        {
-          "board": "gimlet-c",
-          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-          "name": "gimlet-c",
-          "version": "1.0.8"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "oxide-rot-1",
-          "version": "1.0.5"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
-          "name": "oxide-rot-1",
-          "version": "1.0.4"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
-          "name": "oxide-rot-1",
-          "version": "0.0.0-git"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-          "name": "oxide-rot-1",
-          "version": "1.0.6"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-          "name": "oxide-rot-1",
-          "version": "1.0.0"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-          "name": "oxide-rot-1-dev",
-          "version": "0.0.0-git"
+          "board": "SimGimletSp",
+          "git_commit": "ffffffff",
+          "name": "SimGimlet",
+          "version": "0.0.1"
         },
         {
-          "board": "sidecar-b",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "sidecar-b",
-          "version": "1.0.6"
+          "board": "SimRot",
+          "git_commit": "eeeeeeee",
+          "name": "SimGimlet",
+          "version": "0.0.1"
         },
         {
-          "board": "sidecar-b",
-          "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
-          "name": "sidecar-b",
-          "version": "0.0.0-git"
+          "board": "SimRot",
+          "git_commit": "eeeeeeee",
+          "name": "SimSidecar",
+          "version": "0.0.1"
         },
         {
-          "board": "sidecar-b",
-          "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
-          "name": "sidecar-b",
-          "version": "0.0.0-git"
+          "board": "SimSidecarSp",
+          "git_commit": "ffffffff",
+          "name": "SimSidecar",
+          "version": "0.0.1"
         }
       ],
       "rot_pages": [
         {
-          "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+          "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+        },
+        {
+          "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+          "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+          "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+          "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+          "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+          "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         },
         {
-          "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+          "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
         }
       ],
       "sps": [
         [
           {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220005"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:10.815811Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "sp_type": "switch",
-            "sp_slot": 0,
-            "baseboard_revision": 4,
-            "hubris_archive": "9184be93590e655e",
-            "power_state": "A0"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220011"
+            "part_number": "FAKE_SIM_SIDECAR",
+            "serial_number": "SimSidecar0"
           },
           {
-            "time_collected": "2024-03-01T19:18:11.387242Z",
-            "source": "http://[fd00:1122:3344:103::2]:12225",
+            "time_collected": "2024-03-12T19:09:36.934628Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "switch",
             "sp_slot": 1,
-            "baseboard_revision": 4,
-            "hubris_archive": "9184be93590e655e",
-            "power_state": "A0"
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
+            "part_number": "i86pc",
+            "serial_number": "g0"
           },
           {
-            "time_collected": "2024-03-01T19:18:09.650491Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:09:36.940025Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 17,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
+            "sp_slot": 0,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
+            "part_number": "i86pc",
+            "serial_number": "g1"
           },
           {
-            "time_collected": "2024-03-01T19:18:10.237239Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:09:36.945090Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 15,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
+            "sp_slot": 1,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
+            "part_number": "i86pc",
+            "serial_number": "g2"
           },
           {
-            "time_collected": "2024-03-01T19:18:08.884903Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:09:36.949989Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 14,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
+            "sp_slot": 2,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
+            "part_number": "i86pc",
+            "serial_number": "g3"
           },
           {
-            "time_collected": "2024-03-01T19:18:07.768812Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:09:36.929081Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "sp_type": "sled",
-            "sp_slot": 16,
-            "baseboard_revision": 6,
-            "hubris_archive": "3c670b44fdeef4af",
-            "power_state": "A0"
+            "sp_slot": 3,
+            "baseboard_revision": 0,
+            "hubris_archive": "0000000000000000",
+            "power_state": "A2"
           }
         ]
       ],
       "rots": [
         [
           {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220005"
+            "part_number": "FAKE_SIM_SIDECAR",
+            "serial_number": "SimSidecar0"
           },
           {
-            "time_collected": "2024-03-01T19:18:10.815811Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:09:36.934628Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
             "persistent_boot_preference": {
-              "slot": "b"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "4599e4d7bc9b2164e8c0f0f7ce4bcae2bd35d635c8450854ea9a94ce49a2ca0f",
-            "slot_b_sha3_256_digest": "1e42a0761e5bd1d9f9a5e2c44acc34e7b7e007792977cdb0523d271db2e80c8c"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220011"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:11.387242Z",
-            "source": "http://[fd00:1122:3344:103::2]:12225",
-            "active_slot": {
-              "slot": "b"
-            },
-            "persistent_boot_preference": {
-              "slot": "b"
+              "slot": "a"
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "92e07077972c83af6872ec817fb88d74a36f6956276505e9771302c5bb7ec2a8",
-            "slot_b_sha3_256_digest": "fb25825eaf97f8f2307afe3caebf0b70865499f6d116db3aff382c505b4d8dd7"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
+            "part_number": "i86pc",
+            "serial_number": "g0"
           },
           {
-            "time_collected": "2024-03-01T19:18:09.650491Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:09:36.940025Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
@@ -5855,18 +5691,18 @@
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
-            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
+            "part_number": "i86pc",
+            "serial_number": "g1"
           },
           {
-            "time_collected": "2024-03-01T19:18:10.237239Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:09:36.945090Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
@@ -5875,38 +5711,38 @@
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
-            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
+            "part_number": "i86pc",
+            "serial_number": "g2"
           },
           {
-            "time_collected": "2024-03-01T19:18:08.884903Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:09:36.949989Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
-              "slot": "b"
+              "slot": "a"
             },
             "persistent_boot_preference": {
-              "slot": "b"
+              "slot": "a"
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
-            "slot_b_sha3_256_digest": "d416d5ba9d0d3bd495f0f5fcd2aecc28cd83b53d1a8f94a28241d85b30761451"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ],
         [
           {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
+            "part_number": "i86pc",
+            "serial_number": "g3"
           },
           {
-            "time_collected": "2024-03-01T19:18:07.768812Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
+            "time_collected": "2024-03-12T19:09:36.929081Z",
+            "source": "http://[fd00:1122:3344:104::2]:12225",
             "active_slot": {
               "slot": "a"
             },
@@ -5915,8 +5751,8 @@
             },
             "pending_persistent_boot_preference": null,
             "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
-            "slot_b_sha3_256_digest": "4ec328195a23e86a1545d5ee576e73834aae81fb79830a54d0503475875f1760"
+            "slot_a_sha3_256_digest": null,
+            "slot_b_sha3_256_digest": null
           }
         ]
       ],
@@ -5924,97 +5760,81 @@
         "SpSlot0": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.817224Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "sidecar-b",
-                "version": "1.0.6"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:18:11.388557Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.935488Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "sidecar-b",
-                "version": "1.0.6"
+                "board": "SimSidecarSp",
+                "git_commit": "ffffffff",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:09.651842Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.940638Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:10.238618Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.945685Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:08.886274Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.950734Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:07.777292Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.929758Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ]
@@ -6022,97 +5842,81 @@
         "SpSlot1": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.822223Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
-                "name": "sidecar-b",
-                "version": "0.0.0-git"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:18:11.389874Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.936167Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
-                "name": "sidecar-b",
-                "version": "0.0.0-git"
+                "board": "SimSidecarSp",
+                "git_commit": "ffffffff",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:09.653307Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.941223Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:10.240066Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.946254Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:08.887692Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.951335Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:07.779701Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.930670Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
-                "name": "gimlet-c",
-                "version": "1.0.5"
+                "board": "SimGimletSp",
+                "git_commit": "ffffffff",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ]
@@ -6120,97 +5924,81 @@
         "RotSlotA": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:11.029595Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-                "name": "oxide-rot-1-dev",
-                "version": "0.0.0-git"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:18:11.595974Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.936811Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
-                "name": "oxide-rot-1",
-                "version": "0.0.0-git"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:09.905198Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.941800Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:10.491985Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.946826Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:09.148589Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.951934Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:08.142368Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.931245Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ]
@@ -6218,153 +6006,150 @@
         "RotSlotB": [
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:18:11.236464Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.937534Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimSidecar",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:11.830868Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.942409Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-                "name": "oxide-rot-1",
-                "version": "1.0.0"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:10.164175Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.947558Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:10.743940Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.953023Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:09.402608Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.931825Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
+                "board": "SimRot",
+                "git_commit": "eeeeeeee",
+                "name": "SimGimlet",
+                "version": "0.0.1"
               }
             }
-          ],
+          ]
+        ]
+      },
+      "rot_pages_found": {
+        "Cmpa": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
             },
             {
-              "time_collected": "2024-03-01T19:18:08.642339Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
-                "name": "oxide-rot-1",
-                "version": "1.0.4"
+              "time_collected": "2024-03-12T19:09:36.938012Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
-          ]
-        ]
-      },
-      "rot_pages_found": {
-        "Cmpa": [
+          ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:10.181324Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.942853Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:10.760461Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.948038Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:09.419090Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.954159Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:08.722917Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.932312Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
@@ -6372,53 +6157,66 @@
         "CfpaActive": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
+            },
+            {
+              "time_collected": "2024-03-12T19:09:36.938490Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:10.197812Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.943336Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:10.776526Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.948514Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:09.491156Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.954600Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:08.813851Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.932820Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
@@ -6426,53 +6224,66 @@
         "CfpaInactive": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
+            },
+            {
+              "time_collected": "2024-03-12T19:09:36.938996Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:10.213728Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.944180Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:10.792457Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.948958Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:09.562113Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.955267Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:08.845830Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.933319Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
@@ -6480,910 +6291,1010 @@
         "CfpaScratch": [
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
+              "part_number": "FAKE_SIM_SIDECAR",
+              "serial_number": "SimSidecar0"
+            },
+            {
+              "time_collected": "2024-03-12T19:09:36.939491Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
+              "page": {
+                "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+              }
+            }
+          ],
+          [
+            {
+              "part_number": "i86pc",
+              "serial_number": "g0"
             },
             {
-              "time_collected": "2024-03-01T19:18:10.229796Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.944650Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
+              "part_number": "i86pc",
+              "serial_number": "g1"
             },
             {
-              "time_collected": "2024-03-01T19:18:10.809035Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.949453Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
+              "part_number": "i86pc",
+              "serial_number": "g2"
             },
             {
-              "time_collected": "2024-03-01T19:18:09.643193Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.955924Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ],
           [
             {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
+              "part_number": "i86pc",
+              "serial_number": "g3"
             },
             {
-              "time_collected": "2024-03-01T19:18:08.878183Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
+              "time_collected": "2024-03-12T19:09:36.933794Z",
+              "source": "http://[fd00:1122:3344:104::2]:12225",
               "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
+                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
               }
             }
           ]
         ]
       },
       "sled_agents": {
-        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
-          "time_collected": "2024-03-01T19:18:12.000624Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
-          },
-          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
-        },
-        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
-          "time_collected": "2024-03-01T19:18:12.036086Z",
-          "source": "http://[fd00:1122:3344:121::1]:12345",
-          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
-          },
-          "sled_agent_address": "[fd00:1122:3344:121::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
-        },
-        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
-          "time_collected": "2024-03-01T19:18:12.071731Z",
-          "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
-          },
-          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
-        },
-        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
-          "time_collected": "2024-03-01T19:18:12.107230Z",
+        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
+          "time_collected": "2024-03-12T19:09:37.008918Z",
           "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
-          },
+          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
+          "baseboard_id": null,
           "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
-        }
-      },
-      "omicron_zones": {
-        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
-          "time_collected": "2024-03-01T19:18:12.001507Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "0706860a-3801-42a7-8000-fff741f7665b",
-                "underlay_address": "fd00:1122:3344:101::c",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::c]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
-                  }
-                }
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a452a7f5-b528-40fe-80ff-4e4189e2d52b",
+                "model": "synthetic-model"
               },
-              {
-                "id": "34a279a2-a025-41ff-a392-1addac400f38",
-                "underlay_address": "fd00:1122:3344:101::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                  }
-                }
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b452a7f5-b528-40fe-80ff-4e4189e2d52b",
+                "model": "synthetic-model"
               },
-              {
-                "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
-                "underlay_address": "fd00:1122:3344:1::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                  },
-                  "dns_address": "[fd00:1122:3344:1::1]:53",
-                  "gz_address": "fd00:1122:3344:1::2",
-                  "gz_address_index": 0,
-                  "http_address": "[fd00:1122:3344:1::1]:5353"
-                }
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d452a7f5-b528-40fe-80ff-4e4189e2d52b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc85-ab45-49fb-a4b4-d351ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
+        },
+        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
+          "time_collected": "2024-03-12T19:09:37.040839Z",
+          "source": "http://[fd00:1122:3344:104::1]:12345",
+          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
+          "baseboard_id": null,
+          "sled_agent_address": "[fd00:1122:3344:104::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
               },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
+        },
+        "77851655-10a5-4e80-88b0-a001473cab7e": {
+          "time_collected": "2024-03-12T19:09:37.086600Z",
+          "source": "http://[fd00:1122:3344:102::1]:12345",
+          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
+          "baseboard_id": null,
+          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
+          "sled_role": "gimlet",
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a462a7f6-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b462a7f6-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d462a7f6-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc86-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
+        },
+        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
+          "time_collected": "2024-03-12T19:09:37.114717Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
+          "baseboard_id": null,
+          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
+          "sled_role": "scrimlet",
+          "usable_hardware_threads": 8,
+          "usable_physical_ram": 8582262784,
+          "reservoir_size": 0,
+          "disks": [
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-a462a4f4-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1024
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-b462a4f4-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "m2",
+              "slot": 1025
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-d462a4f4-b628-40fe-80ff-4e4189e2d62b",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1026
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-e4b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1027
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-f4b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1028
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-14b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1029
+            },
+            {
+              "identity": {
+                "vendor": "synthetic-vendor",
+                "serial": "synthetic-serial-24b4dc84-ab46-49fb-a4b4-d361ae214c03",
+                "model": "synthetic-model"
+              },
+              "variant": "u2",
+              "slot": 1030
+            }
+          ]
+        }
+      },
+      "omicron_zones": {
+        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
+          "time_collected": "2024-03-12T19:09:37.010960Z",
+          "source": "http://[fd00:1122:3344:103::1]:12345",
+          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
+          "zones": {
+            "generation": 5,
+            "zones": [
               {
-                "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-                "underlay_address": "fd00:1122:3344:101::5",
+                "id": "0db88baf-911d-47bb-af71-3aebd7c96934",
+                "underlay_address": "fd00:1122:3344:103::4",
                 "zone_type": {
                   "type": "nexus",
                   "external_dns_servers": [
                     "1.1.1.1",
                     "9.9.9.9"
                   ],
-                  "external_ip": "172.20.28.2",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:101::5]:12221",
+                  "external_ip": "198.51.100.22",
+                  "external_tls": false,
+                  "internal_address": "[fd00:1122:3344:103::4]:12221",
                   "nic": {
-                    "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
-                    "ip": "172.30.2.5",
+                    "id": "8847a09c-cabc-429b-bf46-66d5e2e1140c",
                     "kind": {
                       "type": "service",
-                      "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
+                      "id": "0db88baf-911d-47bb-af71-3aebd7c96934"
                     },
-                    "mac": "A8:40:25:FF:E1:17",
-                    "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-                    "primary": true,
-                    "slot": 0,
+                    "name": "nexus-0db88baf-911d-47bb-af71-3aebd7c96934",
+                    "ip": "172.30.2.5",
+                    "mac": "A8:40:25:FF:CC:DA",
                     "subnet": "172.30.2.0/24",
-                    "vni": 100
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
-                "underlay_address": "fd00:1122:3344:101::10",
+                "id": "213d7dd3-95a4-45a7-99ca-e4899991dab4",
+                "underlay_address": "fd00:1122:3344:103::6",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::10]:32345",
+                  "address": "[fd00:1122:3344:103::6]:32345",
                   "dataset": {
-                    "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
-                "underlay_address": "fd00:1122:3344:101::f",
+                "id": "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
+                "underlay_address": "fd00:1122:3344:3::1",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::f]:32345",
+                  "type": "internal_dns",
                   "dataset": {
-                    "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
-                  }
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
+                  },
+                  "dns_address": "[fd00:1122:3344:3::1]:53",
+                  "gz_address": "fd00:1122:3344:3::2",
+                  "gz_address_index": 2,
+                  "http_address": "[fd00:1122:3344:3::1]:5353"
                 }
               },
               {
-                "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
-                "underlay_address": "fd00:1122:3344:101::9",
+                "id": "40c47ead-3be8-4724-a8ef-b66233fc5c65",
+                "underlay_address": "fd00:1122:3344:103::7",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::9]:32345",
+                  "address": "[fd00:1122:3344:103::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                    "pool_name": "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
-                "underlay_address": "fd00:1122:3344:101::8",
+                "id": "425d9d2f-72ea-4f53-80b6-04e002771cdb",
+                "underlay_address": "fd00:1122:3344:103::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                  }
-                }
-              },
-              {
-                "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
-                "underlay_address": "fd00:1122:3344:101::4",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::4]:32221",
+                  "address": "[fd00:1122:3344:103::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                    "pool_name": "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
-                "underlay_address": "fd00:1122:3344:101::d",
+                "id": "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
+                "underlay_address": "fd00:1122:3344:103::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::d]:32345",
+                  "address": "[fd00:1122:3344:103::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
+                    "pool_name": "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b"
                   }
                 }
               },
               {
-                "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
-                "underlay_address": "fd00:1122:3344:101::a",
+                "id": "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
+                "underlay_address": "fd00:1122:3344:103::5",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::a]:32345",
+                  "type": "clickhouse",
+                  "address": "[fd00:1122:3344:103::5]:8123",
                   "dataset": {
-                    "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
-                "underlay_address": "fd00:1122:3344:101::6",
-                "zone_type": {
-                  "type": "oximeter",
-                  "address": "[fd00:1122:3344:101::6]:12223"
-                }
-              },
-              {
-                "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
-                "underlay_address": "fd00:1122:3344:101::11",
+                "id": "7611a549-effc-41b6-b6cb-c03e0c70a31f",
+                "underlay_address": "fd00:1122:3344:103::b",
                 "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:101::11]:123",
+                  "type": "internal_ntp",
+                  "address": "[fd00:1122:3344:103::b]:123",
                   "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
+                    "fd00:1122:3344:1::1",
+                    "fd00:1122:3344:2::1",
+                    "fd00:1122:3344:3::1"
                   ],
-                  "nic": {
-                    "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
-                    "ip": "172.30.3.5",
-                    "kind": {
-                      "type": "service",
-                      "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
-                    },
-                    "mac": "A8:40:25:FF:E7:A8",
-                    "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100
-                  },
                   "ntp_servers": [
-                    "ntp.eng.oxide.computer"
-                  ],
-                  "snat_cfg": {
-                    "ip": "172.20.28.5",
-                    "first_port": 0,
-                    "last_port": 16383
-                  }
+                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
+                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
+                  ]
                 }
               },
               {
-                "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
-                "underlay_address": "fd00:1122:3344:101::e",
+                "id": "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
+                "underlay_address": "fd00:1122:3344:103::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::e]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:103::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
               },
               {
-                "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
-                "underlay_address": "fd00:1122:3344:101::b",
+                "id": "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
+                "underlay_address": "fd00:1122:3344:103::a",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:101::b]:32345",
+                  "address": "[fd00:1122:3344:103::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
+                    "pool_name": "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
                   }
                 }
-              },
-              {
-                "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
-                "underlay_address": "fd00:1122:3344:101::7",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:101::7]:17000"
-                }
               }
             ]
           }
         },
-        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
-          "time_collected": "2024-03-01T19:18:12.036797Z",
-          "source": "http://[fd00:1122:3344:121::1]:12345",
-          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
+        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
+          "time_collected": "2024-03-12T19:09:37.049002Z",
+          "source": "http://[fd00:1122:3344:104::1]:12345",
+          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
           "zones": {
-            "generation": 3,
+            "generation": 5,
             "zones": [
               {
-                "id": "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
-                "underlay_address": "fd00:1122:3344:121::28",
+                "id": "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
+                "underlay_address": "fd00:1122:3344:104::b",
+                "zone_type": {
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:104::b]:32345",
+                  "dataset": {
+                    "pool_name": "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
+                  }
+                }
+              },
+              {
+                "id": "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
+                "underlay_address": "fd00:1122:3344:104::4",
+                "zone_type": {
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:104::4]:32221",
+                  "dataset": {
+                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
+                  }
+                }
+              },
+              {
+                "id": "57c712a6-6429-44b7-a9b2-9957cb9054d4",
+                "underlay_address": "fd00:1122:3344:104::a",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::28]:32345",
+                  "address": "[fd00:1122:3344:104::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3"
+                    "pool_name": "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "12431d3e-1516-4822-98b3-3711ff912e69",
-                "underlay_address": "fd00:1122:3344:121::22",
+                "id": "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
+                "underlay_address": "fd00:1122:3344:104::7",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::22]:32345",
+                  "address": "[fd00:1122:3344:104::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_0622b2cf-297e-4356-b94c-d74309f443ba"
+                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
-                "underlay_address": "fd00:1122:3344:121::21",
+                "id": "73916098-788f-48d8-b745-e13fdcfb771b",
+                "underlay_address": "fd00:1122:3344:104::c",
                 "zone_type": {
                   "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:121::21]:123",
+                  "address": "[fd00:1122:3344:104::c]:123",
                   "dns_servers": [
                     "fd00:1122:3344:1::1",
                     "fd00:1122:3344:2::1",
                     "fd00:1122:3344:3::1"
                   ],
                   "ntp_servers": [
-                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
+                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
+                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
                   ]
                 }
               },
               {
-                "id": "56414d76-9b8c-4e15-acf1-cd269c2d8638",
-                "underlay_address": "fd00:1122:3344:121::25",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::25]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_574728ca-1559-48f3-a46c-1e608966e6a0"
-                  }
-                }
-              },
-              {
-                "id": "891cb9da-301d-40cc-b93d-5fcb350fa968",
-                "underlay_address": "fd00:1122:3344:121::29",
+                "id": "9bb513e3-7505-416c-aa70-b26f1152bbdc",
+                "underlay_address": "fd00:1122:3344:104::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::29]:32345",
+                  "address": "[fd00:1122:3344:104::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
+                    "pool_name": "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b"
                   }
                 }
               },
               {
-                "id": "d9014907-24c0-4c33-a475-db15942f4f8f",
-                "underlay_address": "fd00:1122:3344:121::23",
+                "id": "9c14c879-137b-4f5b-8068-1292bffc83ed",
+                "underlay_address": "fd00:1122:3344:104::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::23]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:104::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37"
+                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "e950de25-7306-4cf7-a6f4-6db11adaa310",
-                "underlay_address": "fd00:1122:3344:121::27",
+                "id": "9cf90589-5da1-4b42-b09b-d784e4134637",
+                "underlay_address": "fd00:1122:3344:104::5",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::27]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c"
+                  "type": "nexus",
+                  "external_dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "external_ip": "198.51.100.23",
+                  "external_tls": false,
+                  "internal_address": "[fd00:1122:3344:104::5]:12221",
+                  "nic": {
+                    "id": "fa76e7e1-dee6-4574-b71f-e07ea1cffd08",
+                    "kind": {
+                      "type": "service",
+                      "id": "9cf90589-5da1-4b42-b09b-d784e4134637"
+                    },
+                    "name": "nexus-9cf90589-5da1-4b42-b09b-d784e4134637",
+                    "ip": "172.30.2.6",
+                    "mac": "A8:40:25:FF:C4:6F",
+                    "subnet": "172.30.2.0/24",
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
-                "underlay_address": "fd00:1122:3344:121::26",
+                "id": "d48b2dfa-c749-47dd-98b6-3f478cd63934",
+                "underlay_address": "fd00:1122:3344:104::6",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::26]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945"
-                  }
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:104::6]:17000"
                 }
               },
               {
-                "id": "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00",
-                "underlay_address": "fd00:1122:3344:121::24",
+                "id": "fd003bee-818a-4c95-aae5-e378ac601522",
+                "underlay_address": "fd00:1122:3344:104::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:121::24]:32345",
+                  "address": "[fd00:1122:3344:104::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1"
+                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               }
             ]
           }
         },
-        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
-          "time_collected": "2024-03-01T19:18:12.072810Z",
+        "77851655-10a5-4e80-88b0-a001473cab7e": {
+          "time_collected": "2024-03-12T19:09:37.088155Z",
           "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
+          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
           "zones": {
             "generation": 5,
             "zones": [
               {
-                "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
-                "underlay_address": "fd00:1122:3344:102::11",
+                "id": "084cac6e-44b5-4333-a2ff-b0993b534898",
+                "underlay_address": "fd00:1122:3344:102::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::11]:32345",
+                  "address": "[fd00:1122:3344:102::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                    "pool_name": "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
-                "underlay_address": "fd00:1122:3344:102::e",
+                "id": "11c527c5-0598-428e-9270-be5b74d3461b",
+                "underlay_address": "fd00:1122:3344:102::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::e]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:102::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "479811e7-6409-45d4-b294-5333534e47ed",
-                "underlay_address": "fd00:1122:3344:102::9",
+                "id": "20be9299-495d-428d-b315-694c2c4b64af",
+                "underlay_address": "fd00:1122:3344:102::6",
+                "zone_type": {
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:102::6]:17000"
+                }
+              },
+              {
+                "id": "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
+                "underlay_address": "fd00:1122:3344:102::b",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::9]:32345",
+                  "address": "[fd00:1122:3344:102::b]:32345",
                   "dataset": {
-                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                    "pool_name": "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
+                "id": "8d452bdd-e8b8-4108-a690-46fc59197db0",
                 "underlay_address": "fd00:1122:3344:102::a",
                 "zone_type": {
                   "type": "crucible",
                   "address": "[fd00:1122:3344:102::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                    "pool_name": "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
-                "underlay_address": "fd00:1122:3344:2::1",
+                "id": "953019c0-8e85-4417-b231-d42820470125",
+                "underlay_address": "fd00:1122:3344:102::5",
                 "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                  },
-                  "dns_address": "[fd00:1122:3344:2::1]:53",
-                  "gz_address": "fd00:1122:3344:2::2",
-                  "gz_address_index": 1,
-                  "http_address": "[fd00:1122:3344:2::1]:5353"
+                  "type": "oximeter",
+                  "address": "[fd00:1122:3344:102::5]:12223"
                 }
               },
               {
-                "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
+                "id": "99c635b9-8e97-414a-8316-8261d2d767d0",
                 "underlay_address": "fd00:1122:3344:102::7",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:102::7]:17000"
-                }
-              },
-              {
-                "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
-                "underlay_address": "fd00:1122:3344:102::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::8]:32345",
+                  "address": "[fd00:1122:3344:102::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "658076ed-fb43-49e7-a813-df3d23497848",
-                "underlay_address": "fd00:1122:3344:102::10",
+                "id": "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
+                "underlay_address": "fd00:1122:3344:2::1",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::10]:32345",
+                  "type": "internal_dns",
                   "dataset": {
-                    "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
-                  }
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
+                  },
+                  "dns_address": "[fd00:1122:3344:2::1]:53",
+                  "gz_address": "fd00:1122:3344:2::2",
+                  "gz_address_index": 1,
+                  "http_address": "[fd00:1122:3344:2::1]:5353"
                 }
               },
               {
-                "id": "6edb836e-b100-4080-bb03-738d6441d571",
+                "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
                 "underlay_address": "fd00:1122:3344:102::c",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::c]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
-                  }
-                }
-              },
-              {
-                "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
-                "underlay_address": "fd00:1122:3344:102::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:102::c]:123",
+                  "dns_servers": [
                     "1.1.1.1",
                     "9.9.9.9"
                   ],
-                  "external_ip": "172.20.28.3",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:102::5]:12221",
                   "nic": {
-                    "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
-                    "ip": "172.30.2.6",
+                    "id": "659f33f7-e7bd-4875-9140-bf8a8b2d964a",
                     "kind": {
                       "type": "service",
-                      "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
+                      "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded"
                     },
-                    "mac": "A8:40:25:FF:B2:AF",
-                    "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
+                    "name": "ntp-d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
+                    "ip": "172.30.3.6",
+                    "mac": "A8:40:25:FF:B1:99",
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100,
                     "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100
+                    "slot": 0
+                  },
+                  "ntp_servers": [
+                    "time.cloudflare.com"
+                  ],
+                  "snat_cfg": {
+                    "ip": "198.51.100.26",
+                    "first_port": 16384,
+                    "last_port": 32767
                   }
                 }
               },
               {
-                "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
-                "underlay_address": "fd00:1122:3344:102::d",
+                "id": "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
+                "underlay_address": "fd00:1122:3344:102::9",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:102::d]:32345",
+                  "address": "[fd00:1122:3344:102::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                    "pool_name": "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b"
                   }
                 }
               },
               {
-                "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
+                "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
                 "underlay_address": "fd00:1122:3344:102::4",
                 "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::4]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
-                  }
-                }
-              },
-              {
-                "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
-                "underlay_address": "fd00:1122:3344:102::6",
-                "zone_type": {
-                  "type": "clickhouse",
-                  "address": "[fd00:1122:3344:102::6]:8123",
-                  "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                  }
-                }
-              },
-              {
-                "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
-                "underlay_address": "fd00:1122:3344:102::f",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::f]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
-                  }
-                }
-              },
-              {
-                "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
-                "underlay_address": "fd00:1122:3344:102::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::3]:32221",
+                  "type": "external_dns",
                   "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                  }
-                }
-              },
-              {
-                "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-                "underlay_address": "fd00:1122:3344:102::12",
-                "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:102::12]:123",
-                  "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
+                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
+                  },
+                  "dns_address": "198.51.100.21:53",
+                  "http_address": "[fd00:1122:3344:102::4]:5353",
                   "nic": {
-                    "id": "9f03c82c-3192-498d-978d-a175049ac389",
-                    "ip": "172.30.3.6",
+                    "id": "d4dd2f7a-8a3b-405e-8444-62f99a754a8a",
                     "kind": {
                       "type": "service",
-                      "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
+                      "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802"
                     },
-                    "mac": "A8:40:25:FF:E4:11",
-                    "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
+                    "name": "external-dns-f0ed9838-6f1b-4646-a478-9f74f2e6e802",
+                    "ip": "172.30.1.6",
+                    "mac": "A8:40:25:FF:F4:5C",
+                    "subnet": "172.30.1.0/24",
+                    "vni": 100,
                     "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100
-                  },
-                  "ntp_servers": [
-                    "ntp.eng.oxide.computer"
-                  ],
-                  "snat_cfg": {
-                    "ip": "172.20.28.6",
-                    "first_port": 16384,
-                    "last_port": 32767
-                  }
-                }
-              },
-              {
-                "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
-                "underlay_address": "fd00:1122:3344:102::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
+                    "slot": 0
                   }
                 }
               }
             ]
           }
         },
-        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
-          "time_collected": "2024-03-01T19:18:12.108386Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
+        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
+          "time_collected": "2024-03-12T19:09:37.115884Z",
+          "source": "http://[fd00:1122:3344:101::1]:12345",
+          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
           "zones": {
             "generation": 5,
             "zones": [
               {
-                "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
-                "underlay_address": "fd00:1122:3344:103::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
-                  }
-                }
-              },
-              {
-                "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
-                "underlay_address": "fd00:1122:3344:103::10",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::10]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
-                  }
-                }
-              },
-              {
-                "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
-                "underlay_address": "fd00:1122:3344:3::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  },
-                  "dns_address": "[fd00:1122:3344:3::1]:53",
-                  "gz_address": "fd00:1122:3344:3::2",
-                  "gz_address_index": 2,
-                  "http_address": "[fd00:1122:3344:3::1]:5353"
-                }
-              },
-              {
-                "id": "4b74c418-0a08-4489-b652-e97267c1c220",
-                "underlay_address": "fd00:1122:3344:103::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  }
-                }
-              },
-              {
-                "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
-                "underlay_address": "fd00:1122:3344:103::5",
+                "id": "0d459323-e414-4f32-9944-76c7331da622",
+                "underlay_address": "fd00:1122:3344:101::5",
                 "zone_type": {
                   "type": "nexus",
                   "external_dns_servers": [
                     "1.1.1.1",
                     "9.9.9.9"
                   ],
-                  "external_ip": "172.20.28.4",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:103::5]:12221",
+                  "external_ip": "198.51.100.24",
+                  "external_tls": false,
+                  "internal_address": "[fd00:1122:3344:101::5]:12221",
                   "nic": {
-                    "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
-                    "ip": "172.30.2.7",
+                    "id": "514902b8-7a1a-42a3-95a6-d00861833b5c",
                     "kind": {
                       "type": "service",
-                      "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
+                      "id": "0d459323-e414-4f32-9944-76c7331da622"
                     },
-                    "mac": "A8:40:25:FF:96:05",
-                    "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
-                    "primary": true,
-                    "slot": 0,
+                    "name": "nexus-0d459323-e414-4f32-9944-76c7331da622",
+                    "ip": "172.30.2.7",
+                    "mac": "A8:40:25:FF:DE:AC",
                     "subnet": "172.30.2.0/24",
-                    "vni": 100
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
-                "underlay_address": "fd00:1122:3344:103::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:103::6]:17000"
-                }
-              },
-              {
-                "id": "921958c0-7761-4294-945d-2a078327bf2c",
-                "underlay_address": "fd00:1122:3344:103::8",
+                "id": "2902972b-3ae4-489d-b937-6480352e214a",
+                "underlay_address": "fd00:1122:3344:101::8",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::8]:32345",
+                  "address": "[fd00:1122:3344:101::8]:32345",
                   "dataset": {
-                    "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
+                    "pool_name": "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "9df22034-59f6-431f-82e7-258fe983879b",
-                "underlay_address": "fd00:1122:3344:103::4",
+                "id": "2ee832de-af75-47ac-abac-f91304857ca4",
+                "underlay_address": "fd00:1122:3344:101::4",
                 "zone_type": {
                   "type": "external_dns",
                   "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   },
-                  "dns_address": "172.20.28.1:53",
-                  "http_address": "[fd00:1122:3344:103::4]:5353",
+                  "dns_address": "198.51.100.20:53",
+                  "http_address": "[fd00:1122:3344:101::4]:5353",
                   "nic": {
-                    "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
-                    "ip": "172.30.1.5",
+                    "id": "fadff4ab-62d7-466a-9203-04080e4078ed",
                     "kind": {
                       "type": "service",
-                      "id": "9df22034-59f6-431f-82e7-258fe983879b"
+                      "id": "2ee832de-af75-47ac-abac-f91304857ca4"
                     },
-                    "mac": "A8:40:25:FF:90:12",
-                    "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
-                    "primary": true,
-                    "slot": 0,
+                    "name": "external-dns-2ee832de-af75-47ac-abac-f91304857ca4",
+                    "ip": "172.30.1.5",
+                    "mac": "A8:40:25:FF:B4:93",
                     "subnet": "172.30.1.0/24",
-                    "vni": 100
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
                   }
                 }
               },
               {
-                "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
-                "underlay_address": "fd00:1122:3344:103::11",
+                "id": "30050a7e-7fc3-4965-b07b-1635212393eb",
+                "underlay_address": "fd00:1122:3344:101::6",
                 "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:103::11]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
-                  ]
+                  "type": "crucible_pantry",
+                  "address": "[fd00:1122:3344:101::6]:17000"
                 }
               },
               {
-                "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
-                "underlay_address": "fd00:1122:3344:103::9",
+                "id": "8494befe-6df6-42d1-9baa-599ba4228430",
+                "underlay_address": "fd00:1122:3344:101::c",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
+                  "type": "boundary_ntp",
+                  "address": "[fd00:1122:3344:101::c]:123",
+                  "dns_servers": [
+                    "1.1.1.1",
+                    "9.9.9.9"
+                  ],
+                  "nic": {
+                    "id": "23041748-e3a6-4846-930d-5a348c316e91",
+                    "kind": {
+                      "type": "service",
+                      "id": "8494befe-6df6-42d1-9baa-599ba4228430"
+                    },
+                    "name": "ntp-8494befe-6df6-42d1-9baa-599ba4228430",
+                    "ip": "172.30.3.5",
+                    "mac": "A8:40:25:FF:D1:C4",
+                    "subnet": "172.30.3.0/24",
+                    "vni": 100,
+                    "primary": true,
+                    "slot": 0
+                  },
+                  "ntp_servers": [
+                    "time.cloudflare.com"
+                  ],
+                  "snat_cfg": {
+                    "ip": "198.51.100.25",
+                    "first_port": 0,
+                    "last_port": 16383
                   }
                 }
               },
               {
-                "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
-                "underlay_address": "fd00:1122:3344:103::d",
+                "id": "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
+                "underlay_address": "fd00:1122:3344:101::7",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::d]:32345",
+                  "address": "[fd00:1122:3344:101::7]:32345",
                   "dataset": {
-                    "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
-                "underlay_address": "fd00:1122:3344:103::b",
+                "id": "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
+                "underlay_address": "fd00:1122:3344:101::3",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::b]:32345",
+                  "type": "cockroach_db",
+                  "address": "[fd00:1122:3344:101::3]:32221",
                   "dataset": {
-                    "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
-                "underlay_address": "fd00:1122:3344:103::e",
+                "id": "e668e3f5-5457-4689-97da-b2af537787ac",
+                "underlay_address": "fd00:1122:3344:101::b",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::e]:32345",
+                  "address": "[fd00:1122:3344:101::b]:32345",
                   "dataset": {
-                    "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
+                    "pool_name": "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               },
               {
-                "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
-                "underlay_address": "fd00:1122:3344:103::3",
+                "id": "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
+                "underlay_address": "fd00:1122:3344:101::9",
                 "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:103::3]:32221",
+                  "type": "crucible",
+                  "address": "[fd00:1122:3344:101::9]:32345",
                   "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                    "pool_name": "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b"
                   }
                 }
               },
               {
-                "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
-                "underlay_address": "fd00:1122:3344:103::f",
+                "id": "f8c79ba4-194a-4cfe-8090-5315e297b780",
+                "underlay_address": "fd00:1122:3344:1::1",
                 "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::f]:32345",
+                  "type": "internal_dns",
                   "dataset": {
-                    "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
-                  }
+                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
+                  },
+                  "dns_address": "[fd00:1122:3344:1::1]:53",
+                  "gz_address": "fd00:1122:3344:1::2",
+                  "gz_address_index": 0,
+                  "http_address": "[fd00:1122:3344:1::1]:5353"
                 }
               },
               {
-                "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
-                "underlay_address": "fd00:1122:3344:103::c",
+                "id": "fa5e0176-b4a8-4598-aa31-40fd562d246c",
+                "underlay_address": "fd00:1122:3344:101::a",
                 "zone_type": {
                   "type": "crucible",
-                  "address": "[fd00:1122:3344:103::c]:32345",
+                  "address": "[fd00:1122:3344:101::a]:32345",
                   "dataset": {
-                    "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                    "pool_name": "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03"
                   }
                 }
               }
@@ -7391,3364 +7302,454 @@
           }
         }
       }
-    },
+    }
+  ],
+  "blueprints": [
     {
-      "id": "4e9fbfc2-eb1d-4d45-bdb9-64d965a490c5",
-      "errors": [
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f6c35180-8831-4718-995a-a08427e3f7d0\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"f6c35180-8831-4718-995a-a08427e3f7d0\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2206bf05-edce-4002-bc9c-72f93cfa45b1\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"2206bf05-edce-4002-bc9c-72f93cfa45b1\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"081feef1-085b-4f9a-9c0a-9c07a7a67ece\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"081feef1-085b-4f9a-9c0a-9c07a7a67ece\" }",
-        "MGS \"http://[fd00:1122:3344:101::2]:12225\": SP SpIdentifier { slot: 0, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e04172a0-55e3-475b-a070-a383d22f3bb0\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:11 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 0 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"e04172a0-55e3-475b-a070-a383d22f3bb0\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page Cmpa: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c8feb285-1846-467e-acac-5fbfb5e401cf\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:12 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"c8feb285-1846-467e-acac-5fbfb5e401cf\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaActive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"74c364c5-4838-43e3-af27-16c1f88e771e\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:12 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"74c364c5-4838-43e3-af27-16c1f88e771e\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaInactive: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e72fd27c-41fd-4597-a623-0b90dde3751f\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:12 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"e72fd27c-41fd-4597-a623-0b90dde3751f\" }",
-        "MGS \"http://[fd00:1122:3344:103::2]:12225\": SP SpIdentifier { slot: 1, type_: Switch }: rot page CfpaScratch: Error Response: status: 503 Service Unavailable; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"09377390-54dc-4481-a94c-fdc884b3cf03\", \"content-length\": \"281\", \"date\": \"Fri, 01 Mar 2024 19:18:12 GMT\"}; value: Error { error_code: Some(\"SpCommunicationFailed\"), message: \"error communicating with SP SpIdentifier { typ: Switch, slot: 1 }: Error response from SP: sprot: failed to deserialize message: sprot: failed to deserialize message\", request_id: \"09377390-54dc-4481-a94c-fdc884b3cf03\" }"
-      ],
-      "time_started": "2024-03-01T19:18:07.756995Z",
-      "time_done": "2024-03-01T19:18:13.548706Z",
-      "collector": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
-      "baseboards": [
-        {
-          "part_number": "913-0000006",
-          "serial_number": "BRM44220005"
-        },
-        {
-          "part_number": "913-0000006",
-          "serial_number": "BRM44220011"
-        },
-        {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220004"
-        },
-        {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220046"
-        },
-        {
-          "part_number": "913-0000019",
-          "serial_number": "BRM42220081"
-        },
-        {
-          "part_number": "913-0000019",
-          "serial_number": "BRM44220001"
-        }
-      ],
-      "cabooses": [
-        {
-          "board": "gimlet-c",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "gimlet-c",
-          "version": "1.0.6"
-        },
-        {
-          "board": "gimlet-c",
-          "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
-          "name": "gimlet-c",
-          "version": "1.0.5"
-        },
-        {
-          "board": "gimlet-c",
-          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-          "name": "gimlet-c",
-          "version": "1.0.8"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "oxide-rot-1",
-          "version": "1.0.5"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
-          "name": "oxide-rot-1",
-          "version": "1.0.4"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
-          "name": "oxide-rot-1",
-          "version": "0.0.0-git"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-          "name": "oxide-rot-1",
-          "version": "1.0.6"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-          "name": "oxide-rot-1",
-          "version": "1.0.0"
-        },
-        {
-          "board": "oxide-rot-1",
-          "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-          "name": "oxide-rot-1-dev",
-          "version": "0.0.0-git"
-        },
-        {
-          "board": "sidecar-b",
-          "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-          "name": "sidecar-b",
-          "version": "1.0.6"
-        },
-        {
-          "board": "sidecar-b",
-          "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
-          "name": "sidecar-b",
-          "version": "0.0.0-git"
-        },
-        {
-          "board": "sidecar-b",
-          "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
-          "name": "sidecar-b",
-          "version": "0.0.0-git"
-        }
-      ],
-      "rot_pages": [
-        {
-          "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
-        },
-        {
-          "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
-        },
-        {
-          "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
-        },
-        {
-          "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
-        },
-        {
-          "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
-        },
-        {
-          "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
-        },
-        {
-          "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        }
-      ],
-      "sps": [
-        [
-          {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220005"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:11.402668Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "sp_type": "switch",
-            "sp_slot": 0,
-            "baseboard_revision": 4,
-            "hubris_archive": "9184be93590e655e",
-            "power_state": "A0"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220011"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:12.176222Z",
-            "source": "http://[fd00:1122:3344:103::2]:12225",
-            "sp_type": "switch",
-            "sp_slot": 1,
-            "baseboard_revision": 4,
-            "hubris_archive": "9184be93590e655e",
-            "power_state": "A0"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:10.088383Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 17,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:10.815174Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 15,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:08.950823Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 14,
-            "baseboard_revision": 6,
-            "hubris_archive": "5c745f4b4fa66f47",
-            "power_state": "A0"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:07.775665Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 16,
-            "baseboard_revision": 6,
-            "hubris_archive": "3c670b44fdeef4af",
-            "power_state": "A0"
-          }
-        ]
-      ],
-      "rots": [
-        [
-          {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220005"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:11.402668Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "b"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "4599e4d7bc9b2164e8c0f0f7ce4bcae2bd35d635c8450854ea9a94ce49a2ca0f",
-            "slot_b_sha3_256_digest": "1e42a0761e5bd1d9f9a5e2c44acc34e7b7e007792977cdb0523d271db2e80c8c"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000006",
-            "serial_number": "BRM44220011"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:12.176222Z",
-            "source": "http://[fd00:1122:3344:103::2]:12225",
-            "active_slot": {
-              "slot": "b"
-            },
-            "persistent_boot_preference": {
-              "slot": "b"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "92e07077972c83af6872ec817fb88d74a36f6956276505e9771302c5bb7ec2a8",
-            "slot_b_sha3_256_digest": "fb25825eaf97f8f2307afe3caebf0b70865499f6d116db3aff382c505b4d8dd7"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:10.088383Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
-            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:10.815174Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "96caca4fb6a789c228cdb69ca4afe300cecd46fb611e1fb12c384fe9597a0d1c",
-            "slot_b_sha3_256_digest": "29215490b1984f4cfcea7d95176b8338466b2fc60123cfc27e895209bef506bb"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:08.950823Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "active_slot": {
-              "slot": "b"
-            },
-            "persistent_boot_preference": {
-              "slot": "b"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
-            "slot_b_sha3_256_digest": "d416d5ba9d0d3bd495f0f5fcd2aecc28cd83b53d1a8f94a28241d85b30761451"
-          }
-        ],
-        [
-          {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
-          },
-          {
-            "time_collected": "2024-03-01T19:18:07.775665Z",
-            "source": "http://[fd00:1122:3344:101::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": "7d081250f8eb830b7dc5067334316f497a184b01788b5614a7a7f88b93a87c20",
-            "slot_b_sha3_256_digest": "4ec328195a23e86a1545d5ee576e73834aae81fb79830a54d0503475875f1760"
-          }
-        ]
-      ],
-      "cabooses_found": {
-        "SpSlot0": [
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:11.404063Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "sidecar-b",
-                "version": "1.0.6"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:12.391683Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
-              "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "sidecar-b",
-                "version": "1.0.6"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.181795Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.816554Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:09.149044Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "gimlet-c",
-                "version": "1.0.8"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:07.778136Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
-              }
-            }
-          ]
-        ],
-        "SpSlot1": [
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:11.405544Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "f04c427514943573272da3524aa4d976d069ac95",
-                "name": "sidecar-b",
-                "version": "0.0.0-git"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:12.425692Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
-              "caboose": {
-                "board": "sidecar-b",
-                "git_commit": "a096c4c17b17772c72dbfb03e0b9c96e59d9a0fe",
-                "name": "sidecar-b",
-                "version": "0.0.0-git"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.229994Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.817966Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:09.402701Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "gimlet-c",
-                "version": "1.0.6"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:07.836136Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "gimlet-c",
-                "git_commit": "620f18f924d180d13a0577bd51d7c48c1d670549",
-                "name": "gimlet-c",
-                "version": "1.0.5"
-              }
-            }
-          ]
-        ],
-        "RotSlotA": [
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:11.612405Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-                "name": "oxide-rot-1-dev",
-                "version": "0.0.0-git"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:12.633926Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "4b2a3469ee63f14e237ed26d741c49e8cc27261f",
-                "name": "oxide-rot-1",
-                "version": "0.0.0-git"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.482137Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:11.072878Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:09.718444Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:08.338242Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
-              }
-            }
-          ]
-        ],
-        "RotSlotB": [
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220005"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:11.819343Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000006",
-              "serial_number": "BRM44220011"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:12.889742Z",
-              "source": "http://[fd00:1122:3344:103::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "ec0b7fe84e51e6d320556380a6f1dbb464ab9647",
-                "name": "oxide-rot-1",
-                "version": "1.0.0"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.734260Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:11.324822Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "02cc4345b5def59e96ef1eb576889922e9468113",
-                "name": "oxide-rot-1",
-                "version": "1.0.5"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:09.971380Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "b398d0778a806acad7b015a3e183ad56a28a620d",
-                "name": "oxide-rot-1",
-                "version": "1.0.6"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:08.798226Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "caboose": {
-                "board": "oxide-rot-1",
-                "git_commit": "08f5a258dd48728cfaae68fcc7fbd56c7359329c",
-                "name": "oxide-rot-1",
-                "version": "1.0.4"
-              }
-            }
-          ]
-        ]
-      },
-      "rot_pages_found": {
-        "Cmpa": [
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.750544Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:11.341290Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:09.987916Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:08.829941Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "oAAAeAAAAAAAAAAAAAAAAP8CAP3/AgD9AAAAAAUAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARWUu1VIp1fpGOb+BW4q2eCEKXyVVUF6Al2HiOrPVdrwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ],
-        "CfpaActive": [
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.766709Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:11.357423Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.004467Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:08.861702Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
-              }
-            }
-          ]
-        ],
-        "CfpaInactive": [
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.782522Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "AAAAADYAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKR50f2Jw5Dyd7j2FPBE4r77CMk4azEk2sBLHfrvOts="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:11.373385Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "AAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAa0EBHkNRkpjbiUrOzKdmz2xKt2ubIbVZ4ALX8/sXkig="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.021096Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:08.893754Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
-              }
-            }
-          ]
-        ],
-        "CfpaScratch": [
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220004"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.798536Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "AAAAADcAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAT9AkBD3h7E4HX3ag0CWYdTvYhAwH5B7l6KL6QI/wPEQ="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220046"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:11.395564Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "AAAAABkAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA6n5IZG1N0E70rdhMMrS3k04zLw+wi+p0BE00p4LJpio="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM42220081"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:10.040934Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "AAAAADIAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUPHEZ9TS6v9VwoGSbVIfZcUfeve3umWF/LfWrI991Fg="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "913-0000019",
-              "serial_number": "BRM44220001"
-            },
-            {
-              "time_collected": "2024-03-01T19:18:08.909759Z",
-              "source": "http://[fd00:1122:3344:101::2]:12225",
-              "page": {
-                "data_base64": "AAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAVQAAAAAAAAD/AgD9/wIA/QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9+zE5FW+go+MrbcURmve9xn7h79n9qXBEVZFgbbZLZE="
-              }
-            }
-          ]
-        ]
-      },
-      "sled_agents": {
-        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
-          "time_collected": "2024-03-01T19:18:13.421323Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM44220001"
-          },
-          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
-        },
-        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
-          "time_collected": "2024-03-01T19:18:13.464287Z",
-          "source": "http://[fd00:1122:3344:121::1]:12345",
-          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220004"
-          },
-          "sled_agent_address": "[fd00:1122:3344:121::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
-        },
-        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
-          "time_collected": "2024-03-01T19:18:13.506888Z",
-          "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220046"
-          },
-          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
-        },
-        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
-          "time_collected": "2024-03-01T19:18:13.546670Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
-          "baseboard_id": {
-            "part_number": "913-0000019",
-            "serial_number": "BRM42220081"
-          },
-          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 128,
-          "usable_physical_ram": 1086608900096,
-          "reservoir_size": 869286281216
-        }
-      },
-      "omicron_zones": {
-        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
-          "time_collected": "2024-03-01T19:18:13.422277Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "1762267a-a0cb-4c9a-88b4-8ce828cbc021",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "0706860a-3801-42a7-8000-fff741f7665b",
-                "underlay_address": "fd00:1122:3344:101::c",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::c]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
-                  }
-                }
-              },
-              {
-                "id": "34a279a2-a025-41ff-a392-1addac400f38",
-                "underlay_address": "fd00:1122:3344:101::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                  }
-                }
-              },
-              {
-                "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
-                "underlay_address": "fd00:1122:3344:1::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                  },
-                  "dns_address": "[fd00:1122:3344:1::1]:53",
-                  "gz_address": "fd00:1122:3344:1::2",
-                  "gz_address_index": 0,
-                  "http_address": "[fd00:1122:3344:1::1]:5353"
-                }
-              },
-              {
-                "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-                "underlay_address": "fd00:1122:3344:101::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "172.20.28.2",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:101::5]:12221",
-                  "nic": {
-                    "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
-                    "ip": "172.30.2.5",
-                    "kind": {
-                      "type": "service",
-                      "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
-                    },
-                    "mac": "A8:40:25:FF:E1:17",
-                    "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100
-                  }
-                }
-              },
-              {
-                "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
-                "underlay_address": "fd00:1122:3344:101::10",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::10]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
-                  }
-                }
-              },
-              {
-                "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
-                "underlay_address": "fd00:1122:3344:101::f",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::f]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
-                  }
-                }
-              },
-              {
-                "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
-                "underlay_address": "fd00:1122:3344:101::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
-                  }
-                }
-              },
-              {
-                "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
-                "underlay_address": "fd00:1122:3344:101::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                  }
-                }
-              },
-              {
-                "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
-                "underlay_address": "fd00:1122:3344:101::4",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::4]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
-                  }
-                }
-              },
-              {
-                "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
-                "underlay_address": "fd00:1122:3344:101::d",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::d]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
-                  }
-                }
-              },
-              {
-                "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
-                "underlay_address": "fd00:1122:3344:101::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
-                  }
-                }
-              },
-              {
-                "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
-                "underlay_address": "fd00:1122:3344:101::6",
-                "zone_type": {
-                  "type": "oximeter",
-                  "address": "[fd00:1122:3344:101::6]:12223"
-                }
-              },
-              {
-                "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
-                "underlay_address": "fd00:1122:3344:101::11",
-                "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:101::11]:123",
-                  "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "nic": {
-                    "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
-                    "ip": "172.30.3.5",
-                    "kind": {
-                      "type": "service",
-                      "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
-                    },
-                    "mac": "A8:40:25:FF:E7:A8",
-                    "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100
-                  },
-                  "ntp_servers": [
-                    "ntp.eng.oxide.computer"
-                  ],
-                  "snat_cfg": {
-                    "ip": "172.20.28.5",
-                    "first_port": 0,
-                    "last_port": 16383
-                  }
-                }
-              },
-              {
-                "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
-                "underlay_address": "fd00:1122:3344:101::e",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::e]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
-                  }
-                }
-              },
-              {
-                "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
-                "underlay_address": "fd00:1122:3344:101::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
-                  }
-                }
-              },
-              {
-                "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
-                "underlay_address": "fd00:1122:3344:101::7",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:101::7]:17000"
-                }
-              }
-            ]
-          }
-        },
-        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
-          "time_collected": "2024-03-01T19:18:13.464998Z",
-          "source": "http://[fd00:1122:3344:121::1]:12345",
-          "sled_id": "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f",
-          "zones": {
-            "generation": 3,
-            "zones": [
-              {
-                "id": "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
-                "underlay_address": "fd00:1122:3344:121::28",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::28]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3"
-                  }
-                }
-              },
-              {
-                "id": "12431d3e-1516-4822-98b3-3711ff912e69",
-                "underlay_address": "fd00:1122:3344:121::22",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::22]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_0622b2cf-297e-4356-b94c-d74309f443ba"
-                  }
-                }
-              },
-              {
-                "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
-                "underlay_address": "fd00:1122:3344:121::21",
-                "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:121::21]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
-                  ]
-                }
-              },
-              {
-                "id": "56414d76-9b8c-4e15-acf1-cd269c2d8638",
-                "underlay_address": "fd00:1122:3344:121::25",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::25]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_574728ca-1559-48f3-a46c-1e608966e6a0"
-                  }
-                }
-              },
-              {
-                "id": "891cb9da-301d-40cc-b93d-5fcb350fa968",
-                "underlay_address": "fd00:1122:3344:121::29",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::29]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
-                  }
-                }
-              },
-              {
-                "id": "d9014907-24c0-4c33-a475-db15942f4f8f",
-                "underlay_address": "fd00:1122:3344:121::23",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::23]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37"
-                  }
-                }
-              },
-              {
-                "id": "e950de25-7306-4cf7-a6f4-6db11adaa310",
-                "underlay_address": "fd00:1122:3344:121::27",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::27]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c"
-                  }
-                }
-              },
-              {
-                "id": "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
-                "underlay_address": "fd00:1122:3344:121::26",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::26]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945"
-                  }
-                }
-              },
-              {
-                "id": "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00",
-                "underlay_address": "fd00:1122:3344:121::24",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:121::24]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1"
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
-          "time_collected": "2024-03-01T19:18:13.507881Z",
-          "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "a6634f64-f6fb-426c-b00b-9b30aed9f53a",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
-                "underlay_address": "fd00:1122:3344:102::11",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::11]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
-                  }
-                }
-              },
-              {
-                "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
-                "underlay_address": "fd00:1122:3344:102::e",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::e]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
-                  }
-                }
-              },
-              {
-                "id": "479811e7-6409-45d4-b294-5333534e47ed",
-                "underlay_address": "fd00:1122:3344:102::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
-                  }
-                }
-              },
-              {
-                "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
-                "underlay_address": "fd00:1122:3344:102::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
-                  }
-                }
-              },
-              {
-                "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
-                "underlay_address": "fd00:1122:3344:2::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                  },
-                  "dns_address": "[fd00:1122:3344:2::1]:53",
-                  "gz_address": "fd00:1122:3344:2::2",
-                  "gz_address_index": 1,
-                  "http_address": "[fd00:1122:3344:2::1]:5353"
-                }
-              },
-              {
-                "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
-                "underlay_address": "fd00:1122:3344:102::7",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:102::7]:17000"
-                }
-              },
-              {
-                "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
-                "underlay_address": "fd00:1122:3344:102::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                  }
-                }
-              },
-              {
-                "id": "658076ed-fb43-49e7-a813-df3d23497848",
-                "underlay_address": "fd00:1122:3344:102::10",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::10]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
-                  }
-                }
-              },
-              {
-                "id": "6edb836e-b100-4080-bb03-738d6441d571",
-                "underlay_address": "fd00:1122:3344:102::c",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::c]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
-                  }
-                }
-              },
-              {
-                "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
-                "underlay_address": "fd00:1122:3344:102::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "172.20.28.3",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:102::5]:12221",
-                  "nic": {
-                    "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
-                    "ip": "172.30.2.6",
-                    "kind": {
-                      "type": "service",
-                      "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
-                    },
-                    "mac": "A8:40:25:FF:B2:AF",
-                    "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100
-                  }
-                }
-              },
-              {
-                "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
-                "underlay_address": "fd00:1122:3344:102::d",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::d]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
-                  }
-                }
-              },
-              {
-                "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
-                "underlay_address": "fd00:1122:3344:102::4",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::4]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
-                  }
-                }
-              },
-              {
-                "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
-                "underlay_address": "fd00:1122:3344:102::6",
-                "zone_type": {
-                  "type": "clickhouse",
-                  "address": "[fd00:1122:3344:102::6]:8123",
-                  "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                  }
-                }
-              },
-              {
-                "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
-                "underlay_address": "fd00:1122:3344:102::f",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::f]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
-                  }
-                }
-              },
-              {
-                "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
-                "underlay_address": "fd00:1122:3344:102::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                  }
-                }
-              },
-              {
-                "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-                "underlay_address": "fd00:1122:3344:102::12",
-                "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:102::12]:123",
-                  "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "nic": {
-                    "id": "9f03c82c-3192-498d-978d-a175049ac389",
-                    "ip": "172.30.3.6",
-                    "kind": {
-                      "type": "service",
-                      "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
-                    },
-                    "mac": "A8:40:25:FF:E4:11",
-                    "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100
-                  },
-                  "ntp_servers": [
-                    "ntp.eng.oxide.computer"
-                  ],
-                  "snat_cfg": {
-                    "ip": "172.20.28.6",
-                    "first_port": 16384,
-                    "last_port": 32767
-                  }
-                }
-              },
-              {
-                "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
-                "underlay_address": "fd00:1122:3344:102::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
-          "time_collected": "2024-03-01T19:18:13.547769Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
-                "underlay_address": "fd00:1122:3344:103::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
-                  }
-                }
-              },
-              {
-                "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
-                "underlay_address": "fd00:1122:3344:103::10",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::10]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
-                  }
-                }
-              },
-              {
-                "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
-                "underlay_address": "fd00:1122:3344:3::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  },
-                  "dns_address": "[fd00:1122:3344:3::1]:53",
-                  "gz_address": "fd00:1122:3344:3::2",
-                  "gz_address_index": 2,
-                  "http_address": "[fd00:1122:3344:3::1]:5353"
-                }
-              },
-              {
-                "id": "4b74c418-0a08-4489-b652-e97267c1c220",
-                "underlay_address": "fd00:1122:3344:103::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  }
-                }
-              },
-              {
-                "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
-                "underlay_address": "fd00:1122:3344:103::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "172.20.28.4",
-                  "external_tls": true,
-                  "internal_address": "[fd00:1122:3344:103::5]:12221",
-                  "nic": {
-                    "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
-                    "ip": "172.30.2.7",
-                    "kind": {
-                      "type": "service",
-                      "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
-                    },
-                    "mac": "A8:40:25:FF:96:05",
-                    "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100
-                  }
-                }
-              },
-              {
-                "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
-                "underlay_address": "fd00:1122:3344:103::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:103::6]:17000"
-                }
-              },
-              {
-                "id": "921958c0-7761-4294-945d-2a078327bf2c",
-                "underlay_address": "fd00:1122:3344:103::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
-                  }
-                }
-              },
-              {
-                "id": "9df22034-59f6-431f-82e7-258fe983879b",
-                "underlay_address": "fd00:1122:3344:103::4",
-                "zone_type": {
-                  "type": "external_dns",
-                  "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  },
-                  "dns_address": "172.20.28.1:53",
-                  "http_address": "[fd00:1122:3344:103::4]:5353",
-                  "nic": {
-                    "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
-                    "ip": "172.30.1.5",
-                    "kind": {
-                      "type": "service",
-                      "id": "9df22034-59f6-431f-82e7-258fe983879b"
-                    },
-                    "mac": "A8:40:25:FF:90:12",
-                    "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
-                    "primary": true,
-                    "slot": 0,
-                    "subnet": "172.30.1.0/24",
-                    "vni": 100
-                  }
-                }
-              },
-              {
-                "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
-                "underlay_address": "fd00:1122:3344:103::11",
-                "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:103::11]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                    "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
-                  ]
-                }
-              },
-              {
-                "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
-                "underlay_address": "fd00:1122:3344:103::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
-                  }
-                }
-              },
-              {
-                "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
-                "underlay_address": "fd00:1122:3344:103::d",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::d]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
-                  }
-                }
-              },
-              {
-                "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
-                "underlay_address": "fd00:1122:3344:103::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
-                  }
-                }
-              },
-              {
-                "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
-                "underlay_address": "fd00:1122:3344:103::e",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::e]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
-                  }
-                }
-              },
-              {
-                "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
-                "underlay_address": "fd00:1122:3344:103::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:103::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                  }
-                }
-              },
-              {
-                "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
-                "underlay_address": "fd00:1122:3344:103::f",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::f]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
-                  }
-                }
-              },
-              {
-                "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
-                "underlay_address": "fd00:1122:3344:103::c",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::c]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
-                  }
-                }
-              }
-            ]
-          }
-        }
-      }
-    }
-  ],
-  "blueprints": [
-    {
-      "id": "486de160-c8f3-4600-acca-b0c78e33aca4",
-      "omicron_zones": {
-        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "0706860a-3801-42a7-8000-fff741f7665b",
-              "underlay_address": "fd00:1122:3344:101::c",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::c]:32345",
-                "dataset": {
-                  "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
-                }
-              }
-            },
-            {
-              "id": "34a279a2-a025-41ff-a392-1addac400f38",
-              "underlay_address": "fd00:1122:3344:101::3",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:101::3]:32221",
-                "dataset": {
-                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                }
-              }
-            },
-            {
-              "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
-              "underlay_address": "fd00:1122:3344:1::1",
-              "zone_type": {
-                "type": "internal_dns",
-                "dataset": {
-                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                },
-                "dns_address": "[fd00:1122:3344:1::1]:53",
-                "gz_address": "fd00:1122:3344:1::2",
-                "gz_address_index": 0,
-                "http_address": "[fd00:1122:3344:1::1]:5353"
-              }
-            },
-            {
-              "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-              "underlay_address": "fd00:1122:3344:101::5",
-              "zone_type": {
-                "type": "nexus",
-                "external_dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "external_ip": "172.20.28.2",
-                "external_tls": true,
-                "internal_address": "[fd00:1122:3344:101::5]:12221",
-                "nic": {
-                  "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
-                  "ip": "172.30.2.5",
-                  "kind": {
-                    "type": "service",
-                    "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
-                  },
-                  "mac": "A8:40:25:FF:E1:17",
-                  "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-                  "primary": true,
-                  "slot": 0,
-                  "subnet": "172.30.2.0/24",
-                  "vni": 100
-                }
-              }
-            },
-            {
-              "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
-              "underlay_address": "fd00:1122:3344:101::10",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::10]:32345",
-                "dataset": {
-                  "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
-                }
-              }
-            },
-            {
-              "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
-              "underlay_address": "fd00:1122:3344:101::f",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::f]:32345",
-                "dataset": {
-                  "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
-                }
-              }
-            },
-            {
-              "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
-              "underlay_address": "fd00:1122:3344:101::9",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
-                }
-              }
-            },
-            {
-              "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
-              "underlay_address": "fd00:1122:3344:101::8",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                }
-              }
-            },
-            {
-              "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
-              "underlay_address": "fd00:1122:3344:101::4",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:101::4]:32221",
-                "dataset": {
-                  "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
-                }
-              }
-            },
-            {
-              "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
-              "underlay_address": "fd00:1122:3344:101::d",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::d]:32345",
-                "dataset": {
-                  "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
-                }
-              }
-            },
-            {
-              "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
-              "underlay_address": "fd00:1122:3344:101::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
-                }
-              }
-            },
-            {
-              "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
-              "underlay_address": "fd00:1122:3344:101::6",
-              "zone_type": {
-                "type": "oximeter",
-                "address": "[fd00:1122:3344:101::6]:12223"
-              }
-            },
-            {
-              "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
-              "underlay_address": "fd00:1122:3344:101::11",
-              "zone_type": {
-                "type": "boundary_ntp",
-                "address": "[fd00:1122:3344:101::11]:123",
-                "dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "nic": {
-                  "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
-                  "ip": "172.30.3.5",
-                  "kind": {
-                    "type": "service",
-                    "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
-                  },
-                  "mac": "A8:40:25:FF:E7:A8",
-                  "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
-                  "primary": true,
-                  "slot": 0,
-                  "subnet": "172.30.3.0/24",
-                  "vni": 100
-                },
-                "ntp_servers": [
-                  "ntp.eng.oxide.computer"
-                ],
-                "snat_cfg": {
-                  "ip": "172.20.28.5",
-                  "first_port": 0,
-                  "last_port": 16383
-                }
-              }
-            },
-            {
-              "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
-              "underlay_address": "fd00:1122:3344:101::e",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::e]:32345",
-                "dataset": {
-                  "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
-                }
-              }
-            },
-            {
-              "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
-              "underlay_address": "fd00:1122:3344:101::b",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::b]:32345",
-                "dataset": {
-                  "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
-                }
-              }
-            },
-            {
-              "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
-              "underlay_address": "fd00:1122:3344:101::7",
-              "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:101::7]:17000"
-              }
-            }
-          ]
-        },
-        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
-          "generation": 1,
-          "zones": []
-        },
-        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
-              "underlay_address": "fd00:1122:3344:102::11",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::11]:32345",
-                "dataset": {
-                  "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
-                }
-              }
-            },
-            {
-              "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
-              "underlay_address": "fd00:1122:3344:102::e",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::e]:32345",
-                "dataset": {
-                  "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
-                }
-              }
-            },
-            {
-              "id": "479811e7-6409-45d4-b294-5333534e47ed",
-              "underlay_address": "fd00:1122:3344:102::9",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
-                }
-              }
-            },
-            {
-              "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
-              "underlay_address": "fd00:1122:3344:102::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
-                }
-              }
-            },
-            {
-              "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
-              "underlay_address": "fd00:1122:3344:2::1",
-              "zone_type": {
-                "type": "internal_dns",
-                "dataset": {
-                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                },
-                "dns_address": "[fd00:1122:3344:2::1]:53",
-                "gz_address": "fd00:1122:3344:2::2",
-                "gz_address_index": 1,
-                "http_address": "[fd00:1122:3344:2::1]:5353"
-              }
-            },
-            {
-              "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
-              "underlay_address": "fd00:1122:3344:102::7",
-              "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:102::7]:17000"
-              }
-            },
-            {
-              "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
-              "underlay_address": "fd00:1122:3344:102::8",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                }
-              }
-            },
-            {
-              "id": "658076ed-fb43-49e7-a813-df3d23497848",
-              "underlay_address": "fd00:1122:3344:102::10",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::10]:32345",
-                "dataset": {
-                  "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
-                }
-              }
-            },
-            {
-              "id": "6edb836e-b100-4080-bb03-738d6441d571",
-              "underlay_address": "fd00:1122:3344:102::c",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::c]:32345",
-                "dataset": {
-                  "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
-                }
-              }
-            },
-            {
-              "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
-              "underlay_address": "fd00:1122:3344:102::5",
-              "zone_type": {
-                "type": "nexus",
-                "external_dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "external_ip": "172.20.28.3",
-                "external_tls": true,
-                "internal_address": "[fd00:1122:3344:102::5]:12221",
-                "nic": {
-                  "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
-                  "ip": "172.30.2.6",
-                  "kind": {
-                    "type": "service",
-                    "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
-                  },
-                  "mac": "A8:40:25:FF:B2:AF",
-                  "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
-                  "primary": true,
-                  "slot": 0,
-                  "subnet": "172.30.2.0/24",
-                  "vni": 100
-                }
-              }
-            },
-            {
-              "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
-              "underlay_address": "fd00:1122:3344:102::d",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::d]:32345",
-                "dataset": {
-                  "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
-                }
-              }
-            },
-            {
-              "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
-              "underlay_address": "fd00:1122:3344:102::4",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:102::4]:32221",
-                "dataset": {
-                  "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
-                }
-              }
-            },
-            {
-              "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
-              "underlay_address": "fd00:1122:3344:102::6",
-              "zone_type": {
-                "type": "clickhouse",
-                "address": "[fd00:1122:3344:102::6]:8123",
-                "dataset": {
-                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                }
-              }
-            },
-            {
-              "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
-              "underlay_address": "fd00:1122:3344:102::f",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::f]:32345",
-                "dataset": {
-                  "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
-                }
-              }
-            },
-            {
-              "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
-              "underlay_address": "fd00:1122:3344:102::3",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:102::3]:32221",
-                "dataset": {
-                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                }
-              }
-            },
-            {
-              "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-              "underlay_address": "fd00:1122:3344:102::12",
-              "zone_type": {
-                "type": "boundary_ntp",
-                "address": "[fd00:1122:3344:102::12]:123",
-                "dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "nic": {
-                  "id": "9f03c82c-3192-498d-978d-a175049ac389",
-                  "ip": "172.30.3.6",
-                  "kind": {
-                    "type": "service",
-                    "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
-                  },
-                  "mac": "A8:40:25:FF:E4:11",
-                  "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-                  "primary": true,
-                  "slot": 0,
-                  "subnet": "172.30.3.0/24",
-                  "vni": 100
-                },
-                "ntp_servers": [
-                  "ntp.eng.oxide.computer"
-                ],
-                "snat_cfg": {
-                  "ip": "172.20.28.6",
-                  "first_port": 16384,
-                  "last_port": 32767
-                }
-              }
-            },
-            {
-              "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
-              "underlay_address": "fd00:1122:3344:102::b",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::b]:32345",
-                "dataset": {
-                  "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
-                }
-              }
-            }
-          ]
-        },
-        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
-              "underlay_address": "fd00:1122:3344:103::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
-                }
-              }
-            },
-            {
-              "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
-              "underlay_address": "fd00:1122:3344:103::10",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::10]:32345",
-                "dataset": {
-                  "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
-                }
-              }
-            },
-            {
-              "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
-              "underlay_address": "fd00:1122:3344:3::1",
-              "zone_type": {
-                "type": "internal_dns",
-                "dataset": {
-                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                },
-                "dns_address": "[fd00:1122:3344:3::1]:53",
-                "gz_address": "fd00:1122:3344:3::2",
-                "gz_address_index": 2,
-                "http_address": "[fd00:1122:3344:3::1]:5353"
-              }
-            },
-            {
-              "id": "4b74c418-0a08-4489-b652-e97267c1c220",
-              "underlay_address": "fd00:1122:3344:103::7",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::7]:32345",
-                "dataset": {
-                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                }
-              }
-            },
-            {
-              "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
-              "underlay_address": "fd00:1122:3344:103::5",
-              "zone_type": {
-                "type": "nexus",
-                "external_dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "external_ip": "172.20.28.4",
-                "external_tls": true,
-                "internal_address": "[fd00:1122:3344:103::5]:12221",
-                "nic": {
-                  "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
-                  "ip": "172.30.2.7",
-                  "kind": {
-                    "type": "service",
-                    "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
-                  },
-                  "mac": "A8:40:25:FF:96:05",
-                  "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
-                  "primary": true,
-                  "slot": 0,
-                  "subnet": "172.30.2.0/24",
-                  "vni": 100
-                }
-              }
-            },
-            {
-              "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
-              "underlay_address": "fd00:1122:3344:103::6",
-              "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:103::6]:17000"
-              }
-            },
-            {
-              "id": "921958c0-7761-4294-945d-2a078327bf2c",
-              "underlay_address": "fd00:1122:3344:103::8",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
-                }
-              }
-            },
-            {
-              "id": "9df22034-59f6-431f-82e7-258fe983879b",
-              "underlay_address": "fd00:1122:3344:103::4",
-              "zone_type": {
-                "type": "external_dns",
-                "dataset": {
-                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                },
-                "dns_address": "172.20.28.1:53",
-                "http_address": "[fd00:1122:3344:103::4]:5353",
-                "nic": {
-                  "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
-                  "ip": "172.30.1.5",
-                  "kind": {
-                    "type": "service",
-                    "id": "9df22034-59f6-431f-82e7-258fe983879b"
-                  },
-                  "mac": "A8:40:25:FF:90:12",
-                  "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
-                  "primary": true,
-                  "slot": 0,
-                  "subnet": "172.30.1.0/24",
-                  "vni": 100
-                }
-              }
-            },
-            {
-              "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
-              "underlay_address": "fd00:1122:3344:103::11",
-              "zone_type": {
-                "type": "internal_ntp",
-                "address": "[fd00:1122:3344:103::11]:123",
-                "dns_servers": [
-                  "fd00:1122:3344:1::1",
-                  "fd00:1122:3344:2::1",
-                  "fd00:1122:3344:3::1"
-                ],
-                "ntp_servers": [
-                  "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                  "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
-                ]
-              }
-            },
-            {
-              "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
-              "underlay_address": "fd00:1122:3344:103::9",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
-                }
-              }
-            },
-            {
-              "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
-              "underlay_address": "fd00:1122:3344:103::d",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::d]:32345",
-                "dataset": {
-                  "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
-                }
-              }
-            },
-            {
-              "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
-              "underlay_address": "fd00:1122:3344:103::b",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::b]:32345",
-                "dataset": {
-                  "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
-                }
-              }
-            },
-            {
-              "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
-              "underlay_address": "fd00:1122:3344:103::e",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::e]:32345",
-                "dataset": {
-                  "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
-                }
-              }
-            },
-            {
-              "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
-              "underlay_address": "fd00:1122:3344:103::3",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:103::3]:32221",
-                "dataset": {
-                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                }
-              }
-            },
-            {
-              "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
-              "underlay_address": "fd00:1122:3344:103::f",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::f]:32345",
-                "dataset": {
-                  "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
-                }
-              }
-            },
-            {
-              "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
-              "underlay_address": "fd00:1122:3344:103::c",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::c]:32345",
-                "dataset": {
-                  "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
-                }
-              }
-            }
-          ]
-        }
-      },
-      "zones_in_service": [
-        "0706860a-3801-42a7-8000-fff741f7665b",
-        "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
-        "185e6b00-7e18-456c-9dfc-edd0194ac207",
-        "18b99404-d8b2-4463-bc99-638d9b678ab3",
-        "1fc78bc7-feca-4a10-9beb-58b53fda5210",
-        "34a279a2-a025-41ff-a392-1addac400f38",
-        "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
-        "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
-        "479811e7-6409-45d4-b294-5333534e47ed",
-        "4b74c418-0a08-4489-b652-e97267c1c220",
-        "54c947d2-6355-453c-80fc-8f49cc2129ee",
-        "54d1b693-bc4d-4c47-9009-c6f624073801",
-        "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-        "5b743072-9369-45d7-b7c9-dbdf736301f8",
-        "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
-        "60d64b70-89f5-4774-9e41-2c79c4173911",
-        "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
-        "658076ed-fb43-49e7-a813-df3d23497848",
-        "67852e14-ce7f-4e50-a61d-4059d0c057b9",
-        "6ace1a69-d55a-45f0-a127-99c90f1b070c",
-        "6edb836e-b100-4080-bb03-738d6441d571",
-        "7f83d784-9c5c-498f-b559-4176bbafbc4b",
-        "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
-        "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
-        "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
-        "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
-        "86ee7359-aa35-41d9-8a76-54961bc516a1",
-        "921958c0-7761-4294-945d-2a078327bf2c",
-        "9241b9da-8188-4aae-a9dc-62acb453618f",
-        "925ea436-fad4-4fff-8ae0-24edd725a2b0",
-        "9df22034-59f6-431f-82e7-258fe983879b",
-        "a781a349-62b2-41f0-9dd7-c64d290aef82",
-        "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
-        "be272d1e-4d34-4478-94b4-a76f5fcbef36",
-        "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
-        "c67814bf-9972-4bb0-a666-24778ca70a16",
-        "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-        "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
-        "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
-        "d42fc874-8416-4212-b5a0-febba4c6b0ac",
-        "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
-        "d630cca7-e3f7-47a8-aee8-ad0790895abc",
-        "d8c51c79-46ff-44bd-80be-08d6283f4e21",
-        "da27ee6a-9841-46aa-acda-7cce3e04cb60",
-        "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
-        "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
-        "e09525cf-9c87-4d05-8860-1356dd0dee8a",
-        "f33a2375-212d-42e4-b539-6079aeb4e5b7",
-        "f7264a3e-e8e7-4476-8df3-76c9d00a383b"
-      ],
-      "parent_blueprint_id": null,
-      "internal_dns_version": 1,
-      "time_created": "2024-03-01T19:06:56.467313Z",
-      "creator": "54c947d2-6355-453c-80fc-8f49cc2129ee",
-      "comment": "from collection df8caafd-c444-4f65-a304-b9ceb62a96c2"
-    },
-    {
-      "id": "6c127695-ba15-408d-a992-325a1a888380",
-      "omicron_zones": {
-        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "0706860a-3801-42a7-8000-fff741f7665b",
-              "underlay_address": "fd00:1122:3344:101::c",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::c]:32345",
-                "dataset": {
-                  "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
-                }
-              }
-            },
-            {
-              "id": "34a279a2-a025-41ff-a392-1addac400f38",
-              "underlay_address": "fd00:1122:3344:101::3",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:101::3]:32221",
-                "dataset": {
-                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                }
-              }
-            },
-            {
-              "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
-              "underlay_address": "fd00:1122:3344:1::1",
-              "zone_type": {
-                "type": "internal_dns",
-                "dataset": {
-                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                },
-                "dns_address": "[fd00:1122:3344:1::1]:53",
-                "gz_address": "fd00:1122:3344:1::2",
-                "gz_address_index": 0,
-                "http_address": "[fd00:1122:3344:1::1]:5353"
-              }
-            },
-            {
-              "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-              "underlay_address": "fd00:1122:3344:101::5",
-              "zone_type": {
-                "type": "nexus",
-                "external_dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "external_ip": "172.20.28.2",
-                "external_tls": true,
-                "internal_address": "[fd00:1122:3344:101::5]:12221",
-                "nic": {
-                  "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
-                  "ip": "172.30.2.5",
-                  "kind": {
-                    "type": "service",
-                    "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
-                  },
-                  "mac": "A8:40:25:FF:E1:17",
-                  "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-                  "primary": true,
-                  "slot": 0,
-                  "subnet": "172.30.2.0/24",
-                  "vni": 100
-                }
-              }
-            },
-            {
-              "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
-              "underlay_address": "fd00:1122:3344:101::10",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::10]:32345",
-                "dataset": {
-                  "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
-                }
-              }
-            },
-            {
-              "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
-              "underlay_address": "fd00:1122:3344:101::f",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::f]:32345",
-                "dataset": {
-                  "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
-                }
-              }
-            },
-            {
-              "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
-              "underlay_address": "fd00:1122:3344:101::9",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
-                }
-              }
-            },
-            {
-              "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
-              "underlay_address": "fd00:1122:3344:101::8",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
-                }
-              }
-            },
-            {
-              "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
-              "underlay_address": "fd00:1122:3344:101::4",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:101::4]:32221",
-                "dataset": {
-                  "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
-                }
-              }
-            },
-            {
-              "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
-              "underlay_address": "fd00:1122:3344:101::d",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::d]:32345",
-                "dataset": {
-                  "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
-                }
-              }
-            },
-            {
-              "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
-              "underlay_address": "fd00:1122:3344:101::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
-                }
-              }
-            },
-            {
-              "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
-              "underlay_address": "fd00:1122:3344:101::6",
-              "zone_type": {
-                "type": "oximeter",
-                "address": "[fd00:1122:3344:101::6]:12223"
-              }
-            },
+      "id": "649e444d-d00f-4067-a915-12e15647bf37",
+      "omicron_zones": {
+        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
+          "generation": 5,
+          "zones": [
             {
-              "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
-              "underlay_address": "fd00:1122:3344:101::11",
+              "id": "0db88baf-911d-47bb-af71-3aebd7c96934",
+              "underlay_address": "fd00:1122:3344:103::4",
               "zone_type": {
-                "type": "boundary_ntp",
-                "address": "[fd00:1122:3344:101::11]:123",
-                "dns_servers": [
+                "type": "nexus",
+                "external_dns_servers": [
                   "1.1.1.1",
                   "9.9.9.9"
                 ],
+                "external_ip": "198.51.100.22",
+                "external_tls": false,
+                "internal_address": "[fd00:1122:3344:103::4]:12221",
                 "nic": {
-                  "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
-                  "ip": "172.30.3.5",
+                  "id": "8847a09c-cabc-429b-bf46-66d5e2e1140c",
                   "kind": {
                     "type": "service",
-                    "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
+                    "id": "0db88baf-911d-47bb-af71-3aebd7c96934"
                   },
-                  "mac": "A8:40:25:FF:E7:A8",
-                  "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                  "name": "nexus-0db88baf-911d-47bb-af71-3aebd7c96934",
+                  "ip": "172.30.2.5",
+                  "mac": "A8:40:25:FF:CC:DA",
+                  "subnet": "172.30.2.0/24",
+                  "vni": 100,
                   "primary": true,
-                  "slot": 0,
-                  "subnet": "172.30.3.0/24",
-                  "vni": 100
-                },
-                "ntp_servers": [
-                  "ntp.eng.oxide.computer"
-                ],
-                "snat_cfg": {
-                  "ip": "172.20.28.5",
-                  "first_port": 0,
-                  "last_port": 16383
+                  "slot": 0
                 }
               }
             },
             {
-              "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
-              "underlay_address": "fd00:1122:3344:101::e",
+              "id": "213d7dd3-95a4-45a7-99ca-e4899991dab4",
+              "underlay_address": "fd00:1122:3344:103::6",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:101::e]:32345",
+                "address": "[fd00:1122:3344:103::6]:32345",
                 "dataset": {
-                  "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                 }
               }
             },
             {
-              "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
-              "underlay_address": "fd00:1122:3344:101::b",
+              "id": "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
+              "underlay_address": "fd00:1122:3344:3::1",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::b]:32345",
+                "type": "internal_dns",
                 "dataset": {
-                  "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
-                }
+                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
+                },
+                "dns_address": "[fd00:1122:3344:3::1]:53",
+                "gz_address": "fd00:1122:3344:3::2",
+                "gz_address_index": 2,
+                "http_address": "[fd00:1122:3344:3::1]:5353"
               }
             },
             {
-              "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
-              "underlay_address": "fd00:1122:3344:101::7",
-              "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:101::7]:17000"
-              }
-            }
-          ]
-        },
-        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
-          "generation": 2,
-          "zones": [
-            {
-              "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
-              "underlay_address": "fd00:1122:3344:121::21",
-              "zone_type": {
-                "type": "internal_ntp",
-                "address": "[fd00:1122:3344:121::21]:123",
-                "dns_servers": [
-                  "fd00:1122:3344:1::1",
-                  "fd00:1122:3344:2::1",
-                  "fd00:1122:3344:3::1"
-                ],
-                "ntp_servers": [
-                  "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                  "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
-                ]
-              }
-            }
-          ]
-        },
-        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
-              "underlay_address": "fd00:1122:3344:102::11",
+              "id": "40c47ead-3be8-4724-a8ef-b66233fc5c65",
+              "underlay_address": "fd00:1122:3344:103::7",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:102::11]:32345",
+                "address": "[fd00:1122:3344:103::7]:32345",
                 "dataset": {
-                  "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                  "pool_name": "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03"
                 }
               }
             },
             {
-              "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
-              "underlay_address": "fd00:1122:3344:102::e",
+              "id": "425d9d2f-72ea-4f53-80b6-04e002771cdb",
+              "underlay_address": "fd00:1122:3344:103::9",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:102::e]:32345",
+                "address": "[fd00:1122:3344:103::9]:32345",
                 "dataset": {
-                  "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
+                  "pool_name": "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03"
                 }
               }
             },
             {
-              "id": "479811e7-6409-45d4-b294-5333534e47ed",
-              "underlay_address": "fd00:1122:3344:102::9",
+              "id": "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
+              "underlay_address": "fd00:1122:3344:103::8",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:102::9]:32345",
+                "address": "[fd00:1122:3344:103::8]:32345",
                 "dataset": {
-                  "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                  "pool_name": "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b"
                 }
               }
             },
             {
-              "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
-              "underlay_address": "fd00:1122:3344:102::a",
+              "id": "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
+              "underlay_address": "fd00:1122:3344:103::5",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::a]:32345",
+                "type": "clickhouse",
+                "address": "[fd00:1122:3344:103::5]:8123",
                 "dataset": {
-                  "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                 }
               }
             },
             {
-              "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
-              "underlay_address": "fd00:1122:3344:2::1",
-              "zone_type": {
-                "type": "internal_dns",
-                "dataset": {
-                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                },
-                "dns_address": "[fd00:1122:3344:2::1]:53",
-                "gz_address": "fd00:1122:3344:2::2",
-                "gz_address_index": 1,
-                "http_address": "[fd00:1122:3344:2::1]:5353"
-              }
-            },
-            {
-              "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
-              "underlay_address": "fd00:1122:3344:102::7",
-              "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:102::7]:17000"
-              }
-            },
-            {
-              "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
-              "underlay_address": "fd00:1122:3344:102::8",
+              "id": "7611a549-effc-41b6-b6cb-c03e0c70a31f",
+              "underlay_address": "fd00:1122:3344:103::b",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                }
+                "type": "internal_ntp",
+                "address": "[fd00:1122:3344:103::b]:123",
+                "dns_servers": [
+                  "fd00:1122:3344:1::1",
+                  "fd00:1122:3344:2::1",
+                  "fd00:1122:3344:3::1"
+                ],
+                "ntp_servers": [
+                  "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
+                  "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
+                ]
               }
             },
             {
-              "id": "658076ed-fb43-49e7-a813-df3d23497848",
-              "underlay_address": "fd00:1122:3344:102::10",
+              "id": "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
+              "underlay_address": "fd00:1122:3344:103::3",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::10]:32345",
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:103::3]:32221",
                 "dataset": {
-                  "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
+                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                 }
               }
             },
             {
-              "id": "6edb836e-b100-4080-bb03-738d6441d571",
-              "underlay_address": "fd00:1122:3344:102::c",
+              "id": "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
+              "underlay_address": "fd00:1122:3344:103::a",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:102::c]:32345",
+                "address": "[fd00:1122:3344:103::a]:32345",
                 "dataset": {
-                  "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
-                }
-              }
-            },
-            {
-              "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
-              "underlay_address": "fd00:1122:3344:102::5",
-              "zone_type": {
-                "type": "nexus",
-                "external_dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "external_ip": "172.20.28.3",
-                "external_tls": true,
-                "internal_address": "[fd00:1122:3344:102::5]:12221",
-                "nic": {
-                  "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
-                  "ip": "172.30.2.6",
-                  "kind": {
-                    "type": "service",
-                    "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
-                  },
-                  "mac": "A8:40:25:FF:B2:AF",
-                  "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
-                  "primary": true,
-                  "slot": 0,
-                  "subnet": "172.30.2.0/24",
-                  "vni": 100
+                  "pool_name": "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
                 }
               }
-            },
+            }
+          ]
+        },
+        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
+          "generation": 5,
+          "zones": [
             {
-              "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
-              "underlay_address": "fd00:1122:3344:102::d",
+              "id": "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
+              "underlay_address": "fd00:1122:3344:104::b",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:102::d]:32345",
+                "address": "[fd00:1122:3344:104::b]:32345",
                 "dataset": {
-                  "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                  "pool_name": "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
-              "underlay_address": "fd00:1122:3344:102::4",
+              "id": "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
+              "underlay_address": "fd00:1122:3344:104::4",
               "zone_type": {
                 "type": "cockroach_db",
-                "address": "[fd00:1122:3344:102::4]:32221",
-                "dataset": {
-                  "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
-                }
-              }
-            },
-            {
-              "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
-              "underlay_address": "fd00:1122:3344:102::6",
-              "zone_type": {
-                "type": "clickhouse",
-                "address": "[fd00:1122:3344:102::6]:8123",
+                "address": "[fd00:1122:3344:104::4]:32221",
                 "dataset": {
-                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
-              "underlay_address": "fd00:1122:3344:102::f",
+              "id": "57c712a6-6429-44b7-a9b2-9957cb9054d4",
+              "underlay_address": "fd00:1122:3344:104::a",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:102::f]:32345",
+                "address": "[fd00:1122:3344:104::a]:32345",
                 "dataset": {
-                  "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
+                  "pool_name": "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
-              "underlay_address": "fd00:1122:3344:102::3",
+              "id": "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
+              "underlay_address": "fd00:1122:3344:104::7",
               "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:102::3]:32221",
+                "type": "crucible",
+                "address": "[fd00:1122:3344:104::7]:32345",
                 "dataset": {
-                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-              "underlay_address": "fd00:1122:3344:102::12",
+              "id": "73916098-788f-48d8-b745-e13fdcfb771b",
+              "underlay_address": "fd00:1122:3344:104::c",
               "zone_type": {
-                "type": "boundary_ntp",
-                "address": "[fd00:1122:3344:102::12]:123",
+                "type": "internal_ntp",
+                "address": "[fd00:1122:3344:104::c]:123",
                 "dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
+                  "fd00:1122:3344:1::1",
+                  "fd00:1122:3344:2::1",
+                  "fd00:1122:3344:3::1"
                 ],
-                "nic": {
-                  "id": "9f03c82c-3192-498d-978d-a175049ac389",
-                  "ip": "172.30.3.6",
-                  "kind": {
-                    "type": "service",
-                    "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
-                  },
-                  "mac": "A8:40:25:FF:E4:11",
-                  "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-                  "primary": true,
-                  "slot": 0,
-                  "subnet": "172.30.3.0/24",
-                  "vni": 100
-                },
                 "ntp_servers": [
-                  "ntp.eng.oxide.computer"
-                ],
-                "snat_cfg": {
-                  "ip": "172.20.28.6",
-                  "first_port": 16384,
-                  "last_port": 32767
-                }
-              }
-            },
-            {
-              "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
-              "underlay_address": "fd00:1122:3344:102::b",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::b]:32345",
-                "dataset": {
-                  "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
-                }
-              }
-            }
-          ]
-        },
-        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
-              "underlay_address": "fd00:1122:3344:103::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
-                }
+                  "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
+                  "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
+                ]
               }
             },
             {
-              "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
-              "underlay_address": "fd00:1122:3344:103::10",
+              "id": "9bb513e3-7505-416c-aa70-b26f1152bbdc",
+              "underlay_address": "fd00:1122:3344:104::9",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:103::10]:32345",
+                "address": "[fd00:1122:3344:104::9]:32345",
                 "dataset": {
-                  "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
+                  "pool_name": "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b"
                 }
               }
             },
             {
-              "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
-              "underlay_address": "fd00:1122:3344:3::1",
-              "zone_type": {
-                "type": "internal_dns",
-                "dataset": {
-                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                },
-                "dns_address": "[fd00:1122:3344:3::1]:53",
-                "gz_address": "fd00:1122:3344:3::2",
-                "gz_address_index": 2,
-                "http_address": "[fd00:1122:3344:3::1]:5353"
-              }
-            },
-            {
-              "id": "4b74c418-0a08-4489-b652-e97267c1c220",
-              "underlay_address": "fd00:1122:3344:103::7",
+              "id": "9c14c879-137b-4f5b-8068-1292bffc83ed",
+              "underlay_address": "fd00:1122:3344:104::3",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::7]:32345",
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:104::3]:32221",
                 "dataset": {
-                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
-              "underlay_address": "fd00:1122:3344:103::5",
+              "id": "9cf90589-5da1-4b42-b09b-d784e4134637",
+              "underlay_address": "fd00:1122:3344:104::5",
               "zone_type": {
                 "type": "nexus",
                 "external_dns_servers": [
                   "1.1.1.1",
                   "9.9.9.9"
                 ],
-                "external_ip": "172.20.28.4",
-                "external_tls": true,
-                "internal_address": "[fd00:1122:3344:103::5]:12221",
+                "external_ip": "198.51.100.23",
+                "external_tls": false,
+                "internal_address": "[fd00:1122:3344:104::5]:12221",
                 "nic": {
-                  "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
-                  "ip": "172.30.2.7",
+                  "id": "fa76e7e1-dee6-4574-b71f-e07ea1cffd08",
                   "kind": {
                     "type": "service",
-                    "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
+                    "id": "9cf90589-5da1-4b42-b09b-d784e4134637"
                   },
-                  "mac": "A8:40:25:FF:96:05",
-                  "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
-                  "primary": true,
-                  "slot": 0,
+                  "name": "nexus-9cf90589-5da1-4b42-b09b-d784e4134637",
+                  "ip": "172.30.2.6",
+                  "mac": "A8:40:25:FF:C4:6F",
                   "subnet": "172.30.2.0/24",
-                  "vni": 100
-                }
-              }
-            },
-            {
-              "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
-              "underlay_address": "fd00:1122:3344:103::6",
-              "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:103::6]:17000"
-              }
-            },
-            {
-              "id": "921958c0-7761-4294-945d-2a078327bf2c",
-              "underlay_address": "fd00:1122:3344:103::8",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
-                }
-              }
-            },
-            {
-              "id": "9df22034-59f6-431f-82e7-258fe983879b",
-              "underlay_address": "fd00:1122:3344:103::4",
-              "zone_type": {
-                "type": "external_dns",
-                "dataset": {
-                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
-                },
-                "dns_address": "172.20.28.1:53",
-                "http_address": "[fd00:1122:3344:103::4]:5353",
-                "nic": {
-                  "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
-                  "ip": "172.30.1.5",
-                  "kind": {
-                    "type": "service",
-                    "id": "9df22034-59f6-431f-82e7-258fe983879b"
-                  },
-                  "mac": "A8:40:25:FF:90:12",
-                  "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
+                  "vni": 100,
                   "primary": true,
-                  "slot": 0,
-                  "subnet": "172.30.1.0/24",
-                  "vni": 100
+                  "slot": 0
                 }
               }
             },
             {
-              "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
-              "underlay_address": "fd00:1122:3344:103::11",
-              "zone_type": {
-                "type": "internal_ntp",
-                "address": "[fd00:1122:3344:103::11]:123",
-                "dns_servers": [
-                  "fd00:1122:3344:1::1",
-                  "fd00:1122:3344:2::1",
-                  "fd00:1122:3344:3::1"
-                ],
-                "ntp_servers": [
-                  "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                  "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
-                ]
+              "id": "d48b2dfa-c749-47dd-98b6-3f478cd63934",
+              "underlay_address": "fd00:1122:3344:104::6",
+              "zone_type": {
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:104::6]:17000"
               }
             },
             {
-              "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
-              "underlay_address": "fd00:1122:3344:103::9",
+              "id": "fd003bee-818a-4c95-aae5-e378ac601522",
+              "underlay_address": "fd00:1122:3344:104::8",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:103::9]:32345",
+                "address": "[fd00:1122:3344:104::8]:32345",
                 "dataset": {
-                  "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
+                  "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
-            },
+            }
+          ]
+        },
+        "77851655-10a5-4e80-88b0-a001473cab7e": {
+          "generation": 5,
+          "zones": [
             {
-              "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
-              "underlay_address": "fd00:1122:3344:103::d",
+              "id": "084cac6e-44b5-4333-a2ff-b0993b534898",
+              "underlay_address": "fd00:1122:3344:102::8",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:103::d]:32345",
+                "address": "[fd00:1122:3344:102::8]:32345",
                 "dataset": {
-                  "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
+                  "pool_name": "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
-              "underlay_address": "fd00:1122:3344:103::b",
+              "id": "11c527c5-0598-428e-9270-be5b74d3461b",
+              "underlay_address": "fd00:1122:3344:102::3",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::b]:32345",
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:102::3]:32221",
                 "dataset": {
-                  "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
+                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
-              "underlay_address": "fd00:1122:3344:103::e",
+              "id": "20be9299-495d-428d-b315-694c2c4b64af",
+              "underlay_address": "fd00:1122:3344:102::6",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::e]:32345",
-                "dataset": {
-                  "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
-                }
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:102::6]:17000"
               }
             },
             {
-              "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
-              "underlay_address": "fd00:1122:3344:103::3",
+              "id": "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
+              "underlay_address": "fd00:1122:3344:102::b",
               "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:103::3]:32221",
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::b]:32345",
                 "dataset": {
-                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  "pool_name": "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
-              "underlay_address": "fd00:1122:3344:103::f",
+              "id": "8d452bdd-e8b8-4108-a690-46fc59197db0",
+              "underlay_address": "fd00:1122:3344:102::a",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:103::f]:32345",
+                "address": "[fd00:1122:3344:102::a]:32345",
                 "dataset": {
-                  "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
+                  "pool_name": "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
-              "underlay_address": "fd00:1122:3344:103::c",
+              "id": "953019c0-8e85-4417-b231-d42820470125",
+              "underlay_address": "fd00:1122:3344:102::5",
+              "zone_type": {
+                "type": "oximeter",
+                "address": "[fd00:1122:3344:102::5]:12223"
+              }
+            },
+            {
+              "id": "99c635b9-8e97-414a-8316-8261d2d767d0",
+              "underlay_address": "fd00:1122:3344:102::7",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:103::c]:32345",
+                "address": "[fd00:1122:3344:102::7]:32345",
                 "dataset": {
-                  "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
-            }
-          ]
-        }
-      },
-      "zones_in_service": [
-        "0706860a-3801-42a7-8000-fff741f7665b",
-        "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
-        "185e6b00-7e18-456c-9dfc-edd0194ac207",
-        "18b99404-d8b2-4463-bc99-638d9b678ab3",
-        "1fc78bc7-feca-4a10-9beb-58b53fda5210",
-        "2d5c29f7-4915-4876-82f8-e388b15c5067",
-        "34a279a2-a025-41ff-a392-1addac400f38",
-        "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
-        "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
-        "479811e7-6409-45d4-b294-5333534e47ed",
-        "4b74c418-0a08-4489-b652-e97267c1c220",
-        "54c947d2-6355-453c-80fc-8f49cc2129ee",
-        "54d1b693-bc4d-4c47-9009-c6f624073801",
-        "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-        "5b743072-9369-45d7-b7c9-dbdf736301f8",
-        "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
-        "60d64b70-89f5-4774-9e41-2c79c4173911",
-        "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
-        "658076ed-fb43-49e7-a813-df3d23497848",
-        "67852e14-ce7f-4e50-a61d-4059d0c057b9",
-        "6ace1a69-d55a-45f0-a127-99c90f1b070c",
-        "6edb836e-b100-4080-bb03-738d6441d571",
-        "7f83d784-9c5c-498f-b559-4176bbafbc4b",
-        "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
-        "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
-        "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
-        "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
-        "86ee7359-aa35-41d9-8a76-54961bc516a1",
-        "921958c0-7761-4294-945d-2a078327bf2c",
-        "9241b9da-8188-4aae-a9dc-62acb453618f",
-        "925ea436-fad4-4fff-8ae0-24edd725a2b0",
-        "9df22034-59f6-431f-82e7-258fe983879b",
-        "a781a349-62b2-41f0-9dd7-c64d290aef82",
-        "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
-        "be272d1e-4d34-4478-94b4-a76f5fcbef36",
-        "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
-        "c67814bf-9972-4bb0-a666-24778ca70a16",
-        "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-        "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
-        "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
-        "d42fc874-8416-4212-b5a0-febba4c6b0ac",
-        "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
-        "d630cca7-e3f7-47a8-aee8-ad0790895abc",
-        "d8c51c79-46ff-44bd-80be-08d6283f4e21",
-        "da27ee6a-9841-46aa-acda-7cce3e04cb60",
-        "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
-        "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
-        "e09525cf-9c87-4d05-8860-1356dd0dee8a",
-        "f33a2375-212d-42e4-b539-6079aeb4e5b7",
-        "f7264a3e-e8e7-4476-8df3-76c9d00a383b"
-      ],
-      "parent_blueprint_id": "486de160-c8f3-4600-acca-b0c78e33aca4",
-      "internal_dns_version": 1,
-      "time_created": "2024-03-01T19:07:58.105708Z",
-      "creator": "54c947d2-6355-453c-80fc-8f49cc2129ee",
-      "comment": "sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f: add NTP zone"
-    },
-    {
-      "id": "cfa03594-f438-4736-a416-6307f7bf8f2e",
-      "omicron_zones": {
-        "1762267a-a0cb-4c9a-88b4-8ce828cbc021": {
-          "generation": 5,
-          "zones": [
+            },
             {
-              "id": "0706860a-3801-42a7-8000-fff741f7665b",
-              "underlay_address": "fd00:1122:3344:101::c",
+              "id": "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
+              "underlay_address": "fd00:1122:3344:2::1",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::c]:32345",
+                "type": "internal_dns",
                 "dataset": {
-                  "pool_name": "oxp_1d864940-c723-4f58-82e8-b03772b42356"
+                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
+                },
+                "dns_address": "[fd00:1122:3344:2::1]:53",
+                "gz_address": "fd00:1122:3344:2::2",
+                "gz_address_index": 1,
+                "http_address": "[fd00:1122:3344:2::1]:5353"
+              }
+            },
+            {
+              "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
+              "underlay_address": "fd00:1122:3344:102::c",
+              "zone_type": {
+                "type": "boundary_ntp",
+                "address": "[fd00:1122:3344:102::c]:123",
+                "dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "nic": {
+                  "id": "659f33f7-e7bd-4875-9140-bf8a8b2d964a",
+                  "kind": {
+                    "type": "service",
+                    "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded"
+                  },
+                  "name": "ntp-d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
+                  "ip": "172.30.3.6",
+                  "mac": "A8:40:25:FF:B1:99",
+                  "subnet": "172.30.3.0/24",
+                  "vni": 100,
+                  "primary": true,
+                  "slot": 0
+                },
+                "ntp_servers": [
+                  "time.cloudflare.com"
+                ],
+                "snat_cfg": {
+                  "ip": "198.51.100.26",
+                  "first_port": 16384,
+                  "last_port": 32767
                 }
               }
             },
             {
-              "id": "34a279a2-a025-41ff-a392-1addac400f38",
-              "underlay_address": "fd00:1122:3344:101::3",
+              "id": "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
+              "underlay_address": "fd00:1122:3344:102::9",
               "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:101::3]:32221",
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::9]:32345",
                 "dataset": {
-                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  "pool_name": "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b"
                 }
               }
             },
             {
-              "id": "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
-              "underlay_address": "fd00:1122:3344:1::1",
+              "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
+              "underlay_address": "fd00:1122:3344:102::4",
               "zone_type": {
-                "type": "internal_dns",
+                "type": "external_dns",
                 "dataset": {
-                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                 },
-                "dns_address": "[fd00:1122:3344:1::1]:53",
-                "gz_address": "fd00:1122:3344:1::2",
-                "gz_address_index": 0,
-                "http_address": "[fd00:1122:3344:1::1]:5353"
+                "dns_address": "198.51.100.21:53",
+                "http_address": "[fd00:1122:3344:102::4]:5353",
+                "nic": {
+                  "id": "d4dd2f7a-8a3b-405e-8444-62f99a754a8a",
+                  "kind": {
+                    "type": "service",
+                    "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802"
+                  },
+                  "name": "external-dns-f0ed9838-6f1b-4646-a478-9f74f2e6e802",
+                  "ip": "172.30.1.6",
+                  "mac": "A8:40:25:FF:F4:5C",
+                  "subnet": "172.30.1.0/24",
+                  "vni": 100,
+                  "primary": true,
+                  "slot": 0
+                }
               }
-            },
+            }
+          ]
+        },
+        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
+          "generation": 5,
+          "zones": [
             {
-              "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
+              "id": "0d459323-e414-4f32-9944-76c7331da622",
               "underlay_address": "fd00:1122:3344:101::5",
               "zone_type": {
                 "type": "nexus",
@@ -10756,732 +7757,842 @@
                   "1.1.1.1",
                   "9.9.9.9"
                 ],
-                "external_ip": "172.20.28.2",
-                "external_tls": true,
+                "external_ip": "198.51.100.24",
+                "external_tls": false,
                 "internal_address": "[fd00:1122:3344:101::5]:12221",
                 "nic": {
-                  "id": "81a0385e-86b9-4109-b63d-61c1407c50d1",
-                  "ip": "172.30.2.5",
+                  "id": "514902b8-7a1a-42a3-95a6-d00861833b5c",
                   "kind": {
                     "type": "service",
-                    "id": "59c3ed4f-3670-44b1-9ea0-9a6162a08629"
+                    "id": "0d459323-e414-4f32-9944-76c7331da622"
                   },
-                  "mac": "A8:40:25:FF:E1:17",
-                  "name": "nexus-59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-                  "primary": true,
-                  "slot": 0,
+                  "name": "nexus-0d459323-e414-4f32-9944-76c7331da622",
+                  "ip": "172.30.2.7",
+                  "mac": "A8:40:25:FF:DE:AC",
                   "subnet": "172.30.2.0/24",
-                  "vni": 100
+                  "vni": 100,
+                  "primary": true,
+                  "slot": 0
                 }
               }
             },
             {
-              "id": "67852e14-ce7f-4e50-a61d-4059d0c057b9",
-              "underlay_address": "fd00:1122:3344:101::10",
+              "id": "2902972b-3ae4-489d-b937-6480352e214a",
+              "underlay_address": "fd00:1122:3344:101::8",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:101::10]:32345",
+                "address": "[fd00:1122:3344:101::8]:32345",
                 "dataset": {
-                  "pool_name": "oxp_eae1515c-cd4a-4d7b-9a99-dbc24cf2e337"
+                  "pool_name": "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "6ace1a69-d55a-45f0-a127-99c90f1b070c",
-              "underlay_address": "fd00:1122:3344:101::f",
+              "id": "2ee832de-af75-47ac-abac-f91304857ca4",
+              "underlay_address": "fd00:1122:3344:101::4",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::f]:32345",
+                "type": "external_dns",
                 "dataset": {
-                  "pool_name": "oxp_466351db-2ced-45fa-9431-22de1c4bd480"
+                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
+                },
+                "dns_address": "198.51.100.20:53",
+                "http_address": "[fd00:1122:3344:101::4]:5353",
+                "nic": {
+                  "id": "fadff4ab-62d7-466a-9203-04080e4078ed",
+                  "kind": {
+                    "type": "service",
+                    "id": "2ee832de-af75-47ac-abac-f91304857ca4"
+                  },
+                  "name": "external-dns-2ee832de-af75-47ac-abac-f91304857ca4",
+                  "ip": "172.30.1.5",
+                  "mac": "A8:40:25:FF:B4:93",
+                  "subnet": "172.30.1.0/24",
+                  "vni": 100,
+                  "primary": true,
+                  "slot": 0
                 }
               }
             },
             {
-              "id": "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
-              "underlay_address": "fd00:1122:3344:101::9",
+              "id": "30050a7e-7fc3-4965-b07b-1635212393eb",
+              "underlay_address": "fd00:1122:3344:101::6",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:101::6]:17000"
+              }
+            },
+            {
+              "id": "8494befe-6df6-42d1-9baa-599ba4228430",
+              "underlay_address": "fd00:1122:3344:101::c",
+              "zone_type": {
+                "type": "boundary_ntp",
+                "address": "[fd00:1122:3344:101::c]:123",
+                "dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "nic": {
+                  "id": "23041748-e3a6-4846-930d-5a348c316e91",
+                  "kind": {
+                    "type": "service",
+                    "id": "8494befe-6df6-42d1-9baa-599ba4228430"
+                  },
+                  "name": "ntp-8494befe-6df6-42d1-9baa-599ba4228430",
+                  "ip": "172.30.3.5",
+                  "mac": "A8:40:25:FF:D1:C4",
+                  "subnet": "172.30.3.0/24",
+                  "vni": 100,
+                  "primary": true,
+                  "slot": 0
+                },
+                "ntp_servers": [
+                  "time.cloudflare.com"
+                ],
+                "snat_cfg": {
+                  "ip": "198.51.100.25",
+                  "first_port": 0,
+                  "last_port": 16383
                 }
               }
             },
             {
-              "id": "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
-              "underlay_address": "fd00:1122:3344:101::8",
+              "id": "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
+              "underlay_address": "fd00:1122:3344:101::7",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:101::8]:32345",
+                "address": "[fd00:1122:3344:101::7]:32345",
                 "dataset": {
-                  "pool_name": "oxp_88d39560-b091-4fad-bad0-b0b1c514033b"
+                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "86ee7359-aa35-41d9-8a76-54961bc516a1",
-              "underlay_address": "fd00:1122:3344:101::4",
+              "id": "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
+              "underlay_address": "fd00:1122:3344:101::3",
               "zone_type": {
                 "type": "cockroach_db",
-                "address": "[fd00:1122:3344:101::4]:32221",
+                "address": "[fd00:1122:3344:101::3]:32221",
                 "dataset": {
-                  "pool_name": "oxp_e8d6b9a2-bad4-43f0-b5c0-0618555a7f8c"
+                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "925ea436-fad4-4fff-8ae0-24edd725a2b0",
-              "underlay_address": "fd00:1122:3344:101::d",
+              "id": "e668e3f5-5457-4689-97da-b2af537787ac",
+              "underlay_address": "fd00:1122:3344:101::b",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:101::d]:32345",
+                "address": "[fd00:1122:3344:101::b]:32345",
                 "dataset": {
-                  "pool_name": "oxp_88ddd98f-3181-4797-864a-7e8d5b028657"
+                  "pool_name": "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "be272d1e-4d34-4478-94b4-a76f5fcbef36",
-              "underlay_address": "fd00:1122:3344:101::a",
+              "id": "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
+              "underlay_address": "fd00:1122:3344:101::9",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:101::a]:32345",
+                "address": "[fd00:1122:3344:101::9]:32345",
                 "dataset": {
-                  "pool_name": "oxp_9041d780-b5ee-4159-a1db-8c4075e2176d"
+                  "pool_name": "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b"
                 }
               }
             },
             {
-              "id": "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
-              "underlay_address": "fd00:1122:3344:101::6",
+              "id": "f8c79ba4-194a-4cfe-8090-5315e297b780",
+              "underlay_address": "fd00:1122:3344:1::1",
               "zone_type": {
-                "type": "oximeter",
-                "address": "[fd00:1122:3344:101::6]:12223"
+                "type": "internal_dns",
+                "dataset": {
+                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
+                },
+                "dns_address": "[fd00:1122:3344:1::1]:53",
+                "gz_address": "fd00:1122:3344:1::2",
+                "gz_address_index": 0,
+                "http_address": "[fd00:1122:3344:1::1]:5353"
               }
             },
             {
-              "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc",
-              "underlay_address": "fd00:1122:3344:101::11",
+              "id": "fa5e0176-b4a8-4598-aa31-40fd562d246c",
+              "underlay_address": "fd00:1122:3344:101::a",
               "zone_type": {
-                "type": "boundary_ntp",
-                "address": "[fd00:1122:3344:101::11]:123",
-                "dns_servers": [
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::a]:32345",
+                "dataset": {
+                  "pool_name": "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03"
+                }
+              }
+            }
+          ]
+        }
+      },
+      "zones_in_service": [
+        "084cac6e-44b5-4333-a2ff-b0993b534898",
+        "0d459323-e414-4f32-9944-76c7331da622",
+        "0db88baf-911d-47bb-af71-3aebd7c96934",
+        "11c527c5-0598-428e-9270-be5b74d3461b",
+        "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
+        "20be9299-495d-428d-b315-694c2c4b64af",
+        "213d7dd3-95a4-45a7-99ca-e4899991dab4",
+        "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
+        "2902972b-3ae4-489d-b937-6480352e214a",
+        "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
+        "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
+        "2ee832de-af75-47ac-abac-f91304857ca4",
+        "30050a7e-7fc3-4965-b07b-1635212393eb",
+        "40c47ead-3be8-4724-a8ef-b66233fc5c65",
+        "425d9d2f-72ea-4f53-80b6-04e002771cdb",
+        "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
+        "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
+        "57c712a6-6429-44b7-a9b2-9957cb9054d4",
+        "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
+        "73916098-788f-48d8-b745-e13fdcfb771b",
+        "7611a549-effc-41b6-b6cb-c03e0c70a31f",
+        "8494befe-6df6-42d1-9baa-599ba4228430",
+        "8d452bdd-e8b8-4108-a690-46fc59197db0",
+        "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
+        "953019c0-8e85-4417-b231-d42820470125",
+        "99c635b9-8e97-414a-8316-8261d2d767d0",
+        "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
+        "9bb513e3-7505-416c-aa70-b26f1152bbdc",
+        "9c14c879-137b-4f5b-8068-1292bffc83ed",
+        "9cf90589-5da1-4b42-b09b-d784e4134637",
+        "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
+        "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
+        "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
+        "d48b2dfa-c749-47dd-98b6-3f478cd63934",
+        "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
+        "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
+        "e668e3f5-5457-4689-97da-b2af537787ac",
+        "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
+        "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
+        "f8c79ba4-194a-4cfe-8090-5315e297b780",
+        "fa5e0176-b4a8-4598-aa31-40fd562d246c",
+        "fd003bee-818a-4c95-aae5-e378ac601522"
+      ],
+      "parent_blueprint_id": null,
+      "internal_dns_version": 1,
+      "time_created": "2024-03-12T19:09:41.953854Z",
+      "creator": "0d459323-e414-4f32-9944-76c7331da622",
+      "comment": "from collection 3f61b723-cfe7-40ec-b22c-e3e1f35325f9"
+    },
+    {
+      "id": "cd34e87b-d821-4768-b32f-f41c28db835e",
+      "omicron_zones": {
+        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
+          "generation": 5,
+          "zones": [
+            {
+              "id": "0db88baf-911d-47bb-af71-3aebd7c96934",
+              "underlay_address": "fd00:1122:3344:103::4",
+              "zone_type": {
+                "type": "nexus",
+                "external_dns_servers": [
                   "1.1.1.1",
                   "9.9.9.9"
                 ],
+                "external_ip": "198.51.100.22",
+                "external_tls": false,
+                "internal_address": "[fd00:1122:3344:103::4]:12221",
                 "nic": {
-                  "id": "9741f03f-f150-4229-9f8d-e29a69c87883",
-                  "ip": "172.30.3.5",
+                  "id": "8847a09c-cabc-429b-bf46-66d5e2e1140c",
                   "kind": {
                     "type": "service",
-                    "id": "d630cca7-e3f7-47a8-aee8-ad0790895abc"
+                    "id": "0db88baf-911d-47bb-af71-3aebd7c96934"
                   },
-                  "mac": "A8:40:25:FF:E7:A8",
-                  "name": "ntp-d630cca7-e3f7-47a8-aee8-ad0790895abc",
+                  "name": "nexus-0db88baf-911d-47bb-af71-3aebd7c96934",
+                  "ip": "172.30.2.5",
+                  "mac": "A8:40:25:FF:CC:DA",
+                  "subnet": "172.30.2.0/24",
+                  "vni": 100,
                   "primary": true,
-                  "slot": 0,
-                  "subnet": "172.30.3.0/24",
-                  "vni": 100
-                },
-                "ntp_servers": [
-                  "ntp.eng.oxide.computer"
-                ],
-                "snat_cfg": {
-                  "ip": "172.20.28.5",
-                  "first_port": 0,
-                  "last_port": 16383
+                  "slot": 0
                 }
               }
             },
             {
-              "id": "d8c51c79-46ff-44bd-80be-08d6283f4e21",
-              "underlay_address": "fd00:1122:3344:101::e",
+              "id": "213d7dd3-95a4-45a7-99ca-e4899991dab4",
+              "underlay_address": "fd00:1122:3344:103::6",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:101::e]:32345",
+                "address": "[fd00:1122:3344:103::6]:32345",
                 "dataset": {
-                  "pool_name": "oxp_3f95c036-d5d8-4d08-8c78-c0635ca34698"
+                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                 }
               }
             },
             {
-              "id": "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
-              "underlay_address": "fd00:1122:3344:101::b",
+              "id": "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
+              "underlay_address": "fd00:1122:3344:3::1",
+              "zone_type": {
+                "type": "internal_dns",
+                "dataset": {
+                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
+                },
+                "dns_address": "[fd00:1122:3344:3::1]:53",
+                "gz_address": "fd00:1122:3344:3::2",
+                "gz_address_index": 2,
+                "http_address": "[fd00:1122:3344:3::1]:5353"
+              }
+            },
+            {
+              "id": "40c47ead-3be8-4724-a8ef-b66233fc5c65",
+              "underlay_address": "fd00:1122:3344:103::7",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:101::b]:32345",
+                "address": "[fd00:1122:3344:103::7]:32345",
                 "dataset": {
-                  "pool_name": "oxp_1a4447be-5c84-4dad-b22c-d35e5e1911b0"
+                  "pool_name": "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03"
                 }
               }
             },
             {
-              "id": "f33a2375-212d-42e4-b539-6079aeb4e5b7",
-              "underlay_address": "fd00:1122:3344:101::7",
+              "id": "425d9d2f-72ea-4f53-80b6-04e002771cdb",
+              "underlay_address": "fd00:1122:3344:103::9",
               "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:101::7]:17000"
+                "type": "crucible",
+                "address": "[fd00:1122:3344:103::9]:32345",
+                "dataset": {
+                  "pool_name": "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03"
+                }
               }
-            }
-          ]
-        },
-        "a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f": {
-          "generation": 3,
-          "zones": [
+            },
             {
-              "id": "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
-              "underlay_address": "fd00:1122:3344:121::28",
+              "id": "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
+              "underlay_address": "fd00:1122:3344:103::8",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:121::28]:32345",
+                "address": "[fd00:1122:3344:103::8]:32345",
                 "dataset": {
-                  "pool_name": "oxp_cc36b0cc-818a-4c7f-b50d-057ff7e7e8e3"
+                  "pool_name": "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b"
                 }
               }
             },
             {
-              "id": "12431d3e-1516-4822-98b3-3711ff912e69",
-              "underlay_address": "fd00:1122:3344:121::22",
+              "id": "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
+              "underlay_address": "fd00:1122:3344:103::5",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:121::22]:32345",
+                "type": "clickhouse",
+                "address": "[fd00:1122:3344:103::5]:8123",
                 "dataset": {
-                  "pool_name": "oxp_0622b2cf-297e-4356-b94c-d74309f443ba"
+                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                 }
               }
             },
             {
-              "id": "2d5c29f7-4915-4876-82f8-e388b15c5067",
-              "underlay_address": "fd00:1122:3344:121::21",
+              "id": "7611a549-effc-41b6-b6cb-c03e0c70a31f",
+              "underlay_address": "fd00:1122:3344:103::b",
               "zone_type": {
                 "type": "internal_ntp",
-                "address": "[fd00:1122:3344:121::21]:123",
+                "address": "[fd00:1122:3344:103::b]:123",
                 "dns_servers": [
                   "fd00:1122:3344:1::1",
                   "fd00:1122:3344:2::1",
                   "fd00:1122:3344:3::1"
                 ],
                 "ntp_servers": [
-                  "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                  "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
-                ]
-              }
-            },
-            {
-              "id": "56414d76-9b8c-4e15-acf1-cd269c2d8638",
-              "underlay_address": "fd00:1122:3344:121::25",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:121::25]:32345",
-                "dataset": {
-                  "pool_name": "oxp_574728ca-1559-48f3-a46c-1e608966e6a0"
-                }
+                  "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
+                  "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
+                ]
               }
             },
             {
-              "id": "891cb9da-301d-40cc-b93d-5fcb350fa968",
-              "underlay_address": "fd00:1122:3344:121::29",
+              "id": "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
+              "underlay_address": "fd00:1122:3344:103::3",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:121::29]:32345",
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:103::3]:32221",
                 "dataset": {
-                  "pool_name": "oxp_e4a4a09a-43b8-460c-bc43-1f935600b681"
+                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
                 }
               }
             },
             {
-              "id": "d9014907-24c0-4c33-a475-db15942f4f8f",
-              "underlay_address": "fd00:1122:3344:121::23",
+              "id": "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
+              "underlay_address": "fd00:1122:3344:103::a",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:121::23]:32345",
+                "address": "[fd00:1122:3344:103::a]:32345",
                 "dataset": {
-                  "pool_name": "oxp_276822e2-d7ce-4eae-b5a4-38d2e8586a37"
+                  "pool_name": "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
                 }
               }
-            },
+            }
+          ]
+        },
+        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
+          "generation": 5,
+          "zones": [
             {
-              "id": "e950de25-7306-4cf7-a6f4-6db11adaa310",
-              "underlay_address": "fd00:1122:3344:121::27",
+              "id": "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
+              "underlay_address": "fd00:1122:3344:104::b",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:121::27]:32345",
+                "address": "[fd00:1122:3344:104::b]:32345",
                 "dataset": {
-                  "pool_name": "oxp_8c7a7d84-c269-4178-9dbb-37f363e7958c"
+                  "pool_name": "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
-              "underlay_address": "fd00:1122:3344:121::26",
+              "id": "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
+              "underlay_address": "fd00:1122:3344:104::4",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:121::26]:32345",
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:104::4]:32221",
                 "dataset": {
-                  "pool_name": "oxp_6d2b29c5-aa58-40b7-b630-88301f08e945"
+                  "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00",
-              "underlay_address": "fd00:1122:3344:121::24",
+              "id": "57c712a6-6429-44b7-a9b2-9957cb9054d4",
+              "underlay_address": "fd00:1122:3344:104::a",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:121::24]:32345",
+                "address": "[fd00:1122:3344:104::a]:32345",
                 "dataset": {
-                  "pool_name": "oxp_2b0da7bc-2afc-4a16-95e1-5126bcb9b6b1"
+                  "pool_name": "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
-            }
-          ]
-        },
-        "a6634f64-f6fb-426c-b00b-9b30aed9f53a": {
-          "generation": 5,
-          "zones": [
+            },
             {
-              "id": "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
-              "underlay_address": "fd00:1122:3344:102::11",
+              "id": "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
+              "underlay_address": "fd00:1122:3344:104::7",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:102::11]:32345",
+                "address": "[fd00:1122:3344:104::7]:32345",
                 "dataset": {
-                  "pool_name": "oxp_0c07bcc1-7666-4754-881d-4135992ff561"
+                  "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "18b99404-d8b2-4463-bc99-638d9b678ab3",
-              "underlay_address": "fd00:1122:3344:102::e",
+              "id": "73916098-788f-48d8-b745-e13fdcfb771b",
+              "underlay_address": "fd00:1122:3344:104::c",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::e]:32345",
-                "dataset": {
-                  "pool_name": "oxp_f9a037da-7d15-40d9-957d-7e42aadb333c"
-                }
+                "type": "internal_ntp",
+                "address": "[fd00:1122:3344:104::c]:123",
+                "dns_servers": [
+                  "fd00:1122:3344:1::1",
+                  "fd00:1122:3344:2::1",
+                  "fd00:1122:3344:3::1"
+                ],
+                "ntp_servers": [
+                  "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
+                  "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
+                ]
               }
             },
             {
-              "id": "479811e7-6409-45d4-b294-5333534e47ed",
-              "underlay_address": "fd00:1122:3344:102::9",
+              "id": "9bb513e3-7505-416c-aa70-b26f1152bbdc",
+              "underlay_address": "fd00:1122:3344:104::9",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:102::9]:32345",
+                "address": "[fd00:1122:3344:104::9]:32345",
                 "dataset": {
-                  "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                  "pool_name": "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b"
                 }
               }
             },
             {
-              "id": "54d1b693-bc4d-4c47-9009-c6f624073801",
-              "underlay_address": "fd00:1122:3344:102::a",
+              "id": "9c14c879-137b-4f5b-8068-1292bffc83ed",
+              "underlay_address": "fd00:1122:3344:104::3",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::a]:32345",
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:104::3]:32221",
                 "dataset": {
-                  "pool_name": "oxp_02cc2369-9924-41c7-aa74-79d6d548f2dd"
+                  "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "5b743072-9369-45d7-b7c9-dbdf736301f8",
-              "underlay_address": "fd00:1122:3344:2::1",
+              "id": "9cf90589-5da1-4b42-b09b-d784e4134637",
+              "underlay_address": "fd00:1122:3344:104::5",
               "zone_type": {
-                "type": "internal_dns",
-                "dataset": {
-                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                },
-                "dns_address": "[fd00:1122:3344:2::1]:53",
-                "gz_address": "fd00:1122:3344:2::2",
-                "gz_address_index": 1,
-                "http_address": "[fd00:1122:3344:2::1]:5353"
+                "type": "nexus",
+                "external_dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "external_ip": "198.51.100.23",
+                "external_tls": false,
+                "internal_address": "[fd00:1122:3344:104::5]:12221",
+                "nic": {
+                  "id": "fa76e7e1-dee6-4574-b71f-e07ea1cffd08",
+                  "kind": {
+                    "type": "service",
+                    "id": "9cf90589-5da1-4b42-b09b-d784e4134637"
+                  },
+                  "name": "nexus-9cf90589-5da1-4b42-b09b-d784e4134637",
+                  "ip": "172.30.2.6",
+                  "mac": "A8:40:25:FF:C4:6F",
+                  "subnet": "172.30.2.0/24",
+                  "vni": 100,
+                  "primary": true,
+                  "slot": 0
+                }
               }
             },
             {
-              "id": "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
-              "underlay_address": "fd00:1122:3344:102::7",
+              "id": "d48b2dfa-c749-47dd-98b6-3f478cd63934",
+              "underlay_address": "fd00:1122:3344:104::6",
               "zone_type": {
                 "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:102::7]:17000"
+                "address": "[fd00:1122:3344:104::6]:17000"
               }
             },
             {
-              "id": "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
-              "underlay_address": "fd00:1122:3344:102::8",
+              "id": "fd003bee-818a-4c95-aae5-e378ac601522",
+              "underlay_address": "fd00:1122:3344:104::8",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:102::8]:32345",
+                "address": "[fd00:1122:3344:104::8]:32345",
                 "dataset": {
-                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
+                  "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
-            },
+            }
+          ]
+        },
+        "77851655-10a5-4e80-88b0-a001473cab7e": {
+          "generation": 5,
+          "zones": [
             {
-              "id": "658076ed-fb43-49e7-a813-df3d23497848",
-              "underlay_address": "fd00:1122:3344:102::10",
+              "id": "084cac6e-44b5-4333-a2ff-b0993b534898",
+              "underlay_address": "fd00:1122:3344:102::8",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:102::10]:32345",
+                "address": "[fd00:1122:3344:102::8]:32345",
                 "dataset": {
-                  "pool_name": "oxp_7929b5fb-4ef2-4791-92e6-1a46c7f608b9"
+                  "pool_name": "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "6edb836e-b100-4080-bb03-738d6441d571",
-              "underlay_address": "fd00:1122:3344:102::c",
+              "id": "11c527c5-0598-428e-9270-be5b74d3461b",
+              "underlay_address": "fd00:1122:3344:102::3",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::c]:32345",
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:102::3]:32221",
                 "dataset": {
-                  "pool_name": "oxp_fe74ae4d-d7bc-4293-9ec9-69132aca4aba"
+                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b",
-              "underlay_address": "fd00:1122:3344:102::5",
+              "id": "20be9299-495d-428d-b315-694c2c4b64af",
+              "underlay_address": "fd00:1122:3344:102::6",
               "zone_type": {
-                "type": "nexus",
-                "external_dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "external_ip": "172.20.28.3",
-                "external_tls": true,
-                "internal_address": "[fd00:1122:3344:102::5]:12221",
-                "nic": {
-                  "id": "3771a19c-cc40-4fd5-b99d-ed10071018a7",
-                  "ip": "172.30.2.6",
-                  "kind": {
-                    "type": "service",
-                    "id": "7f83d784-9c5c-498f-b559-4176bbafbc4b"
-                  },
-                  "mac": "A8:40:25:FF:B2:AF",
-                  "name": "nexus-7f83d784-9c5c-498f-b559-4176bbafbc4b",
-                  "primary": true,
-                  "slot": 0,
-                  "subnet": "172.30.2.0/24",
-                  "vni": 100
-                }
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:102::6]:17000"
               }
             },
             {
-              "id": "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
-              "underlay_address": "fd00:1122:3344:102::d",
+              "id": "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
+              "underlay_address": "fd00:1122:3344:102::b",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:102::d]:32345",
+                "address": "[fd00:1122:3344:102::b]:32345",
                 "dataset": {
-                  "pool_name": "oxp_93e4e338-0ad1-4ae1-a120-2cfb59b4df42"
+                  "pool_name": "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
-              "underlay_address": "fd00:1122:3344:102::4",
+              "id": "8d452bdd-e8b8-4108-a690-46fc59197db0",
+              "underlay_address": "fd00:1122:3344:102::a",
               "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:102::4]:32221",
+                "type": "crucible",
+                "address": "[fd00:1122:3344:102::a]:32345",
                 "dataset": {
-                  "pool_name": "oxp_1fb80902-d979-461d-9c6b-a6541a5359c9"
+                  "pool_name": "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "9241b9da-8188-4aae-a9dc-62acb453618f",
-              "underlay_address": "fd00:1122:3344:102::6",
+              "id": "953019c0-8e85-4417-b231-d42820470125",
+              "underlay_address": "fd00:1122:3344:102::5",
               "zone_type": {
-                "type": "clickhouse",
-                "address": "[fd00:1122:3344:102::6]:8123",
-                "dataset": {
-                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                }
+                "type": "oximeter",
+                "address": "[fd00:1122:3344:102::5]:12223"
               }
             },
             {
-              "id": "a781a349-62b2-41f0-9dd7-c64d290aef82",
-              "underlay_address": "fd00:1122:3344:102::f",
+              "id": "99c635b9-8e97-414a-8316-8261d2d767d0",
+              "underlay_address": "fd00:1122:3344:102::7",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:102::f]:32345",
+                "address": "[fd00:1122:3344:102::7]:32345",
                 "dataset": {
-                  "pool_name": "oxp_4e92cbcf-f194-4ffc-97e3-fdf3df7cc9ae"
+                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "c67814bf-9972-4bb0-a666-24778ca70a16",
-              "underlay_address": "fd00:1122:3344:102::3",
+              "id": "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
+              "underlay_address": "fd00:1122:3344:2::1",
               "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:102::3]:32221",
+                "type": "internal_dns",
                 "dataset": {
-                  "pool_name": "oxp_6a275151-d85f-488b-81a6-e558fdede658"
-                }
+                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
+                },
+                "dns_address": "[fd00:1122:3344:2::1]:53",
+                "gz_address": "fd00:1122:3344:2::2",
+                "gz_address_index": 1,
+                "http_address": "[fd00:1122:3344:2::1]:5353"
               }
             },
             {
-              "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-              "underlay_address": "fd00:1122:3344:102::12",
+              "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
+              "underlay_address": "fd00:1122:3344:102::c",
               "zone_type": {
                 "type": "boundary_ntp",
-                "address": "[fd00:1122:3344:102::12]:123",
+                "address": "[fd00:1122:3344:102::c]:123",
                 "dns_servers": [
                   "1.1.1.1",
                   "9.9.9.9"
                 ],
                 "nic": {
-                  "id": "9f03c82c-3192-498d-978d-a175049ac389",
-                  "ip": "172.30.3.6",
+                  "id": "659f33f7-e7bd-4875-9140-bf8a8b2d964a",
                   "kind": {
                     "type": "service",
-                    "id": "c74ba7d7-3d5f-4a29-aadc-effe33a90909"
+                    "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded"
                   },
-                  "mac": "A8:40:25:FF:E4:11",
-                  "name": "ntp-c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-                  "primary": true,
-                  "slot": 0,
+                  "name": "ntp-d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
+                  "ip": "172.30.3.6",
+                  "mac": "A8:40:25:FF:B1:99",
                   "subnet": "172.30.3.0/24",
-                  "vni": 100
+                  "vni": 100,
+                  "primary": true,
+                  "slot": 0
                 },
                 "ntp_servers": [
-                  "ntp.eng.oxide.computer"
+                  "time.cloudflare.com"
                 ],
                 "snat_cfg": {
-                  "ip": "172.20.28.6",
+                  "ip": "198.51.100.26",
                   "first_port": 16384,
                   "last_port": 32767
                 }
               }
             },
             {
-              "id": "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
-              "underlay_address": "fd00:1122:3344:102::b",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::b]:32345",
-                "dataset": {
-                  "pool_name": "oxp_742699b5-4ded-406d-9a08-24648d3b2efb"
-                }
-              }
-            }
-          ]
-        },
-        "fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "185e6b00-7e18-456c-9dfc-edd0194ac207",
-              "underlay_address": "fd00:1122:3344:103::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_6aa59516-2018-4494-af6b-46704b20f6dc"
-                }
-              }
-            },
-            {
-              "id": "1fc78bc7-feca-4a10-9beb-58b53fda5210",
-              "underlay_address": "fd00:1122:3344:103::10",
+              "id": "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
+              "underlay_address": "fd00:1122:3344:102::9",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:103::10]:32345",
+                "address": "[fd00:1122:3344:102::9]:32345",
                 "dataset": {
-                  "pool_name": "oxp_07d66b6b-8059-4d07-a0b3-8ce41e2a8c98"
+                  "pool_name": "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b"
                 }
               }
             },
             {
-              "id": "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
-              "underlay_address": "fd00:1122:3344:3::1",
+              "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
+              "underlay_address": "fd00:1122:3344:102::4",
               "zone_type": {
-                "type": "internal_dns",
+                "type": "external_dns",
                 "dataset": {
-                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
                 },
-                "dns_address": "[fd00:1122:3344:3::1]:53",
-                "gz_address": "fd00:1122:3344:3::2",
-                "gz_address_index": 2,
-                "http_address": "[fd00:1122:3344:3::1]:5353"
-              }
-            },
-            {
-              "id": "4b74c418-0a08-4489-b652-e97267c1c220",
-              "underlay_address": "fd00:1122:3344:103::7",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::7]:32345",
-                "dataset": {
-                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                "dns_address": "198.51.100.21:53",
+                "http_address": "[fd00:1122:3344:102::4]:5353",
+                "nic": {
+                  "id": "d4dd2f7a-8a3b-405e-8444-62f99a754a8a",
+                  "kind": {
+                    "type": "service",
+                    "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802"
+                  },
+                  "name": "external-dns-f0ed9838-6f1b-4646-a478-9f74f2e6e802",
+                  "ip": "172.30.1.6",
+                  "mac": "A8:40:25:FF:F4:5C",
+                  "subnet": "172.30.1.0/24",
+                  "vni": 100,
+                  "primary": true,
+                  "slot": 0
                 }
               }
-            },
+            }
+          ]
+        },
+        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
+          "generation": 5,
+          "zones": [
             {
-              "id": "54c947d2-6355-453c-80fc-8f49cc2129ee",
-              "underlay_address": "fd00:1122:3344:103::5",
+              "id": "0d459323-e414-4f32-9944-76c7331da622",
+              "underlay_address": "fd00:1122:3344:101::5",
               "zone_type": {
                 "type": "nexus",
                 "external_dns_servers": [
                   "1.1.1.1",
                   "9.9.9.9"
                 ],
-                "external_ip": "172.20.28.4",
-                "external_tls": true,
-                "internal_address": "[fd00:1122:3344:103::5]:12221",
+                "external_ip": "198.51.100.24",
+                "external_tls": false,
+                "internal_address": "[fd00:1122:3344:101::5]:12221",
                 "nic": {
-                  "id": "1f55dc2b-18ee-485c-8831-b603a8df78e6",
-                  "ip": "172.30.2.7",
+                  "id": "514902b8-7a1a-42a3-95a6-d00861833b5c",
                   "kind": {
                     "type": "service",
-                    "id": "54c947d2-6355-453c-80fc-8f49cc2129ee"
+                    "id": "0d459323-e414-4f32-9944-76c7331da622"
                   },
-                  "mac": "A8:40:25:FF:96:05",
-                  "name": "nexus-54c947d2-6355-453c-80fc-8f49cc2129ee",
-                  "primary": true,
-                  "slot": 0,
+                  "name": "nexus-0d459323-e414-4f32-9944-76c7331da622",
+                  "ip": "172.30.2.7",
+                  "mac": "A8:40:25:FF:DE:AC",
                   "subnet": "172.30.2.0/24",
-                  "vni": 100
+                  "vni": 100,
+                  "primary": true,
+                  "slot": 0
                 }
               }
             },
             {
-              "id": "60d64b70-89f5-4774-9e41-2c79c4173911",
-              "underlay_address": "fd00:1122:3344:103::6",
-              "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:103::6]:17000"
-              }
-            },
-            {
-              "id": "921958c0-7761-4294-945d-2a078327bf2c",
-              "underlay_address": "fd00:1122:3344:103::8",
+              "id": "2902972b-3ae4-489d-b937-6480352e214a",
+              "underlay_address": "fd00:1122:3344:101::8",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:103::8]:32345",
+                "address": "[fd00:1122:3344:101::8]:32345",
                 "dataset": {
-                  "pool_name": "oxp_7eabbdcc-28a3-4af9-bc4a-8d9b745d4df3"
+                  "pool_name": "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "9df22034-59f6-431f-82e7-258fe983879b",
-              "underlay_address": "fd00:1122:3344:103::4",
+              "id": "2ee832de-af75-47ac-abac-f91304857ca4",
+              "underlay_address": "fd00:1122:3344:101::4",
               "zone_type": {
                 "type": "external_dns",
                 "dataset": {
-                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                 },
-                "dns_address": "172.20.28.1:53",
-                "http_address": "[fd00:1122:3344:103::4]:5353",
+                "dns_address": "198.51.100.20:53",
+                "http_address": "[fd00:1122:3344:101::4]:5353",
                 "nic": {
-                  "id": "0376a963-e3c1-44bc-9cbf-9e8cca58410b",
-                  "ip": "172.30.1.5",
+                  "id": "fadff4ab-62d7-466a-9203-04080e4078ed",
                   "kind": {
                     "type": "service",
-                    "id": "9df22034-59f6-431f-82e7-258fe983879b"
+                    "id": "2ee832de-af75-47ac-abac-f91304857ca4"
                   },
-                  "mac": "A8:40:25:FF:90:12",
-                  "name": "external-dns-9df22034-59f6-431f-82e7-258fe983879b",
-                  "primary": true,
-                  "slot": 0,
+                  "name": "external-dns-2ee832de-af75-47ac-abac-f91304857ca4",
+                  "ip": "172.30.1.5",
+                  "mac": "A8:40:25:FF:B4:93",
                   "subnet": "172.30.1.0/24",
-                  "vni": 100
+                  "vni": 100,
+                  "primary": true,
+                  "slot": 0
                 }
               }
             },
             {
-              "id": "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
-              "underlay_address": "fd00:1122:3344:103::11",
+              "id": "30050a7e-7fc3-4965-b07b-1635212393eb",
+              "underlay_address": "fd00:1122:3344:101::6",
               "zone_type": {
-                "type": "internal_ntp",
-                "address": "[fd00:1122:3344:103::11]:123",
-                "dns_servers": [
-                  "fd00:1122:3344:1::1",
-                  "fd00:1122:3344:2::1",
-                  "fd00:1122:3344:3::1"
-                ],
-                "ntp_servers": [
-                  "d630cca7-e3f7-47a8-aee8-ad0790895abc.host.control-plane.oxide.internal",
-                  "c74ba7d7-3d5f-4a29-aadc-effe33a90909.host.control-plane.oxide.internal"
-                ]
+                "type": "crucible_pantry",
+                "address": "[fd00:1122:3344:101::6]:17000"
               }
             },
             {
-              "id": "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
-              "underlay_address": "fd00:1122:3344:103::9",
+              "id": "8494befe-6df6-42d1-9baa-599ba4228430",
+              "underlay_address": "fd00:1122:3344:101::c",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_c8feed09-6556-4409-af8e-563c97d556eb"
+                "type": "boundary_ntp",
+                "address": "[fd00:1122:3344:101::c]:123",
+                "dns_servers": [
+                  "1.1.1.1",
+                  "9.9.9.9"
+                ],
+                "nic": {
+                  "id": "23041748-e3a6-4846-930d-5a348c316e91",
+                  "kind": {
+                    "type": "service",
+                    "id": "8494befe-6df6-42d1-9baa-599ba4228430"
+                  },
+                  "name": "ntp-8494befe-6df6-42d1-9baa-599ba4228430",
+                  "ip": "172.30.3.5",
+                  "mac": "A8:40:25:FF:D1:C4",
+                  "subnet": "172.30.3.0/24",
+                  "vni": 100,
+                  "primary": true,
+                  "slot": 0
+                },
+                "ntp_servers": [
+                  "time.cloudflare.com"
+                ],
+                "snat_cfg": {
+                  "ip": "198.51.100.25",
+                  "first_port": 0,
+                  "last_port": 16383
                 }
               }
             },
             {
-              "id": "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
-              "underlay_address": "fd00:1122:3344:103::d",
+              "id": "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
+              "underlay_address": "fd00:1122:3344:101::7",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:103::d]:32345",
+                "address": "[fd00:1122:3344:101::7]:32345",
                 "dataset": {
-                  "pool_name": "oxp_5b8b10a4-d527-4176-a049-1e9c2a10edea"
+                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "d42fc874-8416-4212-b5a0-febba4c6b0ac",
-              "underlay_address": "fd00:1122:3344:103::b",
+              "id": "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
+              "underlay_address": "fd00:1122:3344:101::3",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::b]:32345",
+                "type": "cockroach_db",
+                "address": "[fd00:1122:3344:101::3]:32221",
                 "dataset": {
-                  "pool_name": "oxp_83a60822-1299-471c-bc79-3be0f87cf7c3"
+                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
-              "underlay_address": "fd00:1122:3344:103::e",
+              "id": "e668e3f5-5457-4689-97da-b2af537787ac",
+              "underlay_address": "fd00:1122:3344:101::b",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:103::e]:32345",
+                "address": "[fd00:1122:3344:101::b]:32345",
                 "dataset": {
-                  "pool_name": "oxp_4ffc6336-04d1-4f21-9c27-a95aa640f960"
+                  "pool_name": "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             },
             {
-              "id": "da27ee6a-9841-46aa-acda-7cce3e04cb60",
-              "underlay_address": "fd00:1122:3344:103::3",
+              "id": "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
+              "underlay_address": "fd00:1122:3344:101::9",
               "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:103::3]:32221",
+                "type": "crucible",
+                "address": "[fd00:1122:3344:101::9]:32345",
                 "dataset": {
-                  "pool_name": "oxp_d4e08191-c3c4-43f7-9818-1cb1f1e1285a"
+                  "pool_name": "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b"
                 }
               }
             },
             {
-              "id": "e09525cf-9c87-4d05-8860-1356dd0dee8a",
-              "underlay_address": "fd00:1122:3344:103::f",
+              "id": "f8c79ba4-194a-4cfe-8090-5315e297b780",
+              "underlay_address": "fd00:1122:3344:1::1",
               "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::f]:32345",
+                "type": "internal_dns",
                 "dataset": {
-                  "pool_name": "oxp_e2244950-dda0-4dfa-8b63-5ffac16c5a62"
-                }
+                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
+                },
+                "dns_address": "[fd00:1122:3344:1::1]:53",
+                "gz_address": "fd00:1122:3344:1::2",
+                "gz_address_index": 0,
+                "http_address": "[fd00:1122:3344:1::1]:5353"
               }
             },
             {
-              "id": "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
-              "underlay_address": "fd00:1122:3344:103::c",
+              "id": "fa5e0176-b4a8-4598-aa31-40fd562d246c",
+              "underlay_address": "fd00:1122:3344:101::a",
               "zone_type": {
                 "type": "crucible",
-                "address": "[fd00:1122:3344:103::c]:32345",
+                "address": "[fd00:1122:3344:101::a]:32345",
                 "dataset": {
-                  "pool_name": "oxp_f6813b0c-ab5a-4fd0-8e24-95de9b65280d"
+                  "pool_name": "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03"
                 }
               }
             }
@@ -11489,70 +8600,54 @@
         }
       },
       "zones_in_service": [
-        "0706860a-3801-42a7-8000-fff741f7665b",
-        "0ebd2b5b-2dec-4d17-8e23-e468c04dacb6",
-        "0eca39c6-62e2-4a1f-8ea2-e9332a774a26",
-        "12431d3e-1516-4822-98b3-3711ff912e69",
-        "185e6b00-7e18-456c-9dfc-edd0194ac207",
-        "18b99404-d8b2-4463-bc99-638d9b678ab3",
-        "1fc78bc7-feca-4a10-9beb-58b53fda5210",
-        "2d5c29f7-4915-4876-82f8-e388b15c5067",
-        "34a279a2-a025-41ff-a392-1addac400f38",
-        "3d344935-6f48-405f-8794-2c6e8fa4b1c2",
-        "45dccfd6-4674-43c6-a0e7-4dde5f24d90d",
-        "479811e7-6409-45d4-b294-5333534e47ed",
-        "4b74c418-0a08-4489-b652-e97267c1c220",
-        "54c947d2-6355-453c-80fc-8f49cc2129ee",
-        "54d1b693-bc4d-4c47-9009-c6f624073801",
-        "56414d76-9b8c-4e15-acf1-cd269c2d8638",
-        "59c3ed4f-3670-44b1-9ea0-9a6162a08629",
-        "5b743072-9369-45d7-b7c9-dbdf736301f8",
-        "5c4163fa-d6a6-4a0c-9476-3dda135bc9a4",
-        "60d64b70-89f5-4774-9e41-2c79c4173911",
-        "61a39188-dd82-43f6-a87c-ebb4c2d54ac4",
-        "658076ed-fb43-49e7-a813-df3d23497848",
-        "67852e14-ce7f-4e50-a61d-4059d0c057b9",
-        "6ace1a69-d55a-45f0-a127-99c90f1b070c",
-        "6edb836e-b100-4080-bb03-738d6441d571",
-        "7f83d784-9c5c-498f-b559-4176bbafbc4b",
-        "8052938f-73bc-4c54-8ed0-c62f6d77b9fd",
-        "84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe",
-        "854c1544-d0ea-4d65-9cf4-776e91a4fcb5",
-        "86bc0a2a-e24a-4bc1-8222-89207a0937f9",
-        "86ee7359-aa35-41d9-8a76-54961bc516a1",
-        "891cb9da-301d-40cc-b93d-5fcb350fa968",
-        "921958c0-7761-4294-945d-2a078327bf2c",
-        "9241b9da-8188-4aae-a9dc-62acb453618f",
-        "925ea436-fad4-4fff-8ae0-24edd725a2b0",
-        "9df22034-59f6-431f-82e7-258fe983879b",
-        "a781a349-62b2-41f0-9dd7-c64d290aef82",
-        "bb21c0c4-c43f-4913-9bf9-707bd7f1769a",
-        "be272d1e-4d34-4478-94b4-a76f5fcbef36",
-        "bebd754d-d06c-45f4-99bf-e1aab6253ab8",
-        "c67814bf-9972-4bb0-a666-24778ca70a16",
-        "c74ba7d7-3d5f-4a29-aadc-effe33a90909",
-        "ccb0d612-9d7c-41ba-bd51-661c3c75ec91",
-        "ccef3e95-cae5-4072-9b7d-49e56fa9dff4",
-        "d42fc874-8416-4212-b5a0-febba4c6b0ac",
-        "d5ba7d45-26d7-45ca-8c0a-a560f243b5fa",
-        "d630cca7-e3f7-47a8-aee8-ad0790895abc",
-        "d8c51c79-46ff-44bd-80be-08d6283f4e21",
-        "d9014907-24c0-4c33-a475-db15942f4f8f",
-        "da27ee6a-9841-46aa-acda-7cce3e04cb60",
-        "dbbc0422-2b23-4d9e-b1db-8003cfc29ad0",
-        "dfc27183-ed85-44c3-ac3c-4427c4cb03e3",
-        "e09525cf-9c87-4d05-8860-1356dd0dee8a",
-        "e950de25-7306-4cf7-a6f4-6db11adaa310",
-        "f33a2375-212d-42e4-b539-6079aeb4e5b7",
-        "f7264a3e-e8e7-4476-8df3-76c9d00a383b",
-        "fe3dc092-ea23-42d7-9f1d-36eed72b539c",
-        "ff8cce49-a721-4ad1-b09e-2bd6e4e42d00"
+        "084cac6e-44b5-4333-a2ff-b0993b534898",
+        "0d459323-e414-4f32-9944-76c7331da622",
+        "0db88baf-911d-47bb-af71-3aebd7c96934",
+        "11c527c5-0598-428e-9270-be5b74d3461b",
+        "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
+        "20be9299-495d-428d-b315-694c2c4b64af",
+        "213d7dd3-95a4-45a7-99ca-e4899991dab4",
+        "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
+        "2902972b-3ae4-489d-b937-6480352e214a",
+        "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
+        "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
+        "2ee832de-af75-47ac-abac-f91304857ca4",
+        "30050a7e-7fc3-4965-b07b-1635212393eb",
+        "40c47ead-3be8-4724-a8ef-b66233fc5c65",
+        "425d9d2f-72ea-4f53-80b6-04e002771cdb",
+        "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
+        "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
+        "57c712a6-6429-44b7-a9b2-9957cb9054d4",
+        "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
+        "73916098-788f-48d8-b745-e13fdcfb771b",
+        "7611a549-effc-41b6-b6cb-c03e0c70a31f",
+        "8494befe-6df6-42d1-9baa-599ba4228430",
+        "8d452bdd-e8b8-4108-a690-46fc59197db0",
+        "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
+        "953019c0-8e85-4417-b231-d42820470125",
+        "99c635b9-8e97-414a-8316-8261d2d767d0",
+        "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
+        "9bb513e3-7505-416c-aa70-b26f1152bbdc",
+        "9c14c879-137b-4f5b-8068-1292bffc83ed",
+        "9cf90589-5da1-4b42-b09b-d784e4134637",
+        "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
+        "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
+        "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
+        "d48b2dfa-c749-47dd-98b6-3f478cd63934",
+        "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
+        "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
+        "e668e3f5-5457-4689-97da-b2af537787ac",
+        "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
+        "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
+        "f8c79ba4-194a-4cfe-8090-5315e297b780",
+        "fa5e0176-b4a8-4598-aa31-40fd562d246c",
+        "fd003bee-818a-4c95-aae5-e378ac601522"
       ],
-      "parent_blueprint_id": "6c127695-ba15-408d-a992-325a1a888380",
-      "internal_dns_version": 2,
-      "time_created": "2024-03-01T19:08:52.730520Z",
-      "creator": "54c947d2-6355-453c-80fc-8f49cc2129ee",
-      "comment": "sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f: add zones"
+      "parent_blueprint_id": null,
+      "internal_dns_version": 1,
+      "time_created": "2024-03-12T19:11:49.876621Z",
+      "creator": "0d459323-e414-4f32-9944-76c7331da622",
+      "comment": "from collection 5e1d6ae0-c5b4-4884-ac41-680cd4f5762d"
     }
   ]
 }
\ No newline at end of file
diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-complex-stdout b/dev-tools/reconfigurator-cli/tests/output/cmd-complex-stdout
index 0cc5fe01ba..0c8c164d6f 100644
--- a/dev-tools/reconfigurator-cli/tests/output/cmd-complex-stdout
+++ b/dev-tools/reconfigurator-cli/tests/output/cmd-complex-stdout
@@ -14,36 +14,32 @@ ID
 > 
 
 > file-contents tests/input/complex.json
-sled: 1762267a-a0cb-4c9a-88b4-8ce828cbc021 (subnet: fd00:1122:3344:101::/64, zpools: 9)
-sled: a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f (subnet: fd00:1122:3344:121::/64, zpools: 8)
-sled: a6634f64-f6fb-426c-b00b-9b30aed9f53a (subnet: fd00:1122:3344:102::/64, zpools: 10)
-sled: fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89 (subnet: fd00:1122:3344:103::/64, zpools: 10)
-collection: 83ed3949-d221-44f3-a901-02e9ff16d2a5 (errors: 8, completed at: 2024-03-01T19:17:10.594Z)
-collection: 0a8dc8ff-9013-43aa-9f06-9540b24ea902 (errors: 8, completed at: 2024-03-01T19:17:11.153Z)
-collection: cd3547d7-f172-47d2-9bbf-9730e88b0559 (errors: 8, completed at: 2024-03-01T19:18:07.486Z)
-collection: 623f7e48-6c47-4e17-8aa3-cebe98ca4287 (errors: 8, completed at: 2024-03-01T19:18:12.110Z)
-collection: 4e9fbfc2-eb1d-4d45-bdb9-64d965a490c5 (errors: 8, completed at: 2024-03-01T19:18:13.548Z)
-blueprint:  486de160-c8f3-4600-acca-b0c78e33aca4 (created at: 2024-03-01 19:06:56.467313 UTC)
-blueprint:  6c127695-ba15-408d-a992-325a1a888380 (created at: 2024-03-01 19:07:58.105708 UTC)
-blueprint:  cfa03594-f438-4736-a416-6307f7bf8f2e (created at: 2024-03-01 19:08:52.730520 UTC)
+sled: 1f34b5cc-6dac-40cc-9ea1-95f32c33e59b (subnet: fd00:1122:3344:103::/64, zpools: 5)
+sled: 67a1268b-52d3-4159-b3ed-7a0777f1f340 (subnet: fd00:1122:3344:104::/64, zpools: 5)
+sled: 77851655-10a5-4e80-88b0-a001473cab7e (subnet: fd00:1122:3344:102::/64, zpools: 5)
+sled: f122b3a2-e383-4e42-b2a5-74a69a9d110a (subnet: fd00:1122:3344:101::/64, zpools: 5)
+collection: 3f61b723-cfe7-40ec-b22c-e3e1f35325f9 (errors: 0, completed at: 2024-03-12T18:59:34.397Z)
+collection: 37c212f1-bfdd-4cfd-a17c-e65bd93bea5f (errors: 0, completed at: 2024-03-12T19:07:28.472Z)
+collection: 3700dcd4-64bb-4997-bfdb-dcfc7cbbc998 (errors: 0, completed at: 2024-03-12T19:07:28.728Z)
+collection: 5e1d6ae0-c5b4-4884-ac41-680cd4f5762d (errors: 0, completed at: 2024-03-12T19:09:37.116Z)
+blueprint:  649e444d-d00f-4067-a915-12e15647bf37 (created at: 2024-03-12 19:09:41.953854 UTC)
+blueprint:  cd34e87b-d821-4768-b32f-f41c28db835e (created at: 2024-03-12 19:11:49.876621 UTC)
 
 
 > 
 
-> load tests/input/complex.json 83ed3949-d221-44f3-a901-02e9ff16d2a5
-using collection 83ed3949-d221-44f3-a901-02e9ff16d2a5 as source of sled inventory data
-sled 1762267a-a0cb-4c9a-88b4-8ce828cbc021 loaded
-sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f loaded
-sled a6634f64-f6fb-426c-b00b-9b30aed9f53a loaded
-sled fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89 loaded
-collection 83ed3949-d221-44f3-a901-02e9ff16d2a5 loaded
-collection 0a8dc8ff-9013-43aa-9f06-9540b24ea902 loaded
-collection cd3547d7-f172-47d2-9bbf-9730e88b0559 loaded
-collection 623f7e48-6c47-4e17-8aa3-cebe98ca4287 loaded
-collection 4e9fbfc2-eb1d-4d45-bdb9-64d965a490c5 loaded
-blueprint 486de160-c8f3-4600-acca-b0c78e33aca4 loaded
-blueprint 6c127695-ba15-408d-a992-325a1a888380 loaded
-blueprint cfa03594-f438-4736-a416-6307f7bf8f2e loaded
+> load tests/input/complex.json 3f61b723-cfe7-40ec-b22c-e3e1f35325f9
+using collection 3f61b723-cfe7-40ec-b22c-e3e1f35325f9 as source of sled inventory data
+sled 1f34b5cc-6dac-40cc-9ea1-95f32c33e59b loaded
+sled 67a1268b-52d3-4159-b3ed-7a0777f1f340 loaded
+sled 77851655-10a5-4e80-88b0-a001473cab7e loaded
+sled f122b3a2-e383-4e42-b2a5-74a69a9d110a loaded
+collection 3f61b723-cfe7-40ec-b22c-e3e1f35325f9 loaded
+collection 37c212f1-bfdd-4cfd-a17c-e65bd93bea5f loaded
+collection 3700dcd4-64bb-4997-bfdb-dcfc7cbbc998 loaded
+collection 5e1d6ae0-c5b4-4884-ac41-680cd4f5762d loaded
+blueprint 649e444d-d00f-4067-a915-12e15647bf37 loaded
+blueprint cd34e87b-d821-4768-b32f-f41c28db835e loaded
 loaded data from "tests/input/complex.json"
 
 
@@ -51,326 +47,258 @@ loaded data from "tests/input/complex.json"
 
 > sled-list
 ID                                   NZPOOLS SUBNET                  
-1762267a-a0cb-4c9a-88b4-8ce828cbc021 9       fd00:1122:3344:101::/64 
-a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f 8       fd00:1122:3344:121::/64 
-a6634f64-f6fb-426c-b00b-9b30aed9f53a 10      fd00:1122:3344:102::/64 
-fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89 10      fd00:1122:3344:103::/64 
+1f34b5cc-6dac-40cc-9ea1-95f32c33e59b 5       fd00:1122:3344:103::/64 
+67a1268b-52d3-4159-b3ed-7a0777f1f340 5       fd00:1122:3344:104::/64 
+77851655-10a5-4e80-88b0-a001473cab7e 5       fd00:1122:3344:102::/64 
+f122b3a2-e383-4e42-b2a5-74a69a9d110a 5       fd00:1122:3344:101::/64 
 
 > 
 
 > inventory-list
 ID                                   NERRORS TIME_DONE                
-83ed3949-d221-44f3-a901-02e9ff16d2a5 8       2024-03-01T19:17:10.594Z 
-0a8dc8ff-9013-43aa-9f06-9540b24ea902 8       2024-03-01T19:17:11.153Z 
-cd3547d7-f172-47d2-9bbf-9730e88b0559 8       2024-03-01T19:18:07.486Z 
-623f7e48-6c47-4e17-8aa3-cebe98ca4287 8       2024-03-01T19:18:12.110Z 
-4e9fbfc2-eb1d-4d45-bdb9-64d965a490c5 8       2024-03-01T19:18:13.548Z 
+3f61b723-cfe7-40ec-b22c-e3e1f35325f9 0       2024-03-12T18:59:34.397Z 
+37c212f1-bfdd-4cfd-a17c-e65bd93bea5f 0       2024-03-12T19:07:28.472Z 
+3700dcd4-64bb-4997-bfdb-dcfc7cbbc998 0       2024-03-12T19:07:28.728Z 
+5e1d6ae0-c5b4-4884-ac41-680cd4f5762d 0       2024-03-12T19:09:37.116Z 
 
 > 
 
 > blueprint-list
 ID                                   
-486de160-c8f3-4600-acca-b0c78e33aca4 
-6c127695-ba15-408d-a992-325a1a888380 
-cfa03594-f438-4736-a416-6307f7bf8f2e 
+649e444d-d00f-4067-a915-12e15647bf37 
+cd34e87b-d821-4768-b32f-f41c28db835e 
 
 > 
 
-> blueprint-show cfa03594-f438-4736-a416-6307f7bf8f2e
-blueprint  cfa03594-f438-4736-a416-6307f7bf8f2e
-parent:    6c127695-ba15-408d-a992-325a1a888380
-created by 54c947d2-6355-453c-80fc-8f49cc2129ee (likely a Nexus instance)
-created at 2024-03-01T19:08:52.730Z
-internal DNS version: 2
-comment: sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f: add zones
+> blueprint-show cd34e87b-d821-4768-b32f-f41c28db835e
+blueprint  cd34e87b-d821-4768-b32f-f41c28db835e
+parent:    <none>
+created by 0d459323-e414-4f32-9944-76c7331da622 (likely a Nexus instance)
+created at 2024-03-12T19:11:49.876Z
+internal DNS version: 1
+comment: from collection 5e1d6ae0-c5b4-4884-ac41-680cd4f5762d
 zones:
 
-  sled 1762267a-a0cb-4c9a-88b4-8ce828cbc021: Omicron zones at generation 5
-    0706860a-3801-42a7-8000-fff741f7665b in service     crucible
-    34a279a2-a025-41ff-a392-1addac400f38 in service     cockroach_db
-    3d344935-6f48-405f-8794-2c6e8fa4b1c2 in service     internal_dns
-    59c3ed4f-3670-44b1-9ea0-9a6162a08629 in service     nexus
-    67852e14-ce7f-4e50-a61d-4059d0c057b9 in service     crucible
-    6ace1a69-d55a-45f0-a127-99c90f1b070c in service     crucible
-    84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe in service     crucible
-    854c1544-d0ea-4d65-9cf4-776e91a4fcb5 in service     crucible
-    86ee7359-aa35-41d9-8a76-54961bc516a1 in service     cockroach_db
-    925ea436-fad4-4fff-8ae0-24edd725a2b0 in service     crucible
-    be272d1e-4d34-4478-94b4-a76f5fcbef36 in service     crucible
-    bebd754d-d06c-45f4-99bf-e1aab6253ab8 in service     oximeter
-    d630cca7-e3f7-47a8-aee8-ad0790895abc in service     boundary_ntp
-    d8c51c79-46ff-44bd-80be-08d6283f4e21 in service     crucible
-    dbbc0422-2b23-4d9e-b1db-8003cfc29ad0 in service     crucible
-    f33a2375-212d-42e4-b539-6079aeb4e5b7 in service     crucible_pantry
-  sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f: Omicron zones at generation 3
-    0eca39c6-62e2-4a1f-8ea2-e9332a774a26 in service     crucible
-    12431d3e-1516-4822-98b3-3711ff912e69 in service     crucible
-    2d5c29f7-4915-4876-82f8-e388b15c5067 in service     internal_ntp
-    56414d76-9b8c-4e15-acf1-cd269c2d8638 in service     crucible
-    891cb9da-301d-40cc-b93d-5fcb350fa968 in service     crucible
-    d9014907-24c0-4c33-a475-db15942f4f8f in service     crucible
-    e950de25-7306-4cf7-a6f4-6db11adaa310 in service     crucible
-    fe3dc092-ea23-42d7-9f1d-36eed72b539c in service     crucible
-    ff8cce49-a721-4ad1-b09e-2bd6e4e42d00 in service     crucible
-  sled a6634f64-f6fb-426c-b00b-9b30aed9f53a: Omicron zones at generation 5
-    0ebd2b5b-2dec-4d17-8e23-e468c04dacb6 in service     crucible
-    18b99404-d8b2-4463-bc99-638d9b678ab3 in service     crucible
-    479811e7-6409-45d4-b294-5333534e47ed in service     crucible
-    54d1b693-bc4d-4c47-9009-c6f624073801 in service     crucible
-    5b743072-9369-45d7-b7c9-dbdf736301f8 in service     internal_dns
-    5c4163fa-d6a6-4a0c-9476-3dda135bc9a4 in service     crucible_pantry
-    61a39188-dd82-43f6-a87c-ebb4c2d54ac4 in service     crucible
-    658076ed-fb43-49e7-a813-df3d23497848 in service     crucible
-    6edb836e-b100-4080-bb03-738d6441d571 in service     crucible
-    7f83d784-9c5c-498f-b559-4176bbafbc4b in service     nexus
-    8052938f-73bc-4c54-8ed0-c62f6d77b9fd in service     crucible
-    86bc0a2a-e24a-4bc1-8222-89207a0937f9 in service     cockroach_db
-    9241b9da-8188-4aae-a9dc-62acb453618f in service     clickhouse
-    a781a349-62b2-41f0-9dd7-c64d290aef82 in service     crucible
-    c67814bf-9972-4bb0-a666-24778ca70a16 in service     cockroach_db
-    c74ba7d7-3d5f-4a29-aadc-effe33a90909 in service     boundary_ntp
-    dfc27183-ed85-44c3-ac3c-4427c4cb03e3 in service     crucible
-  sled fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89: Omicron zones at generation 5
-    185e6b00-7e18-456c-9dfc-edd0194ac207 in service     crucible
-    1fc78bc7-feca-4a10-9beb-58b53fda5210 in service     crucible
-    45dccfd6-4674-43c6-a0e7-4dde5f24d90d in service     internal_dns
-    4b74c418-0a08-4489-b652-e97267c1c220 in service     crucible
-    54c947d2-6355-453c-80fc-8f49cc2129ee in service     nexus
-    60d64b70-89f5-4774-9e41-2c79c4173911 in service     crucible_pantry
-    921958c0-7761-4294-945d-2a078327bf2c in service     crucible
-    9df22034-59f6-431f-82e7-258fe983879b in service     external_dns
-    bb21c0c4-c43f-4913-9bf9-707bd7f1769a in service     internal_ntp
-    ccb0d612-9d7c-41ba-bd51-661c3c75ec91 in service     crucible
-    ccef3e95-cae5-4072-9b7d-49e56fa9dff4 in service     crucible
-    d42fc874-8416-4212-b5a0-febba4c6b0ac in service     crucible
-    d5ba7d45-26d7-45ca-8c0a-a560f243b5fa in service     crucible
-    da27ee6a-9841-46aa-acda-7cce3e04cb60 in service     cockroach_db
-    e09525cf-9c87-4d05-8860-1356dd0dee8a in service     crucible
-    f7264a3e-e8e7-4476-8df3-76c9d00a383b in service     crucible
+  sled 1f34b5cc-6dac-40cc-9ea1-95f32c33e59b: Omicron zones at generation 5
+    0db88baf-911d-47bb-af71-3aebd7c96934 in service     nexus
+    213d7dd3-95a4-45a7-99ca-e4899991dab4 in service     crucible
+    218f36a5-9e51-4b88-ad1a-da4d85c246c4 in service     internal_dns
+    40c47ead-3be8-4724-a8ef-b66233fc5c65 in service     crucible
+    425d9d2f-72ea-4f53-80b6-04e002771cdb in service     crucible
+    46a050b6-37b8-47b9-b4ae-ccb736ecd6d5 in service     crucible
+    4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd in service     clickhouse
+    7611a549-effc-41b6-b6cb-c03e0c70a31f in service     internal_ntp
+    af7ccead-2bd8-4a8a-9fbf-3b788d180139 in service     cockroach_db
+    d4428e41-ad7b-41f0-bde6-7c7c955f20c1 in service     crucible
+  sled 67a1268b-52d3-4159-b3ed-7a0777f1f340: Omicron zones at generation 5
+    1ebea0c3-1bfd-472c-9f84-ac9e23b32707 in service     crucible
+    2e9dfa8f-5371-48e5-b7f7-62a6af8283c1 in service     cockroach_db
+    57c712a6-6429-44b7-a9b2-9957cb9054d4 in service     crucible
+    59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9 in service     crucible
+    73916098-788f-48d8-b745-e13fdcfb771b in service     internal_ntp
+    9bb513e3-7505-416c-aa70-b26f1152bbdc in service     crucible
+    9c14c879-137b-4f5b-8068-1292bffc83ed in service     cockroach_db
+    9cf90589-5da1-4b42-b09b-d784e4134637 in service     nexus
+    d48b2dfa-c749-47dd-98b6-3f478cd63934 in service     crucible_pantry
+    fd003bee-818a-4c95-aae5-e378ac601522 in service     crucible
+  sled 77851655-10a5-4e80-88b0-a001473cab7e: Omicron zones at generation 5
+    084cac6e-44b5-4333-a2ff-b0993b534898 in service     crucible
+    11c527c5-0598-428e-9270-be5b74d3461b in service     cockroach_db
+    20be9299-495d-428d-b315-694c2c4b64af in service     crucible_pantry
+    2b29cb95-f8c6-4aff-acc0-1c0a627be392 in service     crucible
+    8d452bdd-e8b8-4108-a690-46fc59197db0 in service     crucible
+    953019c0-8e85-4417-b231-d42820470125 in service     oximeter
+    99c635b9-8e97-414a-8316-8261d2d767d0 in service     crucible
+    cbe1abfa-0063-44b2-9ce7-7ad76b403a94 in service     internal_dns
+    d52e797a-8fff-4e1e-a5ff-0ab8c8293ded in service     boundary_ntp
+    dc7f703a-27d1-4802-b85d-5629ddb2d67c in service     crucible
+    f0ed9838-6f1b-4646-a478-9f74f2e6e802 in service     external_dns
+  sled f122b3a2-e383-4e42-b2a5-74a69a9d110a: Omicron zones at generation 5
+    0d459323-e414-4f32-9944-76c7331da622 in service     nexus
+    2902972b-3ae4-489d-b937-6480352e214a in service     crucible
+    2ee832de-af75-47ac-abac-f91304857ca4 in service     external_dns
+    30050a7e-7fc3-4965-b07b-1635212393eb in service     crucible_pantry
+    8494befe-6df6-42d1-9baa-599ba4228430 in service     boundary_ntp
+    92cba955-3a5a-4d82-b0f2-30e89d4556d1 in service     crucible
+    9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b in service     cockroach_db
+    e668e3f5-5457-4689-97da-b2af537787ac in service     crucible
+    f32fd972-7f40-4c58-9e00-7fc5c44273f4 in service     crucible
+    f8c79ba4-194a-4cfe-8090-5315e297b780 in service     internal_dns
+    fa5e0176-b4a8-4598-aa31-40fd562d246c in service     crucible
 
 
 > 
 
-> blueprint-diff-inventory 83ed3949-d221-44f3-a901-02e9ff16d2a5 cfa03594-f438-4736-a416-6307f7bf8f2e
-diff collection 83ed3949-d221-44f3-a901-02e9ff16d2a5 blueprint cfa03594-f438-4736-a416-6307f7bf8f2e
---- collection 83ed3949-d221-44f3-a901-02e9ff16d2a5
-+++ blueprint cfa03594-f438-4736-a416-6307f7bf8f2e
-  sled 1762267a-a0cb-4c9a-88b4-8ce828cbc021
+> blueprint-diff-inventory 3f61b723-cfe7-40ec-b22c-e3e1f35325f9 cd34e87b-d821-4768-b32f-f41c28db835e
+diff collection 3f61b723-cfe7-40ec-b22c-e3e1f35325f9 blueprint cd34e87b-d821-4768-b32f-f41c28db835e
+--- collection 3f61b723-cfe7-40ec-b22c-e3e1f35325f9
++++ blueprint cd34e87b-d821-4768-b32f-f41c28db835e
+  sled 1f34b5cc-6dac-40cc-9ea1-95f32c33e59b
       zone config generation 5
-         zone 0706860a-3801-42a7-8000-fff741f7665b type crucible underlay IP fd00:1122:3344:101::c (unchanged)
-         zone 34a279a2-a025-41ff-a392-1addac400f38 type cockroach_db underlay IP fd00:1122:3344:101::3 (unchanged)
-         zone 3d344935-6f48-405f-8794-2c6e8fa4b1c2 type internal_dns underlay IP fd00:1122:3344:1::1 (unchanged)
-         zone 59c3ed4f-3670-44b1-9ea0-9a6162a08629 type nexus underlay IP fd00:1122:3344:101::5 (unchanged)
-         zone 67852e14-ce7f-4e50-a61d-4059d0c057b9 type crucible underlay IP fd00:1122:3344:101::10 (unchanged)
-         zone 6ace1a69-d55a-45f0-a127-99c90f1b070c type crucible underlay IP fd00:1122:3344:101::f (unchanged)
-         zone 84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe type crucible underlay IP fd00:1122:3344:101::9 (unchanged)
-         zone 854c1544-d0ea-4d65-9cf4-776e91a4fcb5 type crucible underlay IP fd00:1122:3344:101::8 (unchanged)
-         zone 86ee7359-aa35-41d9-8a76-54961bc516a1 type cockroach_db underlay IP fd00:1122:3344:101::4 (unchanged)
-         zone 925ea436-fad4-4fff-8ae0-24edd725a2b0 type crucible underlay IP fd00:1122:3344:101::d (unchanged)
-         zone be272d1e-4d34-4478-94b4-a76f5fcbef36 type crucible underlay IP fd00:1122:3344:101::a (unchanged)
-         zone bebd754d-d06c-45f4-99bf-e1aab6253ab8 type oximeter underlay IP fd00:1122:3344:101::6 (unchanged)
-         zone d630cca7-e3f7-47a8-aee8-ad0790895abc type boundary_ntp underlay IP fd00:1122:3344:101::11 (unchanged)
-         zone d8c51c79-46ff-44bd-80be-08d6283f4e21 type crucible underlay IP fd00:1122:3344:101::e (unchanged)
-         zone dbbc0422-2b23-4d9e-b1db-8003cfc29ad0 type crucible underlay IP fd00:1122:3344:101::b (unchanged)
-         zone f33a2375-212d-42e4-b539-6079aeb4e5b7 type crucible_pantry underlay IP fd00:1122:3344:101::7 (unchanged)
-  sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f
-      zone config generation 3
-         zone 0eca39c6-62e2-4a1f-8ea2-e9332a774a26 type crucible underlay IP fd00:1122:3344:121::28 (unchanged)
-         zone 12431d3e-1516-4822-98b3-3711ff912e69 type crucible underlay IP fd00:1122:3344:121::22 (unchanged)
-         zone 2d5c29f7-4915-4876-82f8-e388b15c5067 type internal_ntp underlay IP fd00:1122:3344:121::21 (unchanged)
-         zone 56414d76-9b8c-4e15-acf1-cd269c2d8638 type crucible underlay IP fd00:1122:3344:121::25 (unchanged)
-         zone 891cb9da-301d-40cc-b93d-5fcb350fa968 type crucible underlay IP fd00:1122:3344:121::29 (unchanged)
-         zone d9014907-24c0-4c33-a475-db15942f4f8f type crucible underlay IP fd00:1122:3344:121::23 (unchanged)
-         zone e950de25-7306-4cf7-a6f4-6db11adaa310 type crucible underlay IP fd00:1122:3344:121::27 (unchanged)
-         zone fe3dc092-ea23-42d7-9f1d-36eed72b539c type crucible underlay IP fd00:1122:3344:121::26 (unchanged)
-         zone ff8cce49-a721-4ad1-b09e-2bd6e4e42d00 type crucible underlay IP fd00:1122:3344:121::24 (unchanged)
-  sled a6634f64-f6fb-426c-b00b-9b30aed9f53a
+         zone 0db88baf-911d-47bb-af71-3aebd7c96934 type nexus underlay IP fd00:1122:3344:103::4 (unchanged)
+         zone 213d7dd3-95a4-45a7-99ca-e4899991dab4 type crucible underlay IP fd00:1122:3344:103::6 (unchanged)
+         zone 218f36a5-9e51-4b88-ad1a-da4d85c246c4 type internal_dns underlay IP fd00:1122:3344:3::1 (unchanged)
+         zone 40c47ead-3be8-4724-a8ef-b66233fc5c65 type crucible underlay IP fd00:1122:3344:103::7 (unchanged)
+         zone 425d9d2f-72ea-4f53-80b6-04e002771cdb type crucible underlay IP fd00:1122:3344:103::9 (unchanged)
+         zone 46a050b6-37b8-47b9-b4ae-ccb736ecd6d5 type crucible underlay IP fd00:1122:3344:103::8 (unchanged)
+         zone 4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd type clickhouse underlay IP fd00:1122:3344:103::5 (unchanged)
+         zone 7611a549-effc-41b6-b6cb-c03e0c70a31f type internal_ntp underlay IP fd00:1122:3344:103::b (unchanged)
+         zone af7ccead-2bd8-4a8a-9fbf-3b788d180139 type cockroach_db underlay IP fd00:1122:3344:103::3 (unchanged)
+         zone d4428e41-ad7b-41f0-bde6-7c7c955f20c1 type crucible underlay IP fd00:1122:3344:103::a (unchanged)
+  sled 67a1268b-52d3-4159-b3ed-7a0777f1f340
       zone config generation 5
-         zone 0ebd2b5b-2dec-4d17-8e23-e468c04dacb6 type crucible underlay IP fd00:1122:3344:102::11 (unchanged)
-         zone 18b99404-d8b2-4463-bc99-638d9b678ab3 type crucible underlay IP fd00:1122:3344:102::e (unchanged)
-         zone 479811e7-6409-45d4-b294-5333534e47ed type crucible underlay IP fd00:1122:3344:102::9 (unchanged)
-         zone 54d1b693-bc4d-4c47-9009-c6f624073801 type crucible underlay IP fd00:1122:3344:102::a (unchanged)
-         zone 5b743072-9369-45d7-b7c9-dbdf736301f8 type internal_dns underlay IP fd00:1122:3344:2::1 (unchanged)
-         zone 5c4163fa-d6a6-4a0c-9476-3dda135bc9a4 type crucible_pantry underlay IP fd00:1122:3344:102::7 (unchanged)
-         zone 61a39188-dd82-43f6-a87c-ebb4c2d54ac4 type crucible underlay IP fd00:1122:3344:102::8 (unchanged)
-         zone 658076ed-fb43-49e7-a813-df3d23497848 type crucible underlay IP fd00:1122:3344:102::10 (unchanged)
-         zone 6edb836e-b100-4080-bb03-738d6441d571 type crucible underlay IP fd00:1122:3344:102::c (unchanged)
-         zone 7f83d784-9c5c-498f-b559-4176bbafbc4b type nexus underlay IP fd00:1122:3344:102::5 (unchanged)
-         zone 8052938f-73bc-4c54-8ed0-c62f6d77b9fd type crucible underlay IP fd00:1122:3344:102::d (unchanged)
-         zone 86bc0a2a-e24a-4bc1-8222-89207a0937f9 type cockroach_db underlay IP fd00:1122:3344:102::4 (unchanged)
-         zone 9241b9da-8188-4aae-a9dc-62acb453618f type clickhouse underlay IP fd00:1122:3344:102::6 (unchanged)
-         zone a781a349-62b2-41f0-9dd7-c64d290aef82 type crucible underlay IP fd00:1122:3344:102::f (unchanged)
-         zone c67814bf-9972-4bb0-a666-24778ca70a16 type cockroach_db underlay IP fd00:1122:3344:102::3 (unchanged)
-         zone c74ba7d7-3d5f-4a29-aadc-effe33a90909 type boundary_ntp underlay IP fd00:1122:3344:102::12 (unchanged)
-         zone dfc27183-ed85-44c3-ac3c-4427c4cb03e3 type crucible underlay IP fd00:1122:3344:102::b (unchanged)
-  sled fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89
+         zone 1ebea0c3-1bfd-472c-9f84-ac9e23b32707 type crucible underlay IP fd00:1122:3344:104::b (unchanged)
+         zone 2e9dfa8f-5371-48e5-b7f7-62a6af8283c1 type cockroach_db underlay IP fd00:1122:3344:104::4 (unchanged)
+         zone 57c712a6-6429-44b7-a9b2-9957cb9054d4 type crucible underlay IP fd00:1122:3344:104::a (unchanged)
+         zone 59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9 type crucible underlay IP fd00:1122:3344:104::7 (unchanged)
+         zone 73916098-788f-48d8-b745-e13fdcfb771b type internal_ntp underlay IP fd00:1122:3344:104::c (unchanged)
+         zone 9bb513e3-7505-416c-aa70-b26f1152bbdc type crucible underlay IP fd00:1122:3344:104::9 (unchanged)
+         zone 9c14c879-137b-4f5b-8068-1292bffc83ed type cockroach_db underlay IP fd00:1122:3344:104::3 (unchanged)
+         zone 9cf90589-5da1-4b42-b09b-d784e4134637 type nexus underlay IP fd00:1122:3344:104::5 (unchanged)
+         zone d48b2dfa-c749-47dd-98b6-3f478cd63934 type crucible_pantry underlay IP fd00:1122:3344:104::6 (unchanged)
+         zone fd003bee-818a-4c95-aae5-e378ac601522 type crucible underlay IP fd00:1122:3344:104::8 (unchanged)
+  sled 77851655-10a5-4e80-88b0-a001473cab7e
       zone config generation 5
-         zone 185e6b00-7e18-456c-9dfc-edd0194ac207 type crucible underlay IP fd00:1122:3344:103::a (unchanged)
-         zone 1fc78bc7-feca-4a10-9beb-58b53fda5210 type crucible underlay IP fd00:1122:3344:103::10 (unchanged)
-         zone 45dccfd6-4674-43c6-a0e7-4dde5f24d90d type internal_dns underlay IP fd00:1122:3344:3::1 (unchanged)
-         zone 4b74c418-0a08-4489-b652-e97267c1c220 type crucible underlay IP fd00:1122:3344:103::7 (unchanged)
-         zone 54c947d2-6355-453c-80fc-8f49cc2129ee type nexus underlay IP fd00:1122:3344:103::5 (unchanged)
-         zone 60d64b70-89f5-4774-9e41-2c79c4173911 type crucible_pantry underlay IP fd00:1122:3344:103::6 (unchanged)
-         zone 921958c0-7761-4294-945d-2a078327bf2c type crucible underlay IP fd00:1122:3344:103::8 (unchanged)
-         zone 9df22034-59f6-431f-82e7-258fe983879b type external_dns underlay IP fd00:1122:3344:103::4 (unchanged)
-         zone bb21c0c4-c43f-4913-9bf9-707bd7f1769a type internal_ntp underlay IP fd00:1122:3344:103::11 (unchanged)
-         zone ccb0d612-9d7c-41ba-bd51-661c3c75ec91 type crucible underlay IP fd00:1122:3344:103::9 (unchanged)
-         zone ccef3e95-cae5-4072-9b7d-49e56fa9dff4 type crucible underlay IP fd00:1122:3344:103::d (unchanged)
-         zone d42fc874-8416-4212-b5a0-febba4c6b0ac type crucible underlay IP fd00:1122:3344:103::b (unchanged)
-         zone d5ba7d45-26d7-45ca-8c0a-a560f243b5fa type crucible underlay IP fd00:1122:3344:103::e (unchanged)
-         zone da27ee6a-9841-46aa-acda-7cce3e04cb60 type cockroach_db underlay IP fd00:1122:3344:103::3 (unchanged)
-         zone e09525cf-9c87-4d05-8860-1356dd0dee8a type crucible underlay IP fd00:1122:3344:103::f (unchanged)
-         zone f7264a3e-e8e7-4476-8df3-76c9d00a383b type crucible underlay IP fd00:1122:3344:103::c (unchanged)
+         zone 084cac6e-44b5-4333-a2ff-b0993b534898 type crucible underlay IP fd00:1122:3344:102::8 (unchanged)
+         zone 11c527c5-0598-428e-9270-be5b74d3461b type cockroach_db underlay IP fd00:1122:3344:102::3 (unchanged)
+         zone 20be9299-495d-428d-b315-694c2c4b64af type crucible_pantry underlay IP fd00:1122:3344:102::6 (unchanged)
+         zone 2b29cb95-f8c6-4aff-acc0-1c0a627be392 type crucible underlay IP fd00:1122:3344:102::b (unchanged)
+         zone 8d452bdd-e8b8-4108-a690-46fc59197db0 type crucible underlay IP fd00:1122:3344:102::a (unchanged)
+         zone 953019c0-8e85-4417-b231-d42820470125 type oximeter underlay IP fd00:1122:3344:102::5 (unchanged)
+         zone 99c635b9-8e97-414a-8316-8261d2d767d0 type crucible underlay IP fd00:1122:3344:102::7 (unchanged)
+         zone cbe1abfa-0063-44b2-9ce7-7ad76b403a94 type internal_dns underlay IP fd00:1122:3344:2::1 (unchanged)
+         zone d52e797a-8fff-4e1e-a5ff-0ab8c8293ded type boundary_ntp underlay IP fd00:1122:3344:102::c (unchanged)
+         zone dc7f703a-27d1-4802-b85d-5629ddb2d67c type crucible underlay IP fd00:1122:3344:102::9 (unchanged)
+         zone f0ed9838-6f1b-4646-a478-9f74f2e6e802 type external_dns underlay IP fd00:1122:3344:102::4 (unchanged)
+  sled f122b3a2-e383-4e42-b2a5-74a69a9d110a
+      zone config generation 5
+         zone 0d459323-e414-4f32-9944-76c7331da622 type nexus underlay IP fd00:1122:3344:101::5 (unchanged)
+         zone 2902972b-3ae4-489d-b937-6480352e214a type crucible underlay IP fd00:1122:3344:101::8 (unchanged)
+         zone 2ee832de-af75-47ac-abac-f91304857ca4 type external_dns underlay IP fd00:1122:3344:101::4 (unchanged)
+         zone 30050a7e-7fc3-4965-b07b-1635212393eb type crucible_pantry underlay IP fd00:1122:3344:101::6 (unchanged)
+         zone 8494befe-6df6-42d1-9baa-599ba4228430 type boundary_ntp underlay IP fd00:1122:3344:101::c (unchanged)
+         zone 92cba955-3a5a-4d82-b0f2-30e89d4556d1 type crucible underlay IP fd00:1122:3344:101::7 (unchanged)
+         zone 9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b type cockroach_db underlay IP fd00:1122:3344:101::3 (unchanged)
+         zone e668e3f5-5457-4689-97da-b2af537787ac type crucible underlay IP fd00:1122:3344:101::b (unchanged)
+         zone f32fd972-7f40-4c58-9e00-7fc5c44273f4 type crucible underlay IP fd00:1122:3344:101::9 (unchanged)
+         zone f8c79ba4-194a-4cfe-8090-5315e297b780 type internal_dns underlay IP fd00:1122:3344:1::1 (unchanged)
+         zone fa5e0176-b4a8-4598-aa31-40fd562d246c type crucible underlay IP fd00:1122:3344:101::a (unchanged)
 
 
 > 
 
-> blueprint-diff 6c127695-ba15-408d-a992-325a1a888380 cfa03594-f438-4736-a416-6307f7bf8f2e
-diff blueprint 6c127695-ba15-408d-a992-325a1a888380 blueprint cfa03594-f438-4736-a416-6307f7bf8f2e
---- blueprint 6c127695-ba15-408d-a992-325a1a888380
-+++ blueprint cfa03594-f438-4736-a416-6307f7bf8f2e
-  sled 1762267a-a0cb-4c9a-88b4-8ce828cbc021
+> blueprint-diff 649e444d-d00f-4067-a915-12e15647bf37 cd34e87b-d821-4768-b32f-f41c28db835e
+diff blueprint 649e444d-d00f-4067-a915-12e15647bf37 blueprint cd34e87b-d821-4768-b32f-f41c28db835e
+--- blueprint 649e444d-d00f-4067-a915-12e15647bf37
++++ blueprint cd34e87b-d821-4768-b32f-f41c28db835e
+  sled 1f34b5cc-6dac-40cc-9ea1-95f32c33e59b
+      zone config generation 5
+         zone 0db88baf-911d-47bb-af71-3aebd7c96934 type nexus underlay IP fd00:1122:3344:103::4 (unchanged)
+         zone 213d7dd3-95a4-45a7-99ca-e4899991dab4 type crucible underlay IP fd00:1122:3344:103::6 (unchanged)
+         zone 218f36a5-9e51-4b88-ad1a-da4d85c246c4 type internal_dns underlay IP fd00:1122:3344:3::1 (unchanged)
+         zone 40c47ead-3be8-4724-a8ef-b66233fc5c65 type crucible underlay IP fd00:1122:3344:103::7 (unchanged)
+         zone 425d9d2f-72ea-4f53-80b6-04e002771cdb type crucible underlay IP fd00:1122:3344:103::9 (unchanged)
+         zone 46a050b6-37b8-47b9-b4ae-ccb736ecd6d5 type crucible underlay IP fd00:1122:3344:103::8 (unchanged)
+         zone 4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd type clickhouse underlay IP fd00:1122:3344:103::5 (unchanged)
+         zone 7611a549-effc-41b6-b6cb-c03e0c70a31f type internal_ntp underlay IP fd00:1122:3344:103::b (unchanged)
+         zone af7ccead-2bd8-4a8a-9fbf-3b788d180139 type cockroach_db underlay IP fd00:1122:3344:103::3 (unchanged)
+         zone d4428e41-ad7b-41f0-bde6-7c7c955f20c1 type crucible underlay IP fd00:1122:3344:103::a (unchanged)
+  sled 67a1268b-52d3-4159-b3ed-7a0777f1f340
       zone config generation 5
-         zone 0706860a-3801-42a7-8000-fff741f7665b type crucible underlay IP fd00:1122:3344:101::c (unchanged)
-         zone 34a279a2-a025-41ff-a392-1addac400f38 type cockroach_db underlay IP fd00:1122:3344:101::3 (unchanged)
-         zone 3d344935-6f48-405f-8794-2c6e8fa4b1c2 type internal_dns underlay IP fd00:1122:3344:1::1 (unchanged)
-         zone 59c3ed4f-3670-44b1-9ea0-9a6162a08629 type nexus underlay IP fd00:1122:3344:101::5 (unchanged)
-         zone 67852e14-ce7f-4e50-a61d-4059d0c057b9 type crucible underlay IP fd00:1122:3344:101::10 (unchanged)
-         zone 6ace1a69-d55a-45f0-a127-99c90f1b070c type crucible underlay IP fd00:1122:3344:101::f (unchanged)
-         zone 84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe type crucible underlay IP fd00:1122:3344:101::9 (unchanged)
-         zone 854c1544-d0ea-4d65-9cf4-776e91a4fcb5 type crucible underlay IP fd00:1122:3344:101::8 (unchanged)
-         zone 86ee7359-aa35-41d9-8a76-54961bc516a1 type cockroach_db underlay IP fd00:1122:3344:101::4 (unchanged)
-         zone 925ea436-fad4-4fff-8ae0-24edd725a2b0 type crucible underlay IP fd00:1122:3344:101::d (unchanged)
-         zone be272d1e-4d34-4478-94b4-a76f5fcbef36 type crucible underlay IP fd00:1122:3344:101::a (unchanged)
-         zone bebd754d-d06c-45f4-99bf-e1aab6253ab8 type oximeter underlay IP fd00:1122:3344:101::6 (unchanged)
-         zone d630cca7-e3f7-47a8-aee8-ad0790895abc type boundary_ntp underlay IP fd00:1122:3344:101::11 (unchanged)
-         zone d8c51c79-46ff-44bd-80be-08d6283f4e21 type crucible underlay IP fd00:1122:3344:101::e (unchanged)
-         zone dbbc0422-2b23-4d9e-b1db-8003cfc29ad0 type crucible underlay IP fd00:1122:3344:101::b (unchanged)
-         zone f33a2375-212d-42e4-b539-6079aeb4e5b7 type crucible_pantry underlay IP fd00:1122:3344:101::7 (unchanged)
-  sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f
--     zone config generation 2
-+     zone config generation 3
-         zone 2d5c29f7-4915-4876-82f8-e388b15c5067 type internal_ntp underlay IP fd00:1122:3344:121::21 (unchanged)
-+        zone 0eca39c6-62e2-4a1f-8ea2-e9332a774a26 type crucible underlay IP fd00:1122:3344:121::28 (added)
-+        zone 12431d3e-1516-4822-98b3-3711ff912e69 type crucible underlay IP fd00:1122:3344:121::22 (added)
-+        zone 56414d76-9b8c-4e15-acf1-cd269c2d8638 type crucible underlay IP fd00:1122:3344:121::25 (added)
-+        zone 891cb9da-301d-40cc-b93d-5fcb350fa968 type crucible underlay IP fd00:1122:3344:121::29 (added)
-+        zone d9014907-24c0-4c33-a475-db15942f4f8f type crucible underlay IP fd00:1122:3344:121::23 (added)
-+        zone e950de25-7306-4cf7-a6f4-6db11adaa310 type crucible underlay IP fd00:1122:3344:121::27 (added)
-+        zone fe3dc092-ea23-42d7-9f1d-36eed72b539c type crucible underlay IP fd00:1122:3344:121::26 (added)
-+        zone ff8cce49-a721-4ad1-b09e-2bd6e4e42d00 type crucible underlay IP fd00:1122:3344:121::24 (added)
-  sled a6634f64-f6fb-426c-b00b-9b30aed9f53a
+         zone 1ebea0c3-1bfd-472c-9f84-ac9e23b32707 type crucible underlay IP fd00:1122:3344:104::b (unchanged)
+         zone 2e9dfa8f-5371-48e5-b7f7-62a6af8283c1 type cockroach_db underlay IP fd00:1122:3344:104::4 (unchanged)
+         zone 57c712a6-6429-44b7-a9b2-9957cb9054d4 type crucible underlay IP fd00:1122:3344:104::a (unchanged)
+         zone 59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9 type crucible underlay IP fd00:1122:3344:104::7 (unchanged)
+         zone 73916098-788f-48d8-b745-e13fdcfb771b type internal_ntp underlay IP fd00:1122:3344:104::c (unchanged)
+         zone 9bb513e3-7505-416c-aa70-b26f1152bbdc type crucible underlay IP fd00:1122:3344:104::9 (unchanged)
+         zone 9c14c879-137b-4f5b-8068-1292bffc83ed type cockroach_db underlay IP fd00:1122:3344:104::3 (unchanged)
+         zone 9cf90589-5da1-4b42-b09b-d784e4134637 type nexus underlay IP fd00:1122:3344:104::5 (unchanged)
+         zone d48b2dfa-c749-47dd-98b6-3f478cd63934 type crucible_pantry underlay IP fd00:1122:3344:104::6 (unchanged)
+         zone fd003bee-818a-4c95-aae5-e378ac601522 type crucible underlay IP fd00:1122:3344:104::8 (unchanged)
+  sled 77851655-10a5-4e80-88b0-a001473cab7e
       zone config generation 5
-         zone 0ebd2b5b-2dec-4d17-8e23-e468c04dacb6 type crucible underlay IP fd00:1122:3344:102::11 (unchanged)
-         zone 18b99404-d8b2-4463-bc99-638d9b678ab3 type crucible underlay IP fd00:1122:3344:102::e (unchanged)
-         zone 479811e7-6409-45d4-b294-5333534e47ed type crucible underlay IP fd00:1122:3344:102::9 (unchanged)
-         zone 54d1b693-bc4d-4c47-9009-c6f624073801 type crucible underlay IP fd00:1122:3344:102::a (unchanged)
-         zone 5b743072-9369-45d7-b7c9-dbdf736301f8 type internal_dns underlay IP fd00:1122:3344:2::1 (unchanged)
-         zone 5c4163fa-d6a6-4a0c-9476-3dda135bc9a4 type crucible_pantry underlay IP fd00:1122:3344:102::7 (unchanged)
-         zone 61a39188-dd82-43f6-a87c-ebb4c2d54ac4 type crucible underlay IP fd00:1122:3344:102::8 (unchanged)
-         zone 658076ed-fb43-49e7-a813-df3d23497848 type crucible underlay IP fd00:1122:3344:102::10 (unchanged)
-         zone 6edb836e-b100-4080-bb03-738d6441d571 type crucible underlay IP fd00:1122:3344:102::c (unchanged)
-         zone 7f83d784-9c5c-498f-b559-4176bbafbc4b type nexus underlay IP fd00:1122:3344:102::5 (unchanged)
-         zone 8052938f-73bc-4c54-8ed0-c62f6d77b9fd type crucible underlay IP fd00:1122:3344:102::d (unchanged)
-         zone 86bc0a2a-e24a-4bc1-8222-89207a0937f9 type cockroach_db underlay IP fd00:1122:3344:102::4 (unchanged)
-         zone 9241b9da-8188-4aae-a9dc-62acb453618f type clickhouse underlay IP fd00:1122:3344:102::6 (unchanged)
-         zone a781a349-62b2-41f0-9dd7-c64d290aef82 type crucible underlay IP fd00:1122:3344:102::f (unchanged)
-         zone c67814bf-9972-4bb0-a666-24778ca70a16 type cockroach_db underlay IP fd00:1122:3344:102::3 (unchanged)
-         zone c74ba7d7-3d5f-4a29-aadc-effe33a90909 type boundary_ntp underlay IP fd00:1122:3344:102::12 (unchanged)
-         zone dfc27183-ed85-44c3-ac3c-4427c4cb03e3 type crucible underlay IP fd00:1122:3344:102::b (unchanged)
-  sled fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89
+         zone 084cac6e-44b5-4333-a2ff-b0993b534898 type crucible underlay IP fd00:1122:3344:102::8 (unchanged)
+         zone 11c527c5-0598-428e-9270-be5b74d3461b type cockroach_db underlay IP fd00:1122:3344:102::3 (unchanged)
+         zone 20be9299-495d-428d-b315-694c2c4b64af type crucible_pantry underlay IP fd00:1122:3344:102::6 (unchanged)
+         zone 2b29cb95-f8c6-4aff-acc0-1c0a627be392 type crucible underlay IP fd00:1122:3344:102::b (unchanged)
+         zone 8d452bdd-e8b8-4108-a690-46fc59197db0 type crucible underlay IP fd00:1122:3344:102::a (unchanged)
+         zone 953019c0-8e85-4417-b231-d42820470125 type oximeter underlay IP fd00:1122:3344:102::5 (unchanged)
+         zone 99c635b9-8e97-414a-8316-8261d2d767d0 type crucible underlay IP fd00:1122:3344:102::7 (unchanged)
+         zone cbe1abfa-0063-44b2-9ce7-7ad76b403a94 type internal_dns underlay IP fd00:1122:3344:2::1 (unchanged)
+         zone d52e797a-8fff-4e1e-a5ff-0ab8c8293ded type boundary_ntp underlay IP fd00:1122:3344:102::c (unchanged)
+         zone dc7f703a-27d1-4802-b85d-5629ddb2d67c type crucible underlay IP fd00:1122:3344:102::9 (unchanged)
+         zone f0ed9838-6f1b-4646-a478-9f74f2e6e802 type external_dns underlay IP fd00:1122:3344:102::4 (unchanged)
+  sled f122b3a2-e383-4e42-b2a5-74a69a9d110a
       zone config generation 5
-         zone 185e6b00-7e18-456c-9dfc-edd0194ac207 type crucible underlay IP fd00:1122:3344:103::a (unchanged)
-         zone 1fc78bc7-feca-4a10-9beb-58b53fda5210 type crucible underlay IP fd00:1122:3344:103::10 (unchanged)
-         zone 45dccfd6-4674-43c6-a0e7-4dde5f24d90d type internal_dns underlay IP fd00:1122:3344:3::1 (unchanged)
-         zone 4b74c418-0a08-4489-b652-e97267c1c220 type crucible underlay IP fd00:1122:3344:103::7 (unchanged)
-         zone 54c947d2-6355-453c-80fc-8f49cc2129ee type nexus underlay IP fd00:1122:3344:103::5 (unchanged)
-         zone 60d64b70-89f5-4774-9e41-2c79c4173911 type crucible_pantry underlay IP fd00:1122:3344:103::6 (unchanged)
-         zone 921958c0-7761-4294-945d-2a078327bf2c type crucible underlay IP fd00:1122:3344:103::8 (unchanged)
-         zone 9df22034-59f6-431f-82e7-258fe983879b type external_dns underlay IP fd00:1122:3344:103::4 (unchanged)
-         zone bb21c0c4-c43f-4913-9bf9-707bd7f1769a type internal_ntp underlay IP fd00:1122:3344:103::11 (unchanged)
-         zone ccb0d612-9d7c-41ba-bd51-661c3c75ec91 type crucible underlay IP fd00:1122:3344:103::9 (unchanged)
-         zone ccef3e95-cae5-4072-9b7d-49e56fa9dff4 type crucible underlay IP fd00:1122:3344:103::d (unchanged)
-         zone d42fc874-8416-4212-b5a0-febba4c6b0ac type crucible underlay IP fd00:1122:3344:103::b (unchanged)
-         zone d5ba7d45-26d7-45ca-8c0a-a560f243b5fa type crucible underlay IP fd00:1122:3344:103::e (unchanged)
-         zone da27ee6a-9841-46aa-acda-7cce3e04cb60 type cockroach_db underlay IP fd00:1122:3344:103::3 (unchanged)
-         zone e09525cf-9c87-4d05-8860-1356dd0dee8a type crucible underlay IP fd00:1122:3344:103::f (unchanged)
-         zone f7264a3e-e8e7-4476-8df3-76c9d00a383b type crucible underlay IP fd00:1122:3344:103::c (unchanged)
+         zone 0d459323-e414-4f32-9944-76c7331da622 type nexus underlay IP fd00:1122:3344:101::5 (unchanged)
+         zone 2902972b-3ae4-489d-b937-6480352e214a type crucible underlay IP fd00:1122:3344:101::8 (unchanged)
+         zone 2ee832de-af75-47ac-abac-f91304857ca4 type external_dns underlay IP fd00:1122:3344:101::4 (unchanged)
+         zone 30050a7e-7fc3-4965-b07b-1635212393eb type crucible_pantry underlay IP fd00:1122:3344:101::6 (unchanged)
+         zone 8494befe-6df6-42d1-9baa-599ba4228430 type boundary_ntp underlay IP fd00:1122:3344:101::c (unchanged)
+         zone 92cba955-3a5a-4d82-b0f2-30e89d4556d1 type crucible underlay IP fd00:1122:3344:101::7 (unchanged)
+         zone 9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b type cockroach_db underlay IP fd00:1122:3344:101::3 (unchanged)
+         zone e668e3f5-5457-4689-97da-b2af537787ac type crucible underlay IP fd00:1122:3344:101::b (unchanged)
+         zone f32fd972-7f40-4c58-9e00-7fc5c44273f4 type crucible underlay IP fd00:1122:3344:101::9 (unchanged)
+         zone f8c79ba4-194a-4cfe-8090-5315e297b780 type internal_dns underlay IP fd00:1122:3344:1::1 (unchanged)
+         zone fa5e0176-b4a8-4598-aa31-40fd562d246c type crucible underlay IP fd00:1122:3344:101::a (unchanged)
 
 
 > 
 
-> blueprint-diff cfa03594-f438-4736-a416-6307f7bf8f2e 6c127695-ba15-408d-a992-325a1a888380
-diff blueprint cfa03594-f438-4736-a416-6307f7bf8f2e blueprint 6c127695-ba15-408d-a992-325a1a888380
---- blueprint cfa03594-f438-4736-a416-6307f7bf8f2e
-+++ blueprint 6c127695-ba15-408d-a992-325a1a888380
-  sled 1762267a-a0cb-4c9a-88b4-8ce828cbc021
+> blueprint-diff cd34e87b-d821-4768-b32f-f41c28db835e 649e444d-d00f-4067-a915-12e15647bf37
+diff blueprint cd34e87b-d821-4768-b32f-f41c28db835e blueprint 649e444d-d00f-4067-a915-12e15647bf37
+--- blueprint cd34e87b-d821-4768-b32f-f41c28db835e
++++ blueprint 649e444d-d00f-4067-a915-12e15647bf37
+  sled 1f34b5cc-6dac-40cc-9ea1-95f32c33e59b
+      zone config generation 5
+         zone 0db88baf-911d-47bb-af71-3aebd7c96934 type nexus underlay IP fd00:1122:3344:103::4 (unchanged)
+         zone 213d7dd3-95a4-45a7-99ca-e4899991dab4 type crucible underlay IP fd00:1122:3344:103::6 (unchanged)
+         zone 218f36a5-9e51-4b88-ad1a-da4d85c246c4 type internal_dns underlay IP fd00:1122:3344:3::1 (unchanged)
+         zone 40c47ead-3be8-4724-a8ef-b66233fc5c65 type crucible underlay IP fd00:1122:3344:103::7 (unchanged)
+         zone 425d9d2f-72ea-4f53-80b6-04e002771cdb type crucible underlay IP fd00:1122:3344:103::9 (unchanged)
+         zone 46a050b6-37b8-47b9-b4ae-ccb736ecd6d5 type crucible underlay IP fd00:1122:3344:103::8 (unchanged)
+         zone 4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd type clickhouse underlay IP fd00:1122:3344:103::5 (unchanged)
+         zone 7611a549-effc-41b6-b6cb-c03e0c70a31f type internal_ntp underlay IP fd00:1122:3344:103::b (unchanged)
+         zone af7ccead-2bd8-4a8a-9fbf-3b788d180139 type cockroach_db underlay IP fd00:1122:3344:103::3 (unchanged)
+         zone d4428e41-ad7b-41f0-bde6-7c7c955f20c1 type crucible underlay IP fd00:1122:3344:103::a (unchanged)
+  sled 67a1268b-52d3-4159-b3ed-7a0777f1f340
       zone config generation 5
-         zone 0706860a-3801-42a7-8000-fff741f7665b type crucible underlay IP fd00:1122:3344:101::c (unchanged)
-         zone 34a279a2-a025-41ff-a392-1addac400f38 type cockroach_db underlay IP fd00:1122:3344:101::3 (unchanged)
-         zone 3d344935-6f48-405f-8794-2c6e8fa4b1c2 type internal_dns underlay IP fd00:1122:3344:1::1 (unchanged)
-         zone 59c3ed4f-3670-44b1-9ea0-9a6162a08629 type nexus underlay IP fd00:1122:3344:101::5 (unchanged)
-         zone 67852e14-ce7f-4e50-a61d-4059d0c057b9 type crucible underlay IP fd00:1122:3344:101::10 (unchanged)
-         zone 6ace1a69-d55a-45f0-a127-99c90f1b070c type crucible underlay IP fd00:1122:3344:101::f (unchanged)
-         zone 84d6b7fc-b5c2-499f-8d17-0eb1ea5affbe type crucible underlay IP fd00:1122:3344:101::9 (unchanged)
-         zone 854c1544-d0ea-4d65-9cf4-776e91a4fcb5 type crucible underlay IP fd00:1122:3344:101::8 (unchanged)
-         zone 86ee7359-aa35-41d9-8a76-54961bc516a1 type cockroach_db underlay IP fd00:1122:3344:101::4 (unchanged)
-         zone 925ea436-fad4-4fff-8ae0-24edd725a2b0 type crucible underlay IP fd00:1122:3344:101::d (unchanged)
-         zone be272d1e-4d34-4478-94b4-a76f5fcbef36 type crucible underlay IP fd00:1122:3344:101::a (unchanged)
-         zone bebd754d-d06c-45f4-99bf-e1aab6253ab8 type oximeter underlay IP fd00:1122:3344:101::6 (unchanged)
-         zone d630cca7-e3f7-47a8-aee8-ad0790895abc type boundary_ntp underlay IP fd00:1122:3344:101::11 (unchanged)
-         zone d8c51c79-46ff-44bd-80be-08d6283f4e21 type crucible underlay IP fd00:1122:3344:101::e (unchanged)
-         zone dbbc0422-2b23-4d9e-b1db-8003cfc29ad0 type crucible underlay IP fd00:1122:3344:101::b (unchanged)
-         zone f33a2375-212d-42e4-b539-6079aeb4e5b7 type crucible_pantry underlay IP fd00:1122:3344:101::7 (unchanged)
-  sled a243c1d0-9051-4b94-ab3e-f2a93fd0ae4f
--     zone config generation 3
-+     zone config generation 2
--        zone 0eca39c6-62e2-4a1f-8ea2-e9332a774a26 type crucible (removed)
--        zone 12431d3e-1516-4822-98b3-3711ff912e69 type crucible (removed)
--        zone 56414d76-9b8c-4e15-acf1-cd269c2d8638 type crucible (removed)
--        zone 891cb9da-301d-40cc-b93d-5fcb350fa968 type crucible (removed)
--        zone d9014907-24c0-4c33-a475-db15942f4f8f type crucible (removed)
--        zone e950de25-7306-4cf7-a6f4-6db11adaa310 type crucible (removed)
--        zone fe3dc092-ea23-42d7-9f1d-36eed72b539c type crucible (removed)
--        zone ff8cce49-a721-4ad1-b09e-2bd6e4e42d00 type crucible (removed)
-         zone 2d5c29f7-4915-4876-82f8-e388b15c5067 type internal_ntp underlay IP fd00:1122:3344:121::21 (unchanged)
-  sled a6634f64-f6fb-426c-b00b-9b30aed9f53a
+         zone 1ebea0c3-1bfd-472c-9f84-ac9e23b32707 type crucible underlay IP fd00:1122:3344:104::b (unchanged)
+         zone 2e9dfa8f-5371-48e5-b7f7-62a6af8283c1 type cockroach_db underlay IP fd00:1122:3344:104::4 (unchanged)
+         zone 57c712a6-6429-44b7-a9b2-9957cb9054d4 type crucible underlay IP fd00:1122:3344:104::a (unchanged)
+         zone 59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9 type crucible underlay IP fd00:1122:3344:104::7 (unchanged)
+         zone 73916098-788f-48d8-b745-e13fdcfb771b type internal_ntp underlay IP fd00:1122:3344:104::c (unchanged)
+         zone 9bb513e3-7505-416c-aa70-b26f1152bbdc type crucible underlay IP fd00:1122:3344:104::9 (unchanged)
+         zone 9c14c879-137b-4f5b-8068-1292bffc83ed type cockroach_db underlay IP fd00:1122:3344:104::3 (unchanged)
+         zone 9cf90589-5da1-4b42-b09b-d784e4134637 type nexus underlay IP fd00:1122:3344:104::5 (unchanged)
+         zone d48b2dfa-c749-47dd-98b6-3f478cd63934 type crucible_pantry underlay IP fd00:1122:3344:104::6 (unchanged)
+         zone fd003bee-818a-4c95-aae5-e378ac601522 type crucible underlay IP fd00:1122:3344:104::8 (unchanged)
+  sled 77851655-10a5-4e80-88b0-a001473cab7e
       zone config generation 5
-         zone 0ebd2b5b-2dec-4d17-8e23-e468c04dacb6 type crucible underlay IP fd00:1122:3344:102::11 (unchanged)
-         zone 18b99404-d8b2-4463-bc99-638d9b678ab3 type crucible underlay IP fd00:1122:3344:102::e (unchanged)
-         zone 479811e7-6409-45d4-b294-5333534e47ed type crucible underlay IP fd00:1122:3344:102::9 (unchanged)
-         zone 54d1b693-bc4d-4c47-9009-c6f624073801 type crucible underlay IP fd00:1122:3344:102::a (unchanged)
-         zone 5b743072-9369-45d7-b7c9-dbdf736301f8 type internal_dns underlay IP fd00:1122:3344:2::1 (unchanged)
-         zone 5c4163fa-d6a6-4a0c-9476-3dda135bc9a4 type crucible_pantry underlay IP fd00:1122:3344:102::7 (unchanged)
-         zone 61a39188-dd82-43f6-a87c-ebb4c2d54ac4 type crucible underlay IP fd00:1122:3344:102::8 (unchanged)
-         zone 658076ed-fb43-49e7-a813-df3d23497848 type crucible underlay IP fd00:1122:3344:102::10 (unchanged)
-         zone 6edb836e-b100-4080-bb03-738d6441d571 type crucible underlay IP fd00:1122:3344:102::c (unchanged)
-         zone 7f83d784-9c5c-498f-b559-4176bbafbc4b type nexus underlay IP fd00:1122:3344:102::5 (unchanged)
-         zone 8052938f-73bc-4c54-8ed0-c62f6d77b9fd type crucible underlay IP fd00:1122:3344:102::d (unchanged)
-         zone 86bc0a2a-e24a-4bc1-8222-89207a0937f9 type cockroach_db underlay IP fd00:1122:3344:102::4 (unchanged)
-         zone 9241b9da-8188-4aae-a9dc-62acb453618f type clickhouse underlay IP fd00:1122:3344:102::6 (unchanged)
-         zone a781a349-62b2-41f0-9dd7-c64d290aef82 type crucible underlay IP fd00:1122:3344:102::f (unchanged)
-         zone c67814bf-9972-4bb0-a666-24778ca70a16 type cockroach_db underlay IP fd00:1122:3344:102::3 (unchanged)
-         zone c74ba7d7-3d5f-4a29-aadc-effe33a90909 type boundary_ntp underlay IP fd00:1122:3344:102::12 (unchanged)
-         zone dfc27183-ed85-44c3-ac3c-4427c4cb03e3 type crucible underlay IP fd00:1122:3344:102::b (unchanged)
-  sled fa3c33bf-bc6c-4d29-8dcc-c16b9ab1ec89
+         zone 084cac6e-44b5-4333-a2ff-b0993b534898 type crucible underlay IP fd00:1122:3344:102::8 (unchanged)
+         zone 11c527c5-0598-428e-9270-be5b74d3461b type cockroach_db underlay IP fd00:1122:3344:102::3 (unchanged)
+         zone 20be9299-495d-428d-b315-694c2c4b64af type crucible_pantry underlay IP fd00:1122:3344:102::6 (unchanged)
+         zone 2b29cb95-f8c6-4aff-acc0-1c0a627be392 type crucible underlay IP fd00:1122:3344:102::b (unchanged)
+         zone 8d452bdd-e8b8-4108-a690-46fc59197db0 type crucible underlay IP fd00:1122:3344:102::a (unchanged)
+         zone 953019c0-8e85-4417-b231-d42820470125 type oximeter underlay IP fd00:1122:3344:102::5 (unchanged)
+         zone 99c635b9-8e97-414a-8316-8261d2d767d0 type crucible underlay IP fd00:1122:3344:102::7 (unchanged)
+         zone cbe1abfa-0063-44b2-9ce7-7ad76b403a94 type internal_dns underlay IP fd00:1122:3344:2::1 (unchanged)
+         zone d52e797a-8fff-4e1e-a5ff-0ab8c8293ded type boundary_ntp underlay IP fd00:1122:3344:102::c (unchanged)
+         zone dc7f703a-27d1-4802-b85d-5629ddb2d67c type crucible underlay IP fd00:1122:3344:102::9 (unchanged)
+         zone f0ed9838-6f1b-4646-a478-9f74f2e6e802 type external_dns underlay IP fd00:1122:3344:102::4 (unchanged)
+  sled f122b3a2-e383-4e42-b2a5-74a69a9d110a
       zone config generation 5
-         zone 185e6b00-7e18-456c-9dfc-edd0194ac207 type crucible underlay IP fd00:1122:3344:103::a (unchanged)
-         zone 1fc78bc7-feca-4a10-9beb-58b53fda5210 type crucible underlay IP fd00:1122:3344:103::10 (unchanged)
-         zone 45dccfd6-4674-43c6-a0e7-4dde5f24d90d type internal_dns underlay IP fd00:1122:3344:3::1 (unchanged)
-         zone 4b74c418-0a08-4489-b652-e97267c1c220 type crucible underlay IP fd00:1122:3344:103::7 (unchanged)
-         zone 54c947d2-6355-453c-80fc-8f49cc2129ee type nexus underlay IP fd00:1122:3344:103::5 (unchanged)
-         zone 60d64b70-89f5-4774-9e41-2c79c4173911 type crucible_pantry underlay IP fd00:1122:3344:103::6 (unchanged)
-         zone 921958c0-7761-4294-945d-2a078327bf2c type crucible underlay IP fd00:1122:3344:103::8 (unchanged)
-         zone 9df22034-59f6-431f-82e7-258fe983879b type external_dns underlay IP fd00:1122:3344:103::4 (unchanged)
-         zone bb21c0c4-c43f-4913-9bf9-707bd7f1769a type internal_ntp underlay IP fd00:1122:3344:103::11 (unchanged)
-         zone ccb0d612-9d7c-41ba-bd51-661c3c75ec91 type crucible underlay IP fd00:1122:3344:103::9 (unchanged)
-         zone ccef3e95-cae5-4072-9b7d-49e56fa9dff4 type crucible underlay IP fd00:1122:3344:103::d (unchanged)
-         zone d42fc874-8416-4212-b5a0-febba4c6b0ac type crucible underlay IP fd00:1122:3344:103::b (unchanged)
-         zone d5ba7d45-26d7-45ca-8c0a-a560f243b5fa type crucible underlay IP fd00:1122:3344:103::e (unchanged)
-         zone da27ee6a-9841-46aa-acda-7cce3e04cb60 type cockroach_db underlay IP fd00:1122:3344:103::3 (unchanged)
-         zone e09525cf-9c87-4d05-8860-1356dd0dee8a type crucible underlay IP fd00:1122:3344:103::f (unchanged)
-         zone f7264a3e-e8e7-4476-8df3-76c9d00a383b type crucible underlay IP fd00:1122:3344:103::c (unchanged)
+         zone 0d459323-e414-4f32-9944-76c7331da622 type nexus underlay IP fd00:1122:3344:101::5 (unchanged)
+         zone 2902972b-3ae4-489d-b937-6480352e214a type crucible underlay IP fd00:1122:3344:101::8 (unchanged)
+         zone 2ee832de-af75-47ac-abac-f91304857ca4 type external_dns underlay IP fd00:1122:3344:101::4 (unchanged)
+         zone 30050a7e-7fc3-4965-b07b-1635212393eb type crucible_pantry underlay IP fd00:1122:3344:101::6 (unchanged)
+         zone 8494befe-6df6-42d1-9baa-599ba4228430 type boundary_ntp underlay IP fd00:1122:3344:101::c (unchanged)
+         zone 92cba955-3a5a-4d82-b0f2-30e89d4556d1 type crucible underlay IP fd00:1122:3344:101::7 (unchanged)
+         zone 9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b type cockroach_db underlay IP fd00:1122:3344:101::3 (unchanged)
+         zone e668e3f5-5457-4689-97da-b2af537787ac type crucible underlay IP fd00:1122:3344:101::b (unchanged)
+         zone f32fd972-7f40-4c58-9e00-7fc5c44273f4 type crucible underlay IP fd00:1122:3344:101::9 (unchanged)
+         zone f8c79ba4-194a-4cfe-8090-5315e297b780 type internal_dns underlay IP fd00:1122:3344:1::1 (unchanged)
+         zone fa5e0176-b4a8-4598-aa31-40fd562d246c type crucible underlay IP fd00:1122:3344:101::a (unchanged)
 
 
 > 
@@ -380,6 +308,6 @@ added sled
 
 > 
 
-> blueprint-plan cfa03594-f438-4736-a416-6307f7bf8f2e 83ed3949-d221-44f3-a901-02e9ff16d2a5
-generated blueprint REDACTED_UUID based on parent blueprint cfa03594-f438-4736-a416-6307f7bf8f2e
+> blueprint-plan cd34e87b-d821-4768-b32f-f41c28db835e 3f61b723-cfe7-40ec-b22c-e3e1f35325f9
+generated blueprint REDACTED_UUID based on parent blueprint cd34e87b-d821-4768-b32f-f41c28db835e
 
diff --git a/nexus/db-model/src/inventory.rs b/nexus/db-model/src/inventory.rs
index 2f69fd998c..1c268c5914 100644
--- a/nexus/db-model/src/inventory.rs
+++ b/nexus/db-model/src/inventory.rs
@@ -7,10 +7,11 @@
 use crate::omicron_zone_config::{OmicronZone, OmicronZoneNic};
 use crate::schema::{
     hw_baseboard_id, inv_caboose, inv_collection, inv_collection_error,
-    inv_omicron_zone, inv_omicron_zone_nic, inv_root_of_trust,
-    inv_root_of_trust_page, inv_service_processor, inv_sled_agent,
-    inv_sled_omicron_zones, sw_caboose, sw_root_of_trust_page,
+    inv_omicron_zone, inv_omicron_zone_nic, inv_physical_disk,
+    inv_root_of_trust, inv_root_of_trust_page, inv_service_processor,
+    inv_sled_agent, inv_sled_omicron_zones, sw_caboose, sw_root_of_trust_page,
 };
+use crate::PhysicalDiskKind;
 use crate::{
     impl_enum_type, ipv6, ByteCount, Generation, MacAddr, Name, ServiceKind,
     SqlU16, SqlU32, SqlU8,
@@ -653,6 +654,51 @@ impl InvSledAgent {
     }
 }
 
+/// See [`nexus_types::inventory::PhysicalDisk`].
+#[derive(Queryable, Clone, Debug, Selectable, Insertable)]
+#[diesel(table_name = inv_physical_disk)]
+pub struct InvPhysicalDisk {
+    pub inv_collection_id: Uuid,
+    pub sled_id: Uuid,
+    pub slot: i64,
+    pub vendor: String,
+    pub model: String,
+    pub serial: String,
+    pub variant: PhysicalDiskKind,
+}
+
+impl InvPhysicalDisk {
+    pub fn new(
+        inv_collection_id: Uuid,
+        sled_id: Uuid,
+        disk: nexus_types::inventory::PhysicalDisk,
+    ) -> Self {
+        Self {
+            inv_collection_id,
+            sled_id,
+            slot: disk.slot,
+            vendor: disk.identity.vendor,
+            model: disk.identity.model,
+            serial: disk.identity.serial,
+            variant: disk.variant.into(),
+        }
+    }
+}
+
+impl From<InvPhysicalDisk> for nexus_types::inventory::PhysicalDisk {
+    fn from(disk: InvPhysicalDisk) -> Self {
+        Self {
+            identity: omicron_common::disk::DiskIdentity {
+                vendor: disk.vendor,
+                serial: disk.serial,
+                model: disk.model,
+            },
+            variant: disk.variant.into(),
+            slot: disk.slot,
+        }
+    }
+}
+
 /// See [`nexus_types::inventory::OmicronZonesFound`].
 #[derive(Queryable, Clone, Debug, Selectable, Insertable)]
 #[diesel(table_name = inv_sled_omicron_zones)]
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index e2b918d805..781f56fe0c 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(40, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(41, 0, 0);
 
 table! {
     disk (id) {
@@ -1361,6 +1361,20 @@ table! {
     }
 }
 
+table! {
+    inv_physical_disk (inv_collection_id, sled_id, slot) {
+        inv_collection_id -> Uuid,
+        sled_id -> Uuid,
+        slot -> Int8,
+
+        vendor -> Text,
+        model -> Text,
+        serial -> Text,
+
+        variant -> crate::PhysicalDiskKindEnum,
+    }
+}
+
 table! {
     inv_sled_omicron_zones (inv_collection_id, sled_id) {
         inv_collection_id -> Uuid,
diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs
index 3f2f4bd127..860725cc08 100644
--- a/nexus/db-queries/src/db/datastore/inventory.rs
+++ b/nexus/db-queries/src/db/datastore/inventory.rs
@@ -39,6 +39,7 @@ use nexus_db_model::InvCollection;
 use nexus_db_model::InvCollectionError;
 use nexus_db_model::InvOmicronZone;
 use nexus_db_model::InvOmicronZoneNic;
+use nexus_db_model::InvPhysicalDisk;
 use nexus_db_model::InvRootOfTrust;
 use nexus_db_model::InvRotPage;
 use nexus_db_model::InvServiceProcessor;
@@ -125,6 +126,17 @@ impl DataStore {
                 ))
             })
             .collect::<Result<Vec<_>, Error>>()?;
+        // Pull disks out of all sled agents
+        let disks: Vec<_> = collection
+            .sled_agents
+            .iter()
+            .flat_map(|(sled_id, sled_agent)| {
+                sled_agent.disks.iter().map(|disk| {
+                    InvPhysicalDisk::new(collection_id, *sled_id, disk.clone())
+                })
+            })
+            .collect();
+
         // Partition the sled agents into those with an associated baseboard id
         // and those without one.  We handle these pretty differently.
         let (sled_agents_baseboards, sled_agents_no_baseboards): (
@@ -639,6 +651,25 @@ impl DataStore {
                 }
             }
 
+            // Insert rows for all the physical disks we found.
+            {
+                use db::schema::inv_physical_disk::dsl;
+
+                let batch_size = SQL_BATCH_SIZE.get().try_into().unwrap();
+                let mut disks = disks.into_iter();
+                loop {
+                    let some_disks =
+                        disks.by_ref().take(batch_size).collect::<Vec<_>>();
+                    if some_disks.is_empty() {
+                        break;
+                    }
+                    let _ = diesel::insert_into(dsl::inv_physical_disk)
+                        .values(some_disks)
+                        .execute_async(&conn)
+                        .await?;
+                }
+            }
+
             // Insert rows for the sled agents that we found.  In practice, we'd
             // expect these to all have baseboards (if using Oxide hardware) or
             // none have baseboards (if not).
@@ -1029,6 +1060,7 @@ impl DataStore {
             ncabooses,
             nrot_pages,
             nsled_agents,
+            nphysical_disks,
             nsled_agent_zones,
             nzones,
             nnics,
@@ -1100,6 +1132,17 @@ impl DataStore {
                     .await?
                 };
 
+                // Remove rows for physical disks found.
+                let nphysical_disks = {
+                    use db::schema::inv_physical_disk::dsl;
+                    diesel::delete(
+                        dsl::inv_physical_disk
+                            .filter(dsl::inv_collection_id.eq(collection_id)),
+                    )
+                    .execute_async(&conn)
+                    .await?
+                };
+
                 // Remove rows associated with Omicron zones
                 let nsled_agent_zones = {
                     use db::schema::inv_sled_omicron_zones::dsl;
@@ -1149,6 +1192,7 @@ impl DataStore {
                     ncabooses,
                     nrot_pages,
                     nsled_agents,
+                    nphysical_disks,
                     nsled_agent_zones,
                     nzones,
                     nnics,
@@ -1171,6 +1215,7 @@ impl DataStore {
             "ncabooses" => ncabooses,
             "nrot_pages" => nrot_pages,
             "nsled_agents" => nsled_agents,
+            "nphysical_disks" => nphysical_disks,
             "nsled_agent_zones" => nsled_agent_zones,
             "nzones" => nzones,
             "nnics" => nnics,
@@ -1391,6 +1436,40 @@ impl DataStore {
             rows
         };
 
+        // Mapping of "Sled ID" -> "All disks reported by that sled"
+        let physical_disks: BTreeMap<
+            Uuid,
+            Vec<nexus_types::inventory::PhysicalDisk>,
+        > = {
+            use db::schema::inv_physical_disk::dsl;
+
+            let mut disks = BTreeMap::<
+                Uuid,
+                Vec<nexus_types::inventory::PhysicalDisk>,
+            >::new();
+            let mut paginator = Paginator::new(batch_size);
+            while let Some(p) = paginator.next() {
+                let batch = paginated_multicolumn(
+                    dsl::inv_physical_disk,
+                    (dsl::sled_id, dsl::slot),
+                    &p.current_pagparams(),
+                )
+                .filter(dsl::inv_collection_id.eq(id))
+                .select(InvPhysicalDisk::as_select())
+                .load_async(&*conn)
+                .await
+                .map_err(|e| {
+                    public_error_from_diesel(e, ErrorHandler::Server)
+                })?;
+                paginator =
+                    p.found_batch(&batch, &|row| (row.sled_id, row.slot));
+                for disk in batch {
+                    disks.entry(disk.sled_id).or_default().push(disk.into());
+                }
+            }
+            disks
+        };
+
         // Collect the unique baseboard ids referenced by SPs, RoTs, and Sled
         // Agents.
         let baseboard_id_ids: BTreeSet<_> = sps
@@ -1494,6 +1573,10 @@ impl DataStore {
                         ),
                         usable_physical_ram: s.usable_physical_ram.into(),
                         reservoir_size: s.reservoir_size.into(),
+                        disks: physical_disks
+                            .get(&sled_id)
+                            .map(|disks| disks.to_vec())
+                            .unwrap_or_default(),
                     };
                     Ok((sled_id, sled_agent))
                 })
@@ -1933,6 +2016,7 @@ mod test {
     use nexus_types::inventory::RotPageWhich;
     use omicron_common::api::external::Error;
     use omicron_test_utils::dev;
+    use pretty_assertions::assert_eq;
     use std::num::NonZeroU32;
 
     struct CollectionCounts {
@@ -2384,6 +2468,12 @@ mod test {
                 .await
                 .unwrap();
             assert_eq!(0, count);
+            let count = schema::inv_physical_disk::dsl::inv_physical_disk
+                .select(diesel::dsl::count_star())
+                .first_async::<i64>(&conn)
+                .await
+                .unwrap();
+            assert_eq!(0, count);
             let count =
                 schema::inv_sled_omicron_zones::dsl::inv_sled_omicron_zones
                     .select(diesel::dsl::count_star())
diff --git a/nexus/inventory/src/builder.rs b/nexus/inventory/src/builder.rs
index 602890a440..80db03f18b 100644
--- a/nexus/inventory/src/builder.rs
+++ b/nexus/inventory/src/builder.rs
@@ -461,6 +461,7 @@ impl CollectionBuilder {
             reservoir_size: inventory.reservoir_size,
             time_collected: now_db_precision(),
             sled_id,
+            disks: inventory.disks.into_iter().map(|d| d.into()).collect(),
         };
 
         if let Some(previous) = self.sleds.get(&sled_id) {
@@ -910,6 +911,11 @@ mod test {
         let sled1_bb = sled1_agent.baseboard_id.as_ref().unwrap();
         assert_eq!(sled1_bb.part_number, "model1");
         assert_eq!(sled1_bb.serial_number, "s1");
+        assert_eq!(sled1_agent.disks.len(), 4);
+        assert_eq!(sled1_agent.disks[0].identity.vendor, "macrohard");
+        assert_eq!(sled1_agent.disks[0].identity.model, "box");
+        assert_eq!(sled1_agent.disks[0].identity.serial, "XXIV");
+
         let sled4_agent = &collection.sled_agents[&sled_agent_id_extra];
         let sled4_bb = sled4_agent.baseboard_id.as_ref().unwrap();
         assert_eq!(sled4_bb.serial_number, "s4");
diff --git a/nexus/inventory/src/examples.rs b/nexus/inventory/src/examples.rs
index 93ba139c85..286ded7276 100644
--- a/nexus/inventory/src/examples.rs
+++ b/nexus/inventory/src/examples.rs
@@ -272,6 +272,48 @@ pub fn representative() -> Representative {
     // This first one will match "sled1_bb"'s baseboard information.
     let sled_agent_id_basic =
         "c5aec1df-b897-49e4-8085-ccd975f9b529".parse().unwrap();
+    // Add some disks to this first sled.
+    let disks = vec![
+        // Let's say we have one manufacturer for our M.2...
+        sled_agent_client::types::InventoryDisk {
+            identity: sled_agent_client::types::DiskIdentity {
+                vendor: "macrohard".to_string(),
+                model: "box".to_string(),
+                serial: "XXIV".to_string(),
+            },
+            variant: sled_agent_client::types::DiskVariant::M2,
+            slot: 0,
+        },
+        // ... and a couple different vendors for our U.2s
+        sled_agent_client::types::InventoryDisk {
+            identity: sled_agent_client::types::DiskIdentity {
+                vendor: "memetendo".to_string(),
+                model: "swatch".to_string(),
+                serial: "0001".to_string(),
+            },
+            variant: sled_agent_client::types::DiskVariant::U2,
+            slot: 1,
+        },
+        sled_agent_client::types::InventoryDisk {
+            identity: sled_agent_client::types::DiskIdentity {
+                vendor: "memetendo".to_string(),
+                model: "swatch".to_string(),
+                serial: "0002".to_string(),
+            },
+            variant: sled_agent_client::types::DiskVariant::U2,
+            slot: 2,
+        },
+        sled_agent_client::types::InventoryDisk {
+            identity: sled_agent_client::types::DiskIdentity {
+                vendor: "tony".to_string(),
+                model: "craystation".to_string(),
+                serial: "5".to_string(),
+            },
+            variant: sled_agent_client::types::DiskVariant::U2,
+            slot: 3,
+        },
+    ];
+
     builder
         .found_sled_inventory(
             "fake sled agent 1",
@@ -283,6 +325,7 @@ pub fn representative() -> Representative {
                     revision: 0,
                 },
                 sled_agent_client::types::SledRole::Gimlet,
+                disks,
             ),
         )
         .unwrap();
@@ -307,6 +350,7 @@ pub fn representative() -> Representative {
                     revision: 0,
                 },
                 sled_agent_client::types::SledRole::Scrimlet,
+                vec![],
             ),
         )
         .unwrap();
@@ -326,6 +370,7 @@ pub fn representative() -> Representative {
                     model: String::from("fellofftruck"),
                 },
                 sled_agent_client::types::SledRole::Gimlet,
+                vec![],
             ),
         )
         .unwrap();
@@ -343,6 +388,7 @@ pub fn representative() -> Representative {
                 sled_agent_id_unknown,
                 sled_agent_client::types::Baseboard::Unknown,
                 sled_agent_client::types::SledRole::Gimlet,
+                vec![],
             ),
         )
         .unwrap();
@@ -439,6 +485,7 @@ pub fn sled_agent(
     sled_id: Uuid,
     baseboard: sled_agent_client::types::Baseboard,
     sled_role: sled_agent_client::types::SledRole,
+    disks: Vec<sled_agent_client::types::InventoryDisk>,
 ) -> sled_agent_client::types::Inventory {
     sled_agent_client::types::Inventory {
         baseboard,
@@ -448,5 +495,6 @@ pub fn sled_agent(
         sled_id,
         usable_hardware_threads: 10,
         usable_physical_ram: ByteCount::from(1024 * 1024),
+        disks,
     }
 }
diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs
index 5aff89df0d..8e8283f37e 100644
--- a/nexus/reconfigurator/planning/src/system.rs
+++ b/nexus/reconfigurator/planning/src/system.rs
@@ -472,9 +472,10 @@ impl Sled {
                 reservoir_size: ByteCount::from(1024),
                 sled_role,
                 sled_agent_address,
-                sled_id: sled_id,
+                sled_id,
                 usable_hardware_threads: 10,
                 usable_physical_ram: ByteCount::from(1024 * 1024),
+                disks: vec![],
             }
         };
 
@@ -556,6 +557,7 @@ impl Sled {
             sled_id,
             usable_hardware_threads: inv_sled_agent.usable_hardware_threads,
             usable_physical_ram: inv_sled_agent.usable_physical_ram,
+            disks: vec![],
         };
 
         Sled {
diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs
index 490661a819..aab25df7e8 100644
--- a/nexus/types/src/external_api/params.rs
+++ b/nexus/types/src/external_api/params.rs
@@ -1327,6 +1327,16 @@ pub enum PhysicalDiskKind {
     U2,
 }
 
+impl From<sled_agent_client::types::DiskVariant> for PhysicalDiskKind {
+    fn from(variant: sled_agent_client::types::DiskVariant) -> Self {
+        use sled_agent_client::types::DiskVariant;
+        match variant {
+            DiskVariant::U2 => Self::U2,
+            DiskVariant::M2 => Self::M2,
+        }
+    }
+}
+
 /// Different sources for a disk
 #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
 #[serde(tag = "type", rename_all = "snake_case")]
diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs
index 92985081dc..b0afe51a76 100644
--- a/nexus/types/src/inventory.rs
+++ b/nexus/types/src/inventory.rs
@@ -9,6 +9,7 @@
 //! nexus/inventory does not currently know about nexus/db-model and it's
 //! convenient to separate these concerns.)
 
+use crate::external_api::params::PhysicalDiskKind;
 use crate::external_api::params::UninitializedSledId;
 use crate::external_api::shared::Baseboard;
 use chrono::DateTime;
@@ -344,6 +345,28 @@ impl IntoRotPage for gateway_client::types::RotCfpa {
     }
 }
 
+/// A physical disk reported by a sled agent.
+///
+/// This identifies that a physical disk appears in a Sled.
+/// The existence of this object does not necessarily imply that
+/// the disk is being actively managed by the control plane.
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
+pub struct PhysicalDisk {
+    pub identity: omicron_common::disk::DiskIdentity,
+    pub variant: PhysicalDiskKind,
+    pub slot: i64,
+}
+
+impl From<sled_agent_client::types::InventoryDisk> for PhysicalDisk {
+    fn from(disk: sled_agent_client::types::InventoryDisk) -> PhysicalDisk {
+        PhysicalDisk {
+            identity: disk.identity.into(),
+            variant: disk.variant.into(),
+            slot: disk.slot,
+        }
+    }
+}
+
 /// Inventory reported by sled agent
 ///
 /// This is a software notion of a sled, distinct from an underlying baseboard.
@@ -361,6 +384,7 @@ pub struct SledAgent {
     pub usable_hardware_threads: u32,
     pub usable_physical_ram: ByteCount,
     pub reservoir_size: ByteCount,
+    pub disks: Vec<PhysicalDisk>,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json
index 43fac710fc..8de8be430f 100644
--- a/openapi/sled-agent.json
+++ b/openapi/sled-agent.json
@@ -3506,6 +3506,26 @@
           "target"
         ]
       },
+      "DiskIdentity": {
+        "description": "Uniquely identifies a disk.",
+        "type": "object",
+        "properties": {
+          "model": {
+            "type": "string"
+          },
+          "serial": {
+            "type": "string"
+          },
+          "vendor": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "model",
+          "serial",
+          "vendor"
+        ]
+      },
       "DiskRequest": {
         "description": "DiskRequest\n\n<details><summary>JSON schema</summary>\n\n```json { \"type\": \"object\", \"required\": [ \"device\", \"name\", \"read_only\", \"slot\", \"volume_construction_request\" ], \"properties\": { \"device\": { \"type\": \"string\" }, \"name\": { \"type\": \"string\" }, \"read_only\": { \"type\": \"boolean\" }, \"slot\": { \"$ref\": \"#/components/schemas/Slot\" }, \"volume_construction_request\": { \"$ref\": \"#/components/schemas/VolumeConstructionRequest\" } } } ``` </details>",
         "type": "object",
@@ -3839,6 +3859,13 @@
           "M2"
         ]
       },
+      "DiskVariant": {
+        "type": "string",
+        "enum": [
+          "U2",
+          "M2"
+        ]
+      },
       "Duration": {
         "type": "object",
         "properties": {
@@ -5214,6 +5241,12 @@
           "baseboard": {
             "$ref": "#/components/schemas/Baseboard"
           },
+          "disks": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/InventoryDisk"
+            }
+          },
           "reservoir_size": {
             "$ref": "#/components/schemas/ByteCount"
           },
@@ -5238,6 +5271,7 @@
         },
         "required": [
           "baseboard",
+          "disks",
           "reservoir_size",
           "sled_agent_address",
           "sled_id",
@@ -5246,6 +5280,27 @@
           "usable_physical_ram"
         ]
       },
+      "InventoryDisk": {
+        "description": "Identifies information about disks which may be attached to Sleds.",
+        "type": "object",
+        "properties": {
+          "identity": {
+            "$ref": "#/components/schemas/DiskIdentity"
+          },
+          "slot": {
+            "type": "integer",
+            "format": "int64"
+          },
+          "variant": {
+            "$ref": "#/components/schemas/DiskVariant"
+          }
+        },
+        "required": [
+          "identity",
+          "slot",
+          "variant"
+        ]
+      },
       "IpNet": {
         "oneOf": [
           {
diff --git a/schema/crdb/41.0.0/up01.sql b/schema/crdb/41.0.0/up01.sql
new file mode 100644
index 0000000000..75270b7dc1
--- /dev/null
+++ b/schema/crdb/41.0.0/up01.sql
@@ -0,0 +1,14 @@
+CREATE TABLE IF NOT EXISTS omicron.public.inv_physical_disk (
+    inv_collection_id UUID NOT NULL,
+
+    sled_id UUID NOT NULL,
+    slot INT8 CHECK (slot >= 0) NOT NULL,
+
+    vendor STRING(63) NOT NULL,
+    model STRING(63) NOT NULL,
+    serial STRING(63) NOT NULL,
+
+    variant omicron.public.physical_disk_kind NOT NULL,
+
+    PRIMARY KEY (inv_collection_id, sled_id, slot)
+);
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 27faf3f79f..99d8fea65b 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -2962,6 +2962,30 @@ CREATE TABLE IF NOT EXISTS omicron.public.inv_sled_agent (
     PRIMARY KEY (inv_collection_id, sled_id)
 );
 
+CREATE TABLE IF NOT EXISTS omicron.public.inv_physical_disk (
+    -- where this observation came from
+    -- (foreign key into `inv_collection` table)
+    inv_collection_id UUID NOT NULL,
+
+    -- unique id for this sled (should be foreign keys into `sled` table, though
+    -- it's conceivable a sled will report an id that we don't know about)
+    sled_id UUID NOT NULL,
+    -- The slot where this disk was last observed
+    slot INT8 CHECK (slot >= 0) NOT NULL,
+
+    vendor STRING(63) NOT NULL,
+    model STRING(63) NOT NULL,
+    serial STRING(63) NOT NULL,
+
+    variant omicron.public.physical_disk_kind NOT NULL,
+
+    -- FK consisting of:
+    -- - Which collection this was
+    -- - The sled reporting the disk
+    -- - The slot in which this disk was found
+    PRIMARY KEY (inv_collection_id, sled_id, slot)
+);
+
 CREATE TABLE IF NOT EXISTS omicron.public.inv_sled_omicron_zones (
     -- where this observation came from
     -- (foreign key into `inv_collection` table)
@@ -3575,7 +3599,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '40.0.0', NULL)
+    ( TRUE, NOW(), NOW(), '41.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;
diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs
index 6ce9d59b1c..bf1102d897 100644
--- a/sled-agent/src/http_entrypoints.rs
+++ b/sled-agent/src/http_entrypoints.rs
@@ -972,7 +972,7 @@ async fn inventory(
     request_context: RequestContext<SledAgent>,
 ) -> Result<HttpResponseOk<Inventory>, HttpError> {
     let sa = request_context.context();
-    Ok(HttpResponseOk(sa.inventory()?))
+    Ok(HttpResponseOk(sa.inventory().await?))
 }
 
 /// Get the internal state of the local bootstore node
diff --git a/sled-agent/src/long_running_tasks.rs b/sled-agent/src/long_running_tasks.rs
index f4a665c098..3b29bdda60 100644
--- a/sled-agent/src/long_running_tasks.rs
+++ b/sled-agent/src/long_running_tasks.rs
@@ -228,13 +228,14 @@ async fn upsert_synthetic_zpools_if_needed(
     config: &Config,
 ) {
     if let Some(pools) = &config.zpools {
-        for pool in pools {
+        for (i, pool) in pools.iter().enumerate() {
             info!(
                 log,
                 "Upserting synthetic zpool to Storage Manager: {}",
                 pool.to_string()
             );
-            let disk = SyntheticDisk::new(pool.clone()).into();
+            let disk =
+                SyntheticDisk::new(pool.clone(), i.try_into().unwrap()).into();
             storage_manager.upsert_disk(disk).await;
         }
     }
diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs
index 82813d3f1e..51951465fd 100644
--- a/sled-agent/src/params.rs
+++ b/sled-agent/src/params.rs
@@ -872,6 +872,14 @@ pub enum InstanceExternalIpBody {
 // becomes easier to maintain a separate copy, we should do that.
 pub type SledRole = nexus_client::types::SledRole;
 
+/// Identifies information about disks which may be attached to Sleds.
+#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]
+pub struct InventoryDisk {
+    pub identity: omicron_common::disk::DiskIdentity,
+    pub variant: sled_hardware::DiskVariant,
+    pub slot: i64,
+}
+
 /// Identity and basic status information about this sled agent
 #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]
 pub struct Inventory {
@@ -882,6 +890,7 @@ pub struct Inventory {
     pub usable_hardware_threads: u32,
     pub usable_physical_ram: ByteCount,
     pub reservoir_size: ByteCount,
+    pub disks: Vec<InventoryDisk>,
 }
 
 #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
index 11ccfad69b..e0a4693877 100644
--- a/sled-agent/src/services.rs
+++ b/sled-agent/src/services.rs
@@ -4374,11 +4374,11 @@ mod test {
 
         let internal_zpool_name = ZpoolName::new_internal(Uuid::new_v4());
         let internal_disk: RawDisk =
-            SyntheticDisk::new(internal_zpool_name).into();
+            SyntheticDisk::new(internal_zpool_name, 0).into();
         handle.upsert_disk(internal_disk).await;
         let external_zpool_name = ZpoolName::new_external(Uuid::new_v4());
         let external_disk: RawDisk =
-            SyntheticDisk::new(external_zpool_name).into();
+            SyntheticDisk::new(external_zpool_name, 1).into();
         handle.upsert_disk(external_disk).await;
 
         handle
diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs
index 494db58828..c886c9534b 100644
--- a/sled-agent/src/sim/http_entrypoints.rs
+++ b/sled-agent/src/sim/http_entrypoints.rs
@@ -435,6 +435,7 @@ async fn inventory(
     let sa = rqctx.context();
     Ok(HttpResponseOk(
         sa.inventory(rqctx.server.local_addr)
+            .await
             .map_err(|e| HttpError::for_internal_error(format!("{:#}", e)))?,
     ))
 }
diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs
index 0b90bef590..425f4e31a4 100644
--- a/sled-agent/src/sim/sled_agent.rs
+++ b/sled-agent/src/sim/sled_agent.rs
@@ -26,7 +26,6 @@ use futures::lock::Mutex;
 use illumos_utils::opte::params::{
     DeleteVirtualNetworkInterfaceHost, SetVirtualNetworkInterfaceHost,
 };
-use nexus_client::types::PhysicalDiskKind;
 use omicron_common::api::external::{
     ByteCount, DiskState, Error, Generation, ResourceType,
 };
@@ -503,7 +502,7 @@ impl SledAgent {
         serial: String,
         model: String,
     ) {
-        let variant = PhysicalDiskKind::U2;
+        let variant = sled_hardware::DiskVariant::U2;
         self.storage
             .lock()
             .await
@@ -730,7 +729,10 @@ impl SledAgent {
         Ok(addr)
     }
 
-    pub fn inventory(&self, addr: SocketAddr) -> anyhow::Result<Inventory> {
+    pub async fn inventory(
+        &self,
+        addr: SocketAddr,
+    ) -> anyhow::Result<Inventory> {
         let sled_agent_address = match addr {
             SocketAddr::V4(_) => {
                 bail!("sled_agent_ip must be v6 for inventory")
@@ -751,6 +753,18 @@ impl SledAgent {
                 self.config.hardware.reservoir_ram,
             )
             .context("reservoir_size")?,
+            disks: self
+                .storage
+                .lock()
+                .await
+                .physical_disks()
+                .iter()
+                .map(|(identity, info)| crate::params::InventoryDisk {
+                    identity: identity.clone(),
+                    variant: info.variant,
+                    slot: info.slot,
+                })
+                .collect(),
         })
     }
 
diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs
index 101228934d..d50cd296ae 100644
--- a/sled-agent/src/sim/storage.rs
+++ b/sled-agent/src/sim/storage.rs
@@ -22,7 +22,9 @@ use futures::lock::Mutex;
 use nexus_client::types::{
     ByteCount, PhysicalDiskKind, PhysicalDiskPutRequest, ZpoolPutRequest,
 };
+use omicron_common::disk::DiskIdentity;
 use propolis_client::types::VolumeConstructionRequest;
+use sled_hardware::DiskVariant;
 use slog::Logger;
 use std::collections::HashMap;
 use std::collections::HashSet;
@@ -471,8 +473,9 @@ impl CrucibleServer {
     }
 }
 
-struct PhysicalDisk {
-    _variant: PhysicalDiskKind,
+pub(crate) struct PhysicalDisk {
+    pub(crate) variant: DiskVariant,
+    pub(crate) slot: i64,
 }
 
 struct Zpool {
@@ -530,19 +533,13 @@ impl Zpool {
     }
 }
 
-#[derive(Clone, PartialEq, Eq, Hash)]
-struct DiskName {
-    vendor: String,
-    serial: String,
-    model: String,
-}
-
 /// Simulated representation of all storage on a sled.
 pub struct Storage {
     sled_id: Uuid,
     nexus_client: Arc<NexusClient>,
     log: Logger,
-    physical_disks: HashMap<DiskName, PhysicalDisk>,
+    physical_disks: HashMap<DiskIdentity, PhysicalDisk>,
+    next_disk_slot: i64,
     zpools: HashMap<Uuid, Zpool>,
     crucible_ip: IpAddr,
     next_crucible_port: u16,
@@ -560,26 +557,38 @@ impl Storage {
             nexus_client,
             log,
             physical_disks: HashMap::new(),
+            next_disk_slot: 0,
             zpools: HashMap::new(),
             crucible_ip,
             next_crucible_port: 100,
         }
     }
 
+    /// Returns an immutable reference to all (currently known) physical disks
+    pub fn physical_disks(&self) -> &HashMap<DiskIdentity, PhysicalDisk> {
+        &self.physical_disks
+    }
+
     pub async fn insert_physical_disk(
         &mut self,
         vendor: String,
         serial: String,
         model: String,
-        variant: PhysicalDiskKind,
+        variant: DiskVariant,
     ) {
-        let identifier = DiskName {
+        let identifier = DiskIdentity {
             vendor: vendor.clone(),
             serial: serial.clone(),
             model: model.clone(),
         };
-        self.physical_disks
-            .insert(identifier, PhysicalDisk { _variant: variant });
+        let slot = self.next_disk_slot;
+        self.next_disk_slot += 1;
+        self.physical_disks.insert(identifier, PhysicalDisk { variant, slot });
+
+        let variant = match variant {
+            DiskVariant::U2 => PhysicalDiskKind::U2,
+            DiskVariant::M2 => PhysicalDiskKind::M2,
+        };
 
         // Notify Nexus
         let request = PhysicalDiskPutRequest {
diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs
index cff13dd43a..91ddfde153 100644
--- a/sled-agent/src/sled_agent.rs
+++ b/sled-agent/src/sled_agent.rs
@@ -1092,7 +1092,7 @@ impl SledAgent {
     ///
     /// This is basically a GET version of the information we push to Nexus on
     /// startup.
-    pub(crate) fn inventory(&self) -> Result<Inventory, InventoryError> {
+    pub(crate) async fn inventory(&self) -> Result<Inventory, InventoryError> {
         let sled_id = self.inner.id;
         let sled_agent_address = self.inner.sled_address();
         let is_scrimlet = self.inner.hardware.is_scrimlet();
@@ -1107,6 +1107,18 @@ impl SledAgent {
         } else {
             crate::params::SledRole::Gimlet
         };
+        let disks = self
+            .storage()
+            .get_latest_resources()
+            .await
+            .disks()
+            .iter()
+            .map(|(identity, (disk, _pool))| crate::params::InventoryDisk {
+                identity: identity.clone(),
+                variant: disk.variant(),
+                slot: disk.slot(),
+            })
+            .collect();
 
         Ok(Inventory {
             sled_id,
@@ -1116,6 +1128,7 @@ impl SledAgent {
             usable_hardware_threads,
             usable_physical_ram: ByteCount::try_from(usable_physical_ram)?,
             reservoir_size,
+            disks,
         })
     }
 }
diff --git a/sled-agent/src/zone_bundle.rs b/sled-agent/src/zone_bundle.rs
index f9e1c7b3be..4c4cbf5490 100644
--- a/sled-agent/src/zone_bundle.rs
+++ b/sled-agent/src/zone_bundle.rs
@@ -2236,10 +2236,10 @@ mod illumos_tests {
         });
 
         // These must be internal zpools
-        for _ in 0..2 {
+        for i in 0..2 {
             let internal_zpool_name = ZpoolName::new_internal(Uuid::new_v4());
             let internal_disk: RawDisk =
-                SyntheticDisk::new(internal_zpool_name.clone()).into();
+                SyntheticDisk::new(internal_zpool_name.clone(), i).into();
             handle.upsert_disk(internal_disk).await;
         }
         handle
diff --git a/sled-hardware/src/disk.rs b/sled-hardware/src/disk.rs
index 6a4c968f14..a649b205e1 100644
--- a/sled-hardware/src/disk.rs
+++ b/sled-hardware/src/disk.rs
@@ -8,6 +8,8 @@ use illumos_utils::zpool::Zpool;
 use illumos_utils::zpool::ZpoolKind;
 use illumos_utils::zpool::ZpoolName;
 use omicron_common::disk::DiskIdentity;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
 use slog::Logger;
 use slog::{info, warn};
 use uuid::Uuid;
@@ -303,8 +305,9 @@ impl PooledDisk {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-#[allow(dead_code)]
+#[derive(
+    Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema,
+)]
 pub enum DiskVariant {
     U2,
     M2,
diff --git a/sled-storage/src/disk.rs b/sled-storage/src/disk.rs
index cfe189a409..705b38718a 100644
--- a/sled-storage/src/disk.rs
+++ b/sled-storage/src/disk.rs
@@ -31,14 +31,24 @@ pub enum DiskError {
 pub struct SyntheticDisk {
     pub identity: DiskIdentity,
     pub zpool_name: ZpoolName,
+    pub slot: i64,
 }
 
+// By adding slots at an "offset", this acts as a barrier against synthetic
+// disks overlapping with "real disk" in a mixed-disk deployment.
+//
+// This shouldn't happen in prod, and is an unlikely test-only scenario, but
+// we'd still like to protect against it, since it could confuse the inventory
+// system.
+const SYNTHETIC_SLOT_OFFSET: i64 = 1024;
+
 impl SyntheticDisk {
     // Create a zpool and import it for the synthetic disk
     // Zpools willl be set to the min size of 64Mib
     pub fn create_zpool(
         dir: &Utf8Path,
         zpool_name: &ZpoolName,
+        slot: i64,
     ) -> SyntheticDisk {
         // 64 MiB (min size of zpool)
         const DISK_SIZE: u64 = 64 * 1024 * 1024;
@@ -49,17 +59,21 @@ impl SyntheticDisk {
         Zpool::create(zpool_name, &path).unwrap();
         Zpool::import(zpool_name).unwrap();
         Zpool::set_failmode_continue(zpool_name).unwrap();
-        Self::new(zpool_name.clone())
+        Self::new(zpool_name.clone(), slot)
     }
 
-    pub fn new(zpool_name: ZpoolName) -> SyntheticDisk {
+    pub fn new(zpool_name: ZpoolName, slot: i64) -> SyntheticDisk {
         let id = zpool_name.id();
         let identity = DiskIdentity {
             vendor: "synthetic-vendor".to_string(),
             serial: format!("synthetic-serial-{id}"),
             model: "synthetic-model".to_string(),
         };
-        SyntheticDisk { identity, zpool_name }
+        SyntheticDisk {
+            identity,
+            zpool_name,
+            slot: slot + SYNTHETIC_SLOT_OFFSET,
+        }
     }
 }
 
@@ -247,7 +261,7 @@ impl Disk {
     pub fn slot(&self) -> i64 {
         match self {
             Self::Real(disk) => disk.slot,
-            Self::Synthetic(_) => unreachable!(),
+            Self::Synthetic(disk) => disk.slot,
         }
     }
 }
diff --git a/sled-storage/src/manager.rs b/sled-storage/src/manager.rs
index 547c4ec2d7..bb749cc366 100644
--- a/sled-storage/src/manager.rs
+++ b/sled-storage/src/manager.rs
@@ -686,7 +686,7 @@ mod tests {
             KeyManager::new(&logctx.log, HardcodedSecretRetriever::default());
         let (mut manager, _) = StorageManager::new(&logctx.log, key_requester);
         let zpool_name = ZpoolName::new_external(Uuid::new_v4());
-        let raw_disk: RawDisk = SyntheticDisk::new(zpool_name).into();
+        let raw_disk: RawDisk = SyntheticDisk::new(zpool_name, 0).into();
         assert_eq!(StorageManagerState::WaitingForKeyManager, manager.state);
         manager.add_u2_disk(raw_disk.clone()).await.unwrap();
         assert!(manager.resources.all_u2_zpools().is_empty());
@@ -710,7 +710,8 @@ mod tests {
         let (mut manager, _) = StorageManager::new(&logctx.log, key_requester);
         let zpool_name = ZpoolName::new_external(Uuid::new_v4());
         let dir = tempdir().unwrap();
-        let disk = SyntheticDisk::create_zpool(dir.path(), &zpool_name).into();
+        let disk =
+            SyntheticDisk::create_zpool(dir.path(), &zpool_name, 0).into();
 
         // Spawn the key_manager so that it will respond to requests for encryption keys
         tokio::spawn(async move { key_manager.run().await });
@@ -742,7 +743,8 @@ mod tests {
         // Create a synthetic internal disk
         let zpool_name = ZpoolName::new_internal(Uuid::new_v4());
         let dir = tempdir().unwrap();
-        let disk = SyntheticDisk::create_zpool(dir.path(), &zpool_name).into();
+        let disk =
+            SyntheticDisk::create_zpool(dir.path(), &zpool_name, 0).into();
 
         handle.upsert_disk(disk).await;
         handle.wait_for_boot_disk().await;
@@ -770,7 +772,8 @@ mod tests {
         // the `KeyManager` is ready yet.
         let zpool_name = ZpoolName::new_external(Uuid::new_v4());
         let dir = tempdir().unwrap();
-        let disk = SyntheticDisk::create_zpool(dir.path(), &zpool_name).into();
+        let disk =
+            SyntheticDisk::create_zpool(dir.path(), &zpool_name, 0).into();
         handle.upsert_disk(disk).await;
         let resources = handle.get_latest_resources().await;
         assert!(resources.all_u2_zpools().is_empty());
@@ -808,7 +811,8 @@ mod tests {
         // the `KeyManager` is ready yet.
         let zpool_name = ZpoolName::new_external(Uuid::new_v4());
         let dir = tempdir().unwrap();
-        let disk = SyntheticDisk::create_zpool(dir.path(), &zpool_name).into();
+        let disk =
+            SyntheticDisk::create_zpool(dir.path(), &zpool_name, 0).into();
         handle.upsert_disk(disk).await;
         manager.step().await.unwrap();
 
@@ -865,7 +869,7 @@ mod tests {
         let zpool_name = ZpoolName::new_external(Uuid::new_v4());
         let dir = tempdir().unwrap();
         let disk: RawDisk =
-            SyntheticDisk::create_zpool(dir.path(), &zpool_name).into();
+            SyntheticDisk::create_zpool(dir.path(), &zpool_name, 0).into();
         handle.upsert_disk(disk.clone()).await;
 
         // Wait for the add disk notification
@@ -904,8 +908,14 @@ mod tests {
             (0..10).map(|_| ZpoolName::new_external(Uuid::new_v4())).collect();
         let disks: Vec<RawDisk> = zpools
             .iter()
-            .map(|zpool_name| {
-                SyntheticDisk::create_zpool(dir.path(), zpool_name).into()
+            .enumerate()
+            .map(|(slot, zpool_name)| {
+                SyntheticDisk::create_zpool(
+                    dir.path(),
+                    zpool_name,
+                    slot.try_into().unwrap(),
+                )
+                .into()
             })
             .collect();
 
@@ -1020,7 +1030,7 @@ mod tests {
         let zpool_name = ZpoolName::new_external(Uuid::new_v4());
         let dir = tempdir().unwrap();
         let disk: RawDisk =
-            SyntheticDisk::create_zpool(dir.path(), &zpool_name).into();
+            SyntheticDisk::create_zpool(dir.path(), &zpool_name, 0).into();
         handle.upsert_disk(disk.clone()).await;
 
         // Create a filesystem

From c7ac255b56732e2d7e689f92994746f9364371a7 Mon Sep 17 00:00:00 2001
From: James MacMahon <james@oxide.computer>
Date: Wed, 13 Mar 2024 17:09:48 -0400
Subject: [PATCH 110/157] One shall be the number of `db_metadata` (#5256)

Two thou shalt not count.

Three is right out.
---
 schema/crdb/dbinit.sql | 69 +++++++++++++++++-------------------------
 1 file changed, 27 insertions(+), 42 deletions(-)

diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 99d8fea65b..7c3b8fcc34 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -3308,25 +3308,6 @@ CREATE TABLE IF NOT EXISTS omicron.public.bp_omicron_zones_not_in_service (
 COMMIT;
 BEGIN;
 
-CREATE TABLE IF NOT EXISTS omicron.public.db_metadata (
-    -- There should only be one row of this table for the whole DB.
-    -- It's a little goofy, but filter on "singleton = true" before querying
-    -- or applying updates, and you'll access the singleton row.
-    --
-    -- We also add a constraint on this table to ensure it's not possible to
-    -- access the version of this table with "singleton = false".
-    singleton BOOL NOT NULL PRIMARY KEY,
-    time_created TIMESTAMPTZ NOT NULL,
-    time_modified TIMESTAMPTZ NOT NULL,
-    -- Semver representation of the DB version
-    version STRING(64) NOT NULL,
-
-    -- (Optional) Semver representation of the DB version to which we're upgrading
-    target_version STRING(64),
-
-    CHECK (singleton = true)
-);
-
 -- Per-VMM state.
 CREATE TABLE IF NOT EXISTS omicron.public.vmm (
     id UUID PRIMARY KEY,
@@ -3471,29 +3452,6 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_bfd_session ON omicron.public.bfd_sessi
     switch
 ) WHERE time_deleted IS NULL;
 
-/*
- * Metadata for the schema itself. This version number isn't great, as there's
- * nothing to ensure it gets bumped when it should be, but it's a start.
- */
-CREATE TABLE IF NOT EXISTS omicron.public.db_metadata (
-    -- There should only be one row of this table for the whole DB.
-    -- It's a little goofy, but filter on "singleton = true" before querying
-    -- or applying updates, and you'll access the singleton row.
-    --
-    -- We also add a constraint on this table to ensure it's not possible to
-    -- access the version of this table with "singleton = false".
-    singleton BOOL NOT NULL PRIMARY KEY,
-    time_created TIMESTAMPTZ NOT NULL,
-    time_modified TIMESTAMPTZ NOT NULL,
-    -- Semver representation of the DB version
-    version STRING(64) NOT NULL,
-
-    -- (Optional) Semver representation of the DB version to which we're upgrading
-    target_version STRING(64),
-
-    CHECK (singleton = true)
-);
-
 ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS autoneg BOOL NOT NULL DEFAULT false;
 
 CREATE INDEX IF NOT EXISTS ipv4_nat_lookup_by_vni ON omicron.public.ipv4_nat_entry (
@@ -3592,6 +3550,33 @@ ALTER TABLE omicron.public.external_ip ADD COLUMN IF NOT EXISTS is_probe BOOL NO
 
 ALTER TYPE omicron.public.network_interface_kind ADD VALUE IF NOT EXISTS 'probe';
 
+/*
+ * Metadata for the schema itself. This version number isn't great, as there's
+ * nothing to ensure it gets bumped when it should be, but it's a start.
+ */
+CREATE TABLE IF NOT EXISTS omicron.public.db_metadata (
+    -- There should only be one row of this table for the whole DB.
+    -- It's a little goofy, but filter on "singleton = true" before querying
+    -- or applying updates, and you'll access the singleton row.
+    --
+    -- We also add a constraint on this table to ensure it's not possible to
+    -- access the version of this table with "singleton = false".
+    singleton BOOL NOT NULL PRIMARY KEY,
+    time_created TIMESTAMPTZ NOT NULL,
+    time_modified TIMESTAMPTZ NOT NULL,
+    -- Semver representation of the DB version
+    version STRING(64) NOT NULL,
+
+    -- (Optional) Semver representation of the DB version to which we're upgrading
+    target_version STRING(64),
+
+    CHECK (singleton = true)
+);
+
+/*
+ * Keep this at the end of file so that the database does not contain a version
+ * until it is fully populated.
+ */
 INSERT INTO omicron.public.db_metadata (
     singleton,
     time_created,

From 5f5790ca9cabd59627ed4366f9aaf51f7c358ac2 Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Wed, 13 Mar 2024 18:11:01 -0400
Subject: [PATCH 111/157] Add internal API and omdb commands to toggle current
 target blueprint's enabled bit (#5231)

I chose `PATCH` for the new endpoint; I think that's correct? Although
it's the first use of `PATCH` in omicron AFAICT so maybe it should just
be `PUT`?

There are a couple related-but-not-required changes as a part of this:

* `omdb nexus blueprints list` now has an `ENA: yes/no` column that is
only populated for the current target
* We activate the blueprint execution bg task when setting an enabled
target or changing the current target to be enabled. (The latter might
be superfluous if the target was already enabled, but that's fine.)

I tried this locally against `omicron-dev run-all` and it all seemed to
check out. If folks would be more comfortable with a test on a real
system before merging I can do that; otherwise I'll test it next time I
take a lap for some other reason.

Closes #5210.
---
 dev-tools/omdb/src/bin/omdb/nexus.rs          |  70 +++++-
 .../db-queries/src/db/datastore/deployment.rs | 211 +++++++++++++++++-
 nexus/src/app/background/blueprint_load.rs    |  58 +++--
 nexus/src/app/deployment.rs                   |  29 ++-
 nexus/src/internal_api/http_entrypoints.rs    |  21 ++
 openapi/nexus-internal.json                   |  34 +++
 6 files changed, 398 insertions(+), 25 deletions(-)

diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs
index f27e84f4a2..3d63f48469 100644
--- a/dev-tools/omdb/src/bin/omdb/nexus.rs
+++ b/dev-tools/omdb/src/bin/omdb/nexus.rs
@@ -128,6 +128,14 @@ enum BlueprintTargetCommands {
     Show,
     /// Change the current target blueprint
     Set(BlueprintTargetSetArgs),
+    /// Enable the current target blueprint
+    ///
+    /// Fails if the specified blueprint id is not the current target
+    Enable(BlueprintIdArgs),
+    /// Disable the current target blueprint
+    ///
+    /// Fails if the specified blueprint id is not the current target
+    Disable(BlueprintIdArgs),
 }
 
 #[derive(Debug, Args)]
@@ -254,6 +262,30 @@ impl NexusArgs {
                 let token = omdb.check_allow_destructive()?;
                 cmd_nexus_blueprints_target_set(&client, args, token).await
             }
+            NexusCommands::Blueprints(BlueprintsArgs {
+                command:
+                    BlueprintsCommands::Target(BlueprintsTargetArgs {
+                        command: BlueprintTargetCommands::Enable(args),
+                    }),
+            }) => {
+                let token = omdb.check_allow_destructive()?;
+                cmd_nexus_blueprints_target_set_enabled(
+                    &client, args, true, token,
+                )
+                .await
+            }
+            NexusCommands::Blueprints(BlueprintsArgs {
+                command:
+                    BlueprintsCommands::Target(BlueprintsTargetArgs {
+                        command: BlueprintTargetCommands::Disable(args),
+                    }),
+            }) => {
+                let token = omdb.check_allow_destructive()?;
+                cmd_nexus_blueprints_target_set_enabled(
+                    &client, args, false, token,
+                )
+                .await
+            }
             NexusCommands::Blueprints(BlueprintsArgs {
                 command: BlueprintsCommands::Regenerate,
             }) => {
@@ -858,13 +890,15 @@ async fn cmd_nexus_blueprints_list(
     struct BlueprintRow {
         #[tabled(rename = "T")]
         is_target: &'static str,
+        #[tabled(rename = "ENA")]
+        enabled: &'static str,
         id: String,
         parent: String,
         time_created: String,
     }
 
-    let target_id = match client.blueprint_target_view().await {
-        Ok(result) => Some(result.into_inner().target_id),
+    let target = match client.blueprint_target_view().await {
+        Ok(result) => Some(result.into_inner()),
         Err(error) => {
             // This request will fail if there's no target configured, so it's
             // not necessarily a big deal.
@@ -883,13 +917,17 @@ async fn cmd_nexus_blueprints_list(
         .context("listing blueprints")?
         .into_iter()
         .map(|blueprint| {
-            let is_target = match target_id {
-                Some(target_id) if target_id == blueprint.id => "*",
-                _ => "",
+            let (is_target, enabled) = match &target {
+                Some(target) if target.target_id == blueprint.id => {
+                    let enabled = if target.enabled { "yes" } else { "no" };
+                    ("*", enabled)
+                }
+                _ => ("", ""),
             };
 
             BlueprintRow {
                 is_target,
+                enabled,
                 id: blueprint.id.to_string(),
                 parent: blueprint
                     .parent_blueprint_id
@@ -1000,6 +1038,28 @@ async fn cmd_nexus_blueprints_target_set(
     Ok(())
 }
 
+async fn cmd_nexus_blueprints_target_set_enabled(
+    client: &nexus_client::Client,
+    args: &BlueprintIdArgs,
+    enabled: bool,
+    _destruction_token: DestructiveOperationToken,
+) -> Result<(), anyhow::Error> {
+    let description = if enabled { "enabled" } else { "disabled" };
+    client
+        .blueprint_target_set_enabled(
+            &nexus_client::types::BlueprintTargetSet {
+                target_id: args.blueprint_id,
+                enabled,
+            },
+        )
+        .await
+        .with_context(|| {
+            format!("setting blueprint {} to {description}", args.blueprint_id)
+        })?;
+    eprintln!("set target blueprint {} to {description}", args.blueprint_id);
+    Ok(())
+}
+
 async fn cmd_nexus_blueprints_generate_from_collection(
     client: &nexus_client::Client,
     args: &CollectionIdArgs,
diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs
index 7e80f1be6b..c2b4e998ce 100644
--- a/nexus/db-queries/src/db/datastore/deployment.rs
+++ b/nexus/db-queries/src/db/datastore/deployment.rs
@@ -29,6 +29,8 @@ use diesel::result::Error as DieselError;
 use diesel::sql_types;
 use diesel::Column;
 use diesel::ExpressionMethods;
+use diesel::Insertable;
+use diesel::IntoSql;
 use diesel::OptionalExtension;
 use diesel::QueryDsl;
 use diesel::RunQueryDsl;
@@ -615,7 +617,9 @@ impl DataStore {
     /// Set the current target blueprint
     ///
     /// In order to become the target blueprint, `target`'s parent blueprint
-    /// must be the current target
+    /// must be the current target. To instead change the current target's
+    /// properties (particularly whether it's enabled), use
+    /// [`DataStore::blueprint_target_set_current_enabled`].
     pub async fn blueprint_target_set_current(
         &self,
         opctx: &OpContext,
@@ -641,6 +645,96 @@ impl DataStore {
         Ok(())
     }
 
+    /// Set the current target blueprint's `enabled` field
+    ///
+    /// In order to change the enabled field, `target` must already be the
+    /// current target blueprint. To instead set a new blueprint target, use
+    /// [`DataStore::blueprint_target_set_current`].
+    // Although this function is like `blueprint_target_set_current()` in that
+    // both store the given `BlueprintTarget` into the table, the functions are
+    // distinct because the preconditions and error cases are different. We
+    // could reconsider this and make `blueprint_target_set_current` accept
+    // blueprints where either their own or their parent is the current
+    // blueprint, although this would require some rework in the nontrivial
+    // `InsertTargetQuery` CTE.
+    pub async fn blueprint_target_set_current_enabled(
+        &self,
+        opctx: &OpContext,
+        target: BlueprintTarget,
+    ) -> Result<(), Error> {
+        use db::schema::bp_target::dsl;
+
+        opctx
+            .authorize(authz::Action::Modify, &authz::BLUEPRINT_CONFIG)
+            .await?;
+
+        // Diesel requires us to use an alias in order to refer to the
+        // `bp_target` table twice in the same query.
+        let bp_target2 = diesel::alias!(db::schema::bp_target as bp_target1);
+
+        // The following diesel produces this query:
+        //
+        // ```sql
+        // INSERT INTO bp_target
+        //   (SELECT
+        //        version + 1,
+        //        blueprint_id,
+        //        <target.enabled>,
+        //        <target.time_made_target>
+        //    FROM bp_target
+        //    WHERE
+        //        -- This part of the subquery restricts us to only the
+        //        -- current target (i.e., the bp_target with maximal version)
+        //        version IN (SELECT version FROM bp_target
+        //                    ORDER BY version DESC LIMIT 1)
+        //
+        //        -- ... and that current target must exactly equal the target
+        //        -- blueprint on which we're trying to set `enabled`
+        //        AND blueprint_id = <target.blueprint_id>
+        //   );
+        // ```
+        //
+        // This will either insert one new row (if the filters were satisified)
+        // or no new rows (if the filters were not satisfied).
+        let query = dsl::bp_target
+            .select((
+                dsl::version + 1,
+                dsl::blueprint_id,
+                target.enabled.into_sql::<sql_types::Bool>(),
+                target.time_made_target.into_sql::<sql_types::Timestamptz>(),
+            ))
+            .filter(
+                dsl::version.eq_any(
+                    bp_target2
+                        .select(bp_target2.field(dsl::version))
+                        .order_by(bp_target2.field(dsl::version).desc())
+                        .limit(1),
+                ),
+            )
+            .filter(dsl::blueprint_id.eq(target.target_id))
+            .insert_into(dsl::bp_target);
+
+        let conn = self.pool_connection_authorized(opctx).await?;
+
+        let num_inserted = query
+            .execute_async(&*conn)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        match num_inserted {
+            0 => Err(Error::invalid_request(format!(
+                "Blueprint {} is not the current target blueprint",
+                target.target_id
+            ))),
+            1 => Ok(()),
+            // This is impossible, not only due to the `.limit(1)` in the
+            // subquery above, but also because we're inserting `version + 1`
+            // which would fail with pkey conflicts if we matched more than one
+            // existing row in the subquery.
+            _ => unreachable!("query inserted more than one row"),
+        }
+    }
+
     /// Get the current target blueprint, if one exists
     ///
     /// Returns both the metadata about the target and the full blueprint
@@ -1626,4 +1720,119 @@ mod tests {
         db.cleanup().await.unwrap();
         logctx.cleanup_successful();
     }
+
+    #[tokio::test]
+    async fn test_set_target_enabled() {
+        // Setup
+        let logctx = dev::test_setup_log("test_set_target_enabled");
+        let mut db = test_setup_database(&logctx.log).await;
+        let (opctx, datastore) = datastore_test(&logctx, &db).await;
+
+        // Create an initial blueprint and a child.
+        let collection =
+            nexus_inventory::CollectionBuilder::new("test").build();
+        let blueprint1 = BlueprintBuilder::build_initial_from_collection(
+            &collection,
+            Generation::new(),
+            &EMPTY_POLICY,
+            "test1",
+        )
+        .unwrap();
+        let blueprint2 = BlueprintBuilder::new_based_on(
+            &logctx.log,
+            &blueprint1,
+            Generation::new(),
+            &EMPTY_POLICY,
+            "test2",
+        )
+        .expect("failed to create builder")
+        .build();
+        assert_eq!(blueprint1.parent_blueprint_id, None);
+        assert_eq!(blueprint2.parent_blueprint_id, Some(blueprint1.id));
+
+        // Insert both into the blueprint table.
+        datastore.blueprint_insert(&opctx, &blueprint1).await.unwrap();
+        datastore.blueprint_insert(&opctx, &blueprint2).await.unwrap();
+
+        let mut bp1_target = BlueprintTarget {
+            target_id: blueprint1.id,
+            enabled: true,
+            time_made_target: now_db_precision(),
+        };
+        let mut bp2_target = BlueprintTarget {
+            target_id: blueprint2.id,
+            enabled: true,
+            time_made_target: now_db_precision(),
+        };
+
+        // Set bp1_target as the current target.
+        datastore
+            .blueprint_target_set_current(&opctx, bp1_target)
+            .await
+            .unwrap();
+        assert_eq!(
+            datastore.blueprint_target_get_current(&opctx).await.unwrap(),
+            Some(bp1_target),
+        );
+
+        // We should be able to toggle its enabled status an arbitrary number of
+        // times.
+        for _ in 0..10 {
+            bp1_target.enabled = !bp1_target.enabled;
+            datastore
+                .blueprint_target_set_current_enabled(&opctx, bp1_target)
+                .await
+                .unwrap();
+            assert_eq!(
+                datastore.blueprint_target_get_current(&opctx).await.unwrap(),
+                Some(bp1_target),
+            );
+        }
+
+        // We cannot use `blueprint_target_set_current_enabled` to make
+        // bp2_target the target...
+        let err = datastore
+            .blueprint_target_set_current_enabled(&opctx, bp2_target)
+            .await
+            .unwrap_err();
+        assert!(err
+            .to_string()
+            .contains("is not the current target blueprint"));
+
+        // ...but we can make it the target via `blueprint_target_set_current`.
+        datastore
+            .blueprint_target_set_current(&opctx, bp2_target)
+            .await
+            .unwrap();
+        assert_eq!(
+            datastore.blueprint_target_get_current(&opctx).await.unwrap(),
+            Some(bp2_target),
+        );
+
+        // We can no longer toggle the enabled bit of bp1_target.
+        let err = datastore
+            .blueprint_target_set_current_enabled(&opctx, bp1_target)
+            .await
+            .unwrap_err();
+        assert!(err
+            .to_string()
+            .contains("is not the current target blueprint"));
+
+        // We can toggle bp2_target.enabled an arbitrary number of times.
+        for _ in 0..10 {
+            bp2_target.enabled = !bp2_target.enabled;
+            datastore
+                .blueprint_target_set_current_enabled(&opctx, bp2_target)
+                .await
+                .unwrap();
+            assert_eq!(
+                datastore.blueprint_target_get_current(&opctx).await.unwrap(),
+                Some(bp2_target),
+            );
+        }
+
+        // Clean up.
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
 }
diff --git a/nexus/src/app/background/blueprint_load.rs b/nexus/src/app/background/blueprint_load.rs
index 8886df81cd..603bc6b7d2 100644
--- a/nexus/src/app/background/blueprint_load.rs
+++ b/nexus/src/app/background/blueprint_load.rs
@@ -84,7 +84,7 @@ impl BackgroundTask for TargetBlueprintLoader {
                         longer any target blueprint",
                         old.1.id
                     );
-                    let old_id = old.1.id.to_string();
+                    let old_id = old.1.id;
                     self.last = None;
                     self.tx.send_replace(self.last.clone());
                     error!(&log, "{message:?}");
@@ -97,33 +97,33 @@ impl BackgroundTask for TargetBlueprintLoader {
                 (None, Ok(Some((new_bp_target, new_blueprint)))) => {
                     // We've found a target blueprint for the first time.
                     // Save it and notify any watchers.
-                    let target_id = new_blueprint.id.to_string();
-                    let time_created = new_blueprint.time_created.to_string();
+                    let target_id = new_blueprint.id;
+                    let time_created = new_blueprint.time_created;
                     info!(
                         log,
                         "found new target blueprint (first find)";
-                        "target_id" => &target_id,
-                        "time_created" => &time_created
+                        "target_id" => %target_id,
+                        "time_created" => %time_created
                     );
                     self.last = Some(Arc::new((new_bp_target, new_blueprint)));
                     self.tx.send_replace(self.last.clone());
                     json!({
                         "target_id": target_id,
                         "time_created": time_created,
-                        "time_found": chrono::Utc::now().to_string(),
-                        "status": "first target blueprint"
+                        "time_found": chrono::Utc::now(),
+                        "status": "first target blueprint",
                     })
                 }
                 (Some(old), Ok(Some((new_bp_target, new_blueprint)))) => {
-                    let target_id = new_blueprint.id.to_string();
-                    let time_created = new_blueprint.time_created.to_string();
+                    let target_id = new_blueprint.id;
+                    let time_created = new_blueprint.time_created;
                     if old.1.id != new_blueprint.id {
                         // The current target blueprint has been updated
                         info!(
                             log,
                             "found new target blueprint";
-                            "target_id" => &target_id,
-                            "time_created" => &time_created
+                            "target_id" => %target_id,
+                            "time_created" => %time_created
                         );
                         self.last =
                             Some(Arc::new((new_bp_target, new_blueprint)));
@@ -131,7 +131,7 @@ impl BackgroundTask for TargetBlueprintLoader {
                         json!({
                             "target_id": target_id,
                             "time_created": time_created,
-                            "time_found": chrono::Utc::now().to_string(),
+                            "time_found": chrono::Utc::now(),
                             "status": "target blueprint updated"
                         })
                     } else {
@@ -153,15 +153,39 @@ impl BackgroundTask for TargetBlueprintLoader {
                                 "status": "target blueprint unchanged (error)",
                                 "error": message
                             })
+                        } else if old.0.enabled != new_bp_target.enabled {
+                            // The blueprints have the same contents, but its
+                            // enabled bit has flipped.
+                            let status = if new_bp_target.enabled {
+                                "enabled"
+                            } else {
+                                "disabled"
+                            };
+                            info!(
+                                log,
+                                "target blueprint enabled state changed";
+                                "target_id" => %target_id,
+                                "time_created" => %time_created,
+                                "state" => status,
+                            );
+                            self.last =
+                                Some(Arc::new((new_bp_target, new_blueprint)));
+                            self.tx.send_replace(self.last.clone());
+                            json!({
+                                "target_id": target_id,
+                                "time_created": time_created,
+                                "time_found": chrono::Utc::now(),
+                                "status": format!("target blueprint {status}"),
+                            })
                         } else {
-                            // We found a new target blueprint that exactly matches
-                            // the old target blueprint. This is the common case
-                            // when we're activated by a timeout.
+                            // We found a new target blueprint that exactly
+                            // matches the old target blueprint. This is the
+                            // common case when we're activated by a timeout.
                             debug!(
                                log,
                                 "found latest target blueprint (unchanged)";
-                                "target_id" => &target_id,
-                                "time_created" => &time_created.clone()
+                                "target_id" => %target_id,
+                                "time_created" => %time_created.clone()
                             );
                             json!({
                                 "target_id": target_id,
diff --git a/nexus/src/app/deployment.rs b/nexus/src/app/deployment.rs
index 778e768878..4f66f2af78 100644
--- a/nexus/src/app/deployment.rs
+++ b/nexus/src/app/deployment.rs
@@ -94,8 +94,33 @@ impl super::Nexus {
             .blueprint_target_set_current(opctx, new_target)
             .await?;
 
-        // When we add a background task executing the target blueprint,
-        // this is the point where we'd signal it to update its target.
+        // We have a new target: trigger the background task to load this
+        // blueprint.
+        self.background_tasks
+            .activate(&self.background_tasks.task_blueprint_loader);
+
+        Ok(new_target)
+    }
+
+    pub async fn blueprint_target_set_enabled(
+        &self,
+        opctx: &OpContext,
+        params: BlueprintTargetSet,
+    ) -> Result<BlueprintTarget, Error> {
+        let new_target = BlueprintTarget {
+            target_id: params.target_id,
+            enabled: params.enabled,
+            time_made_target: chrono::Utc::now(),
+        };
+
+        self.db_datastore
+            .blueprint_target_set_current_enabled(opctx, new_target)
+            .await?;
+
+        // We don't know whether this actually changed the enabled bit; activate
+        // the background task to load this blueprint which does know.
+        self.background_tasks
+            .activate(&self.background_tasks.task_blueprint_loader);
 
         Ok(new_target)
     }
diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs
index d6649cd944..9871c38117 100644
--- a/nexus/src/internal_api/http_entrypoints.rs
+++ b/nexus/src/internal_api/http_entrypoints.rs
@@ -91,6 +91,7 @@ pub(crate) fn internal_api() -> NexusApiDescription {
         api.register(blueprint_delete)?;
         api.register(blueprint_target_view)?;
         api.register(blueprint_target_set)?;
+        api.register(blueprint_target_set_enabled)?;
         api.register(blueprint_generate_from_collection)?;
         api.register(blueprint_regenerate)?;
 
@@ -781,6 +782,26 @@ async fn blueprint_target_set(
     apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
 }
 
+/// Set the `enabled` field of the current target blueprint
+#[endpoint {
+    method = PUT,
+    path = "/deployment/blueprints/target/enabled",
+}]
+async fn blueprint_target_set_enabled(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    target: TypedBody<BlueprintTargetSet>,
+) -> Result<HttpResponseOk<BlueprintTarget>, HttpError> {
+    let apictx = rqctx.context();
+    let handler = async {
+        let opctx = crate::context::op_context_for_internal_api(&rqctx).await;
+        let nexus = &apictx.nexus;
+        let target = target.into_inner();
+        let target = nexus.blueprint_target_set_enabled(&opctx, target).await?;
+        Ok(HttpResponseOk(target))
+    };
+    apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
 // Generating blueprints
 
 #[derive(Debug, Deserialize, JsonSchema)]
diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json
index cc831fff73..c67fe27652 100644
--- a/openapi/nexus-internal.json
+++ b/openapi/nexus-internal.json
@@ -359,6 +359,40 @@
         }
       }
     },
+    "/deployment/blueprints/target/enabled": {
+      "put": {
+        "summary": "Set the `enabled` field of the current target blueprint",
+        "operationId": "blueprint_target_set_enabled",
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/BlueprintTargetSet"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/BlueprintTarget"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/disk/{disk_id}/remove-read-only-parent": {
       "post": {
         "summary": "Request removal of a read_only_parent from a disk",

From d6c7dc0de461a7e52d250788d2510f728addbef0 Mon Sep 17 00:00:00 2001
From: Kyle Simpson <kyle@oxide.computer>
Date: Wed, 13 Mar 2024 22:31:05 +0000
Subject: [PATCH 112/157] Bump OPTE to `a4c956e` (#5257)

* a4c956e chore(deps): lock file maintenance (oxidecomputer/opte#474)
* 6847b04 chore(deps): update rust crate nix to 0.28
(oxidecomputer/opte#473)
* c30aa36 Benchmarks and instrumentation (oxidecomputer/opte#395)
* 53201de Fix xtask non-package install for fresh systems.
(oxidecomputer/opte#470)
* 7e56634 Correctly preserve inbound UFID on outbound TCP close
(oxidecomputer/opte#469)
---
 Cargo.lock         | 14 +++++++-------
 Cargo.toml         |  4 ++--
 tools/opte_version |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 98b24ff3ac..de1956b659 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1738,7 +1738,7 @@ dependencies = [
 [[package]]
 name = "derror-macro"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
+source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
 dependencies = [
  "darling 0.20.3",
  "proc-macro2",
@@ -3425,7 +3425,7 @@ dependencies = [
 [[package]]
 name = "illumos-sys-hdrs"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
+source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
 
 [[package]]
 name = "illumos-utils"
@@ -3829,7 +3829,7 @@ dependencies = [
 [[package]]
 name = "kstat-macro"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
+source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
 dependencies = [
  "quote",
  "syn 2.0.51",
@@ -5760,7 +5760,7 @@ dependencies = [
 [[package]]
 name = "opte"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
+source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
 dependencies = [
  "cfg-if",
  "derror-macro",
@@ -5778,7 +5778,7 @@ dependencies = [
 [[package]]
 name = "opte-api"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
+source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
 dependencies = [
  "illumos-sys-hdrs",
  "ipnetwork",
@@ -5790,7 +5790,7 @@ dependencies = [
 [[package]]
 name = "opte-ioctl"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
+source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
 dependencies = [
  "libc",
  "libnet",
@@ -5864,7 +5864,7 @@ dependencies = [
 [[package]]
 name = "oxide-vpc"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=b85995f92ae94cdc78b97b0a610c69e103e00423#b85995f92ae94cdc78b97b0a610c69e103e00423"
+source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
 dependencies = [
  "cfg-if",
  "illumos-sys-hdrs",
diff --git a/Cargo.toml b/Cargo.toml
index 1ac1e3d2eb..f60bca4185 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -299,7 +299,7 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.11.0"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "b85995f92ae94cdc78b97b0a610c69e103e00423", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "a4c956e44fc9b75b58b83ad2eec22f6bd9005262", features = [ "api", "std" ] }
 once_cell = "1.19.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "2.0.0"
@@ -307,7 +307,7 @@ openapiv3 = "2.0.0"
 openssl = "0.10"
 openssl-sys = "0.9"
 openssl-probe = "0.1.5"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "b85995f92ae94cdc78b97b0a610c69e103e00423" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "a4c956e44fc9b75b58b83ad2eec22f6bd9005262" }
 oso = "0.27"
 owo-colors = "4.0.0"
 oximeter = { path = "oximeter/oximeter" }
diff --git a/tools/opte_version b/tools/opte_version
index 23d9da8fa3..6a9619566a 100644
--- a/tools/opte_version
+++ b/tools/opte_version
@@ -1 +1 @@
-0.28.227
+0.28.232

From da36ea6c22fff28ded9905840745e456facbe56d Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Wed, 13 Mar 2024 18:04:15 -0700
Subject: [PATCH 113/157] chore(deps): update rust crate serde_path_to_error to
 0.1.16 (#5251)

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 Cargo.lock | 4 ++--
 Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index de1956b659..a45d47faf7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8068,9 +8068,9 @@ dependencies = [
 
 [[package]]
 name = "serde_path_to_error"
-version = "0.1.15"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c"
+checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6"
 dependencies = [
  "itoa",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index f60bca4185..c27a4bef05 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -363,7 +363,7 @@ serde = { version = "1.0", default-features = false, features = [ "derive", "rc"
 serde_derive = "1.0"
 serde_human_bytes = { git = "http://github.com/oxidecomputer/serde_human_bytes", branch = "main" }
 serde_json = "1.0.114"
-serde_path_to_error = "0.1.15"
+serde_path_to_error = "0.1.16"
 serde_tokenstream = "0.2"
 serde_urlencoded = "0.7.1"
 serde_with = "3.6.1"

From 45ccacd2c1c4168cacb3d8bff5a58c77666ac760 Mon Sep 17 00:00:00 2001
From: David Pacheco <dap@oxidecomputer.com>
Date: Wed, 13 Mar 2024 20:53:10 -0700
Subject: [PATCH 114/157] update external DNS during blueprint execution
 (#5212)

---
 Cargo.lock                                    |    1 +
 clients/dns-service-client/src/diff.rs        |    9 +-
 clients/dns-service-client/src/lib.rs         |   39 +
 dev-tools/omdb/src/bin/omdb/nexus.rs          |    5 +-
 dev-tools/reconfigurator-cli/src/main.rs      |    2 +
 .../tests/input/complex.json                  |    4 +-
 nexus/db-model/src/deployment.rs              |    3 +
 nexus/db-model/src/schema.rs                  |    3 +-
 .../db-queries/src/db/datastore/deployment.rs |   20 +-
 nexus/db-queries/src/db/datastore/rack.rs     |   93 +-
 nexus/db-queries/src/db/datastore/silo.rs     |   31 +
 nexus/db-queries/src/db/datastore/vpc.rs      |    4 +
 nexus/reconfigurator/execution/Cargo.toml     |    3 +-
 nexus/reconfigurator/execution/src/dns.rs     | 1037 +++++++++++++++--
 nexus/reconfigurator/execution/src/lib.rs     |   25 +
 .../execution/src/omicron_zones.rs            |    1 +
 .../execution/src/overridables.rs             |  121 ++
 .../execution/src/resource_allocation.rs      |   13 +
 nexus/reconfigurator/planning/Cargo.toml      |    1 -
 .../planning/src/blueprint_builder.rs         |  178 +--
 nexus/reconfigurator/planning/src/example.rs  |  147 +++
 nexus/reconfigurator/planning/src/lib.rs      |    1 +
 nexus/reconfigurator/planning/src/planner.rs  |   24 +-
 .../src/app/background/blueprint_execution.rs |    5 +-
 nexus/src/app/background/blueprint_load.rs    |    1 +
 nexus/src/app/deployment.rs                   |   16 +-
 nexus/src/app/external_endpoints.rs           |    2 +-
 nexus/src/app/rack.rs                         |    2 +-
 nexus/src/app/silo.rs                         |   21 +-
 nexus/types/src/deployment.rs                 |    8 +-
 openapi/nexus-internal.json                   |   18 +
 schema/crdb/42.0.0/up1.sql                    |    8 +
 schema/crdb/42.0.0/up2.sql                    |    2 +
 schema/crdb/dbinit.sql                        |    6 +-
 34 files changed, 1534 insertions(+), 320 deletions(-)
 create mode 100644 nexus/reconfigurator/execution/src/overridables.rs
 create mode 100644 nexus/reconfigurator/planning/src/example.rs
 create mode 100644 schema/crdb/42.0.0/up1.sql
 create mode 100644 schema/crdb/42.0.0/up2.sql

diff --git a/Cargo.lock b/Cargo.lock
index a45d47faf7..88c78a83d4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4601,6 +4601,7 @@ dependencies = [
  "nexus-inventory",
  "nexus-networking",
  "nexus-reconfigurator-planning",
+ "nexus-reconfigurator-preparation",
  "nexus-test-utils",
  "nexus-test-utils-macros",
  "nexus-types",
diff --git a/clients/dns-service-client/src/diff.rs b/clients/dns-service-client/src/diff.rs
index 39d51cc974..ce04319dff 100644
--- a/clients/dns-service-client/src/diff.rs
+++ b/clients/dns-service-client/src/diff.rs
@@ -59,8 +59,13 @@ impl<'a> DnsDiff<'a> {
         &self,
     ) -> impl Iterator<Item = (&str, &[DnsRecord], &[DnsRecord])> {
         self.left.iter().filter_map(|(k, v1)| match self.right.get(k) {
-            Some(v2) if v1 != v2 => {
-                Some((k.as_ref(), v1.as_ref(), v2.as_ref()))
+            Some(v2) => {
+                let mut v1_sorted = v1.clone();
+                let mut v2_sorted = v2.clone();
+                v1_sorted.sort();
+                v2_sorted.sort();
+                (v1_sorted != v2_sorted)
+                    .then(|| (k.as_ref(), v1.as_ref(), v2.as_ref()))
             }
             _ => None,
         })
diff --git a/clients/dns-service-client/src/lib.rs b/clients/dns-service-client/src/lib.rs
index cd17a1559c..316c4787b0 100644
--- a/clients/dns-service-client/src/lib.rs
+++ b/clients/dns-service-client/src/lib.rs
@@ -109,3 +109,42 @@ impl types::DnsConfigParams {
         Ok(&self.zones[0])
     }
 }
+
+impl Ord for types::DnsRecord {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        use types::DnsRecord;
+        match (self, other) {
+            // Same kinds: compare the items in them
+            (DnsRecord::A(addr1), DnsRecord::A(addr2)) => addr1.cmp(addr2),
+            (DnsRecord::Aaaa(addr1), DnsRecord::Aaaa(addr2)) => {
+                addr1.cmp(addr2)
+            }
+            (DnsRecord::Srv(srv1), DnsRecord::Srv(srv2)) => srv1
+                .target
+                .cmp(&srv2.target)
+                .then_with(|| srv1.port.cmp(&srv2.port)),
+
+            // Different kinds: define an arbitrary order among the kinds.
+            // We could use std::mem::discriminant() here but it'd be nice if
+            // this were stable over time.
+            // We define (arbitrarily): A < Aaaa < Srv
+            (DnsRecord::A(_), DnsRecord::Aaaa(_) | DnsRecord::Srv(_)) => {
+                std::cmp::Ordering::Less
+            }
+            (DnsRecord::Aaaa(_), DnsRecord::Srv(_)) => std::cmp::Ordering::Less,
+
+            // Anything else will result in "Greater".  But let's be explicit.
+            (DnsRecord::Aaaa(_), DnsRecord::A(_))
+            | (DnsRecord::Srv(_), DnsRecord::A(_))
+            | (DnsRecord::Srv(_), DnsRecord::Aaaa(_)) => {
+                std::cmp::Ordering::Greater
+            }
+        }
+    }
+}
+
+impl PartialOrd for types::DnsRecord {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs
index 3d63f48469..26f2e07a41 100644
--- a/dev-tools/omdb/src/bin/omdb/nexus.rs
+++ b/dev-tools/omdb/src/bin/omdb/nexus.rs
@@ -1073,7 +1073,10 @@ async fn cmd_nexus_blueprints_generate_from_collection(
         )
         .await
         .context("creating blueprint from collection id")?;
-    eprintln!("created blueprint {} from collection id", blueprint.id);
+    eprintln!(
+        "created blueprint {} from collection id {}",
+        blueprint.id, args.collection_id
+    );
     Ok(())
 }
 
diff --git a/dev-tools/reconfigurator-cli/src/main.rs b/dev-tools/reconfigurator-cli/src/main.rs
index fab4237ebd..82b71cff49 100644
--- a/dev-tools/reconfigurator-cli/src/main.rs
+++ b/dev-tools/reconfigurator-cli/src/main.rs
@@ -454,6 +454,7 @@ fn cmd_blueprint_from_inventory(
     let blueprint = BlueprintBuilder::build_initial_from_collection(
         collection,
         dns_version,
+        dns_version,
         &policy,
         creator,
     )
@@ -487,6 +488,7 @@ fn cmd_blueprint_plan(
         sim.log.clone(),
         parent_blueprint,
         dns_version,
+        dns_version,
         &policy,
         creator,
         collection,
diff --git a/dev-tools/reconfigurator-cli/tests/input/complex.json b/dev-tools/reconfigurator-cli/tests/input/complex.json
index 5bdba202d3..67f1881147 100644
--- a/dev-tools/reconfigurator-cli/tests/input/complex.json
+++ b/dev-tools/reconfigurator-cli/tests/input/complex.json
@@ -7973,6 +7973,7 @@
       ],
       "parent_blueprint_id": null,
       "internal_dns_version": 1,
+      "external_dns_version": 1,
       "time_created": "2024-03-12T19:09:41.953854Z",
       "creator": "0d459323-e414-4f32-9944-76c7331da622",
       "comment": "from collection 3f61b723-cfe7-40ec-b22c-e3e1f35325f9"
@@ -8645,9 +8646,10 @@
       ],
       "parent_blueprint_id": null,
       "internal_dns_version": 1,
+      "external_dns_version": 1,
       "time_created": "2024-03-12T19:11:49.876621Z",
       "creator": "0d459323-e414-4f32-9944-76c7331da622",
       "comment": "from collection 5e1d6ae0-c5b4-4884-ac41-680cd4f5762d"
     }
   ]
-}
\ No newline at end of file
+}
diff --git a/nexus/db-model/src/deployment.rs b/nexus/db-model/src/deployment.rs
index e9a650812b..e34148bede 100644
--- a/nexus/db-model/src/deployment.rs
+++ b/nexus/db-model/src/deployment.rs
@@ -25,6 +25,7 @@ pub struct Blueprint {
     pub id: Uuid,
     pub parent_blueprint_id: Option<Uuid>,
     pub internal_dns_version: Generation,
+    pub external_dns_version: Generation,
     pub time_created: DateTime<Utc>,
     pub creator: String,
     pub comment: String,
@@ -36,6 +37,7 @@ impl From<&'_ nexus_types::deployment::Blueprint> for Blueprint {
             id: bp.id,
             parent_blueprint_id: bp.parent_blueprint_id,
             internal_dns_version: Generation(bp.internal_dns_version),
+            external_dns_version: Generation(bp.external_dns_version),
             time_created: bp.time_created,
             creator: bp.creator.clone(),
             comment: bp.comment.clone(),
@@ -49,6 +51,7 @@ impl From<Blueprint> for nexus_types::deployment::BlueprintMetadata {
             id: value.id,
             parent_blueprint_id: value.parent_blueprint_id,
             internal_dns_version: *value.internal_dns_version,
+            external_dns_version: *value.external_dns_version,
             time_created: value.time_created,
             creator: value.creator,
             comment: value.comment,
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index 781f56fe0c..bcbf7fa88f 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(41, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(42, 0, 0);
 
 table! {
     disk (id) {
@@ -1441,6 +1441,7 @@ table! {
         comment -> Text,
 
         internal_dns_version -> Int8,
+        external_dns_version -> Int8,
     }
 }
 
diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs
index c2b4e998ce..1a12b9d250 100644
--- a/nexus/db-queries/src/db/datastore/deployment.rs
+++ b/nexus/db-queries/src/db/datastore/deployment.rs
@@ -229,6 +229,7 @@ impl DataStore {
         let (
             parent_blueprint_id,
             internal_dns_version,
+            external_dns_version,
             time_created,
             creator,
             comment,
@@ -251,6 +252,7 @@ impl DataStore {
             (
                 blueprint.parent_blueprint_id,
                 *blueprint.internal_dns_version,
+                *blueprint.external_dns_version,
                 blueprint.time_created,
                 blueprint.creator,
                 blueprint.comment,
@@ -480,6 +482,7 @@ impl DataStore {
             zones_in_service,
             parent_blueprint_id,
             internal_dns_version,
+            external_dns_version,
             time_created,
             creator,
             comment,
@@ -1271,6 +1274,7 @@ mod tests {
         let blueprint = BlueprintBuilder::build_initial_from_collection(
             &collection,
             Generation::new(),
+            Generation::new(),
             &policy,
             "test",
         )
@@ -1305,6 +1309,7 @@ mod tests {
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
             &collection,
             Generation::new(),
+            Generation::new(),
             &EMPTY_POLICY,
             "test",
         )
@@ -1432,11 +1437,13 @@ mod tests {
 
         // Create a builder for a child blueprint.  While we're at it, use a
         // different DNS version to test that that works.
-        let new_dns_version = blueprint1.internal_dns_version.next();
+        let new_internal_dns_version = blueprint1.internal_dns_version.next();
+        let new_external_dns_version = new_internal_dns_version.next();
         let mut builder = BlueprintBuilder::new_based_on(
             &logctx.log,
             &blueprint1,
-            new_dns_version,
+            new_internal_dns_version,
+            new_external_dns_version,
             &policy,
             "test",
         )
@@ -1488,7 +1495,8 @@ mod tests {
             .expect("failed to read collection back");
         println!("diff: {}", blueprint2.diff_sleds(&blueprint_read).display());
         assert_eq!(blueprint2, blueprint_read);
-        assert_eq!(blueprint2.internal_dns_version, new_dns_version);
+        assert_eq!(blueprint2.internal_dns_version, new_internal_dns_version);
+        assert_eq!(blueprint2.external_dns_version, new_external_dns_version);
         {
             let mut expected_ids = [blueprint1.id, blueprint2.id];
             expected_ids.sort();
@@ -1581,6 +1589,7 @@ mod tests {
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
             &collection,
             Generation::new(),
+            Generation::new(),
             &EMPTY_POLICY,
             "test1",
         )
@@ -1589,6 +1598,7 @@ mod tests {
             &logctx.log,
             &blueprint1,
             Generation::new(),
+            Generation::new(),
             &EMPTY_POLICY,
             "test2",
         )
@@ -1598,6 +1608,7 @@ mod tests {
             &logctx.log,
             &blueprint1,
             Generation::new(),
+            Generation::new(),
             &EMPTY_POLICY,
             "test3",
         )
@@ -1695,6 +1706,7 @@ mod tests {
             &logctx.log,
             &blueprint3,
             Generation::new(),
+            Generation::new(),
             &EMPTY_POLICY,
             "test3",
         )
@@ -1734,6 +1746,7 @@ mod tests {
         let blueprint1 = BlueprintBuilder::build_initial_from_collection(
             &collection,
             Generation::new(),
+            Generation::new(),
             &EMPTY_POLICY,
             "test1",
         )
@@ -1742,6 +1755,7 @@ mod tests {
             &logctx.log,
             &blueprint1,
             Generation::new(),
+            Generation::new(),
             &EMPTY_POLICY,
             "test2",
         )
diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs
index 3224f36b8d..fa4d6a4210 100644
--- a/nexus/db-queries/src/db/datastore/rack.rs
+++ b/nexus/db-queries/src/db/datastore/rack.rs
@@ -27,7 +27,6 @@ use crate::db::model::Rack;
 use crate::db::model::Zpool;
 use crate::db::pagination::paginated;
 use crate::db::pool::DbConnection;
-use crate::transaction_retry::OptionalError;
 use async_bb8_diesel::AsyncConnection;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use chrono::Utc;
@@ -44,6 +43,8 @@ use nexus_db_model::PasswordHashString;
 use nexus_db_model::SiloUser;
 use nexus_db_model::SiloUserPasswordHash;
 use nexus_db_model::SledUnderlaySubnetAllocation;
+use nexus_types::deployment::Blueprint;
+use nexus_types::deployment::OmicronZoneType;
 use nexus_types::external_api::params as external_params;
 use nexus_types::external_api::shared;
 use nexus_types::external_api::shared::IdentityType;
@@ -54,6 +55,7 @@ use nexus_types::internal_api::params as internal_params;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::IdentityMetadataCreateParams;
+use omicron_common::api::external::InternalContext;
 use omicron_common::api::external::ListResultVec;
 use omicron_common::api::external::LookupType;
 use omicron_common::api::external::ResourceType;
@@ -793,60 +795,49 @@ impl DataStore {
     pub async fn nexus_external_addresses(
         &self,
         opctx: &OpContext,
+        blueprint: Option<&Blueprint>,
     ) -> Result<(Vec<IpAddr>, Vec<DnsZone>), Error> {
         opctx.authorize(authz::Action::Read, &authz::DNS_CONFIG).await?;
 
-        use crate::db::schema::external_ip::dsl as extip_dsl;
-        use crate::db::schema::service::dsl as service_dsl;
-
-        let err = OptionalError::new();
-        let conn = self.pool_connection_authorized(opctx).await?;
-        self.transaction_retry_wrapper("nexus_external_addresses")
-            .transaction(&conn, |conn| {
-                let err = err.clone();
-                async move {
-                    let ips = extip_dsl::external_ip
-                        .inner_join(
-                            service_dsl::service.on(service_dsl::id
-                                .eq(extip_dsl::parent_id.assume_not_null())),
-                        )
-                        .filter(extip_dsl::parent_id.is_not_null())
-                        .filter(extip_dsl::time_deleted.is_null())
-                        .filter(extip_dsl::is_service)
-                        .filter(
-                            service_dsl::kind.eq(db::model::ServiceKind::Nexus),
-                        )
-                        .select(ExternalIp::as_select())
-                        .get_results_async(&conn)
-                        .await?
-                        .into_iter()
-                        .map(|external_ip| external_ip.ip.ip())
-                        .collect();
-
-                    let dns_zones = self
-                        .dns_zones_list_all_on_connection(
-                            opctx,
-                            &conn,
-                            DnsGroup::External,
-                        )
-                        .await
-                        .map_err(|e| match e.retryable() {
-                            NotRetryable(not_retryable_err) => {
-                                err.bail(not_retryable_err)
-                            }
-                            Retryable(retryable_err) => retryable_err,
-                        })?;
-
-                    Ok((ips, dns_zones))
-                }
-            })
+        let dns_zones = self
+            .dns_zones_list_all(opctx, DnsGroup::External)
             .await
-            .map_err(|e| {
-                if let Some(err) = err.take() {
-                    return err.into();
-                }
-                public_error_from_diesel(e, ErrorHandler::Server)
-            })
+            .internal_context("listing DNS zones to list external addresses")?;
+
+        let nexus_external_ips = if let Some(blueprint) = blueprint {
+            blueprint
+                .all_omicron_zones()
+                .filter_map(|(_, z)| match z.zone_type {
+                    OmicronZoneType::Nexus { external_ip, .. } => {
+                        Some(external_ip)
+                    }
+                    _ => None,
+                })
+                .collect()
+        } else {
+            use crate::db::schema::external_ip::dsl as extip_dsl;
+            use crate::db::schema::service::dsl as service_dsl;
+
+            let conn = self.pool_connection_authorized(opctx).await?;
+
+            extip_dsl::external_ip
+                .inner_join(service_dsl::service.on(
+                    service_dsl::id.eq(extip_dsl::parent_id.assume_not_null()),
+                ))
+                .filter(extip_dsl::parent_id.is_not_null())
+                .filter(extip_dsl::time_deleted.is_null())
+                .filter(extip_dsl::is_service)
+                .filter(service_dsl::kind.eq(db::model::ServiceKind::Nexus))
+                .select(ExternalIp::as_select())
+                .get_results_async(&*conn)
+                .await
+                .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?
+                .into_iter()
+                .map(|external_ip| external_ip.ip.ip())
+                .collect()
+        };
+
+        Ok((nexus_external_ips, dns_zones))
     }
 }
 
diff --git a/nexus/db-queries/src/db/datastore/silo.rs b/nexus/db-queries/src/db/datastore/silo.rs
index df10b1c072..0fd858b900 100644
--- a/nexus/db-queries/src/db/datastore/silo.rs
+++ b/nexus/db-queries/src/db/datastore/silo.rs
@@ -6,6 +6,7 @@
 
 use super::dns::DnsVersionUpdateBuilder;
 use super::DataStore;
+use super::SQL_BATCH_SIZE;
 use crate::authz;
 use crate::context::OpContext;
 use crate::db;
@@ -22,6 +23,7 @@ use crate::db::model::Name;
 use crate::db::model::Silo;
 use crate::db::model::VirtualProvisioningCollection;
 use crate::db::pagination::paginated;
+use crate::db::pagination::Paginator;
 use crate::db::pool::DbConnection;
 use async_bb8_diesel::AsyncConnection;
 use async_bb8_diesel::AsyncRunQueryDsl;
@@ -45,6 +47,7 @@ use ref_cast::RefCast;
 use uuid::Uuid;
 
 /// Filter a "silo_list" query based on silos' discoverability
+#[derive(Debug, Clone, Copy)]
 pub enum Discoverability {
     /// Show all Silos
     All,
@@ -351,6 +354,34 @@ impl DataStore {
             .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
     }
 
+    /// List all Silos, making as many queries as needed to get them all
+    ///
+    /// This should generally not be used in API handlers or other
+    /// latency-sensitive contexts, but it can make sense in saga actions or
+    /// background tasks.
+    pub async fn silo_list_all_batched(
+        &self,
+        opctx: &OpContext,
+        discoverability: Discoverability,
+    ) -> ListResultVec<Silo> {
+        opctx.check_complex_operations_allowed()?;
+        let mut all_silos = Vec::new();
+        let mut paginator = Paginator::new(SQL_BATCH_SIZE);
+        while let Some(p) = paginator.next() {
+            let batch = self
+                .silos_list(
+                    opctx,
+                    &PaginatedBy::Id(p.current_pagparams()),
+                    discoverability,
+                )
+                .await?;
+            paginator =
+                p.found_batch(&batch, &|s: &nexus_db_model::Silo| s.id());
+            all_silos.extend(batch);
+        }
+        Ok(all_silos)
+    }
+
     pub async fn silo_delete(
         &self,
         opctx: &OpContext,
diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs
index 8c5d5f4f46..8be21348bf 100644
--- a/nexus/db-queries/src/db/datastore/vpc.rs
+++ b/nexus/db-queries/src/db/datastore/vpc.rs
@@ -1674,6 +1674,7 @@ mod tests {
             zones_in_service: BTreeSet::new(),
             parent_blueprint_id: None,
             internal_dns_version: Generation::new(),
+            external_dns_version: Generation::new(),
             time_created: Utc::now(),
             creator: "test".to_string(),
             comment: "test".to_string(),
@@ -1725,6 +1726,7 @@ mod tests {
             zones_in_service: BTreeSet::new(),
             parent_blueprint_id: Some(bp1_id),
             internal_dns_version: Generation::new(),
+            external_dns_version: Generation::new(),
             time_created: Utc::now(),
             creator: "test".to_string(),
             comment: "test".to_string(),
@@ -1785,6 +1787,7 @@ mod tests {
             zones_in_service: BTreeSet::new(),
             parent_blueprint_id: Some(bp2_id),
             internal_dns_version: Generation::new(),
+            external_dns_version: Generation::new(),
             time_created: Utc::now(),
             creator: "test".to_string(),
             comment: "test".to_string(),
@@ -1837,6 +1840,7 @@ mod tests {
             zones_in_service: BTreeSet::new(),
             parent_blueprint_id: Some(bp3_id),
             internal_dns_version: Generation::new(),
+            external_dns_version: Generation::new(),
             time_created: Utc::now(),
             creator: "test".to_string(),
             comment: "test".to_string(),
diff --git a/nexus/reconfigurator/execution/Cargo.toml b/nexus/reconfigurator/execution/Cargo.toml
index d48f4c6b5b..b479ae67ee 100644
--- a/nexus/reconfigurator/execution/Cargo.toml
+++ b/nexus/reconfigurator/execution/Cargo.toml
@@ -9,6 +9,7 @@ omicron-rpaths.workspace = true
 [dependencies]
 anyhow.workspace = true
 dns-service-client.workspace = true
+chrono.workspace = true
 futures.workspace = true
 illumos-utils.workspace = true
 internal-dns.workspace = true
@@ -32,10 +33,10 @@ pq-sys = "*"
 omicron-workspace-hack.workspace = true
 
 [dev-dependencies]
-chrono.workspace = true
 httptest.workspace = true
 ipnet.workspace = true
 nexus-reconfigurator-planning.workspace = true
+nexus-reconfigurator-preparation.workspace = true
 nexus-inventory.workspace = true
 nexus-test-utils.workspace = true
 nexus-test-utils-macros.workspace = true
diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs
index 0fa8eb1c10..9e0434c59f 100644
--- a/nexus/reconfigurator/execution/src/dns.rs
+++ b/nexus/reconfigurator/execution/src/dns.rs
@@ -2,37 +2,35 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//! Propagates internal DNS changes in a given blueprint
+//! Propagates DNS changes in a given blueprint
 
+use crate::overridables::Overridables;
 use crate::Sled;
+use anyhow::Context;
 use dns_service_client::DnsDiff;
 use internal_dns::DnsConfigBuilder;
 use internal_dns::ServiceName;
 use nexus_db_model::DnsGroup;
+use nexus_db_model::Silo;
 use nexus_db_queries::context::OpContext;
+use nexus_db_queries::db::datastore::Discoverability;
 use nexus_db_queries::db::datastore::DnsVersionUpdateBuilder;
+use nexus_db_queries::db::fixed_data::silo::SILO_ID;
 use nexus_db_queries::db::DataStore;
 use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::OmicronZoneType;
+use nexus_types::identity::Resource;
 use nexus_types::internal_api::params::DnsConfigParams;
-use omicron_common::address::get_switch_zone_address;
-use omicron_common::address::CLICKHOUSE_KEEPER_PORT;
-use omicron_common::address::CLICKHOUSE_PORT;
-use omicron_common::address::COCKROACH_PORT;
-use omicron_common::address::CRUCIBLE_PANTRY_PORT;
-use omicron_common::address::CRUCIBLE_PORT;
-use omicron_common::address::DENDRITE_PORT;
-use omicron_common::address::DNS_HTTP_PORT;
-use omicron_common::address::MGD_PORT;
-use omicron_common::address::MGS_PORT;
-use omicron_common::address::NEXUS_INTERNAL_PORT;
-use omicron_common::address::NTP_PORT;
-use omicron_common::address::OXIMETER_PORT;
+use nexus_types::internal_api::params::DnsConfigZone;
+use nexus_types::internal_api::params::DnsRecord;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::Generation;
 use omicron_common::api::external::InternalContext;
 use slog::{debug, info, o};
 use std::collections::BTreeMap;
+use std::collections::HashMap;
+use std::net::IpAddr;
+use std::net::SocketAddrV6;
 use uuid::Uuid;
 
 pub(crate) async fn deploy_dns(
@@ -41,12 +39,17 @@ pub(crate) async fn deploy_dns(
     creator: String,
     blueprint: &Blueprint,
     sleds_by_id: &BTreeMap<Uuid, Sled>,
+    overrides: &Overridables,
 ) -> Result<(), Error> {
-    // First, fetch the current DNS config.
-    let dns_config_current = datastore
+    // First, fetch the current DNS configs.
+    let internal_dns_config_current = datastore
         .dns_config_read(opctx, DnsGroup::Internal)
         .await
-        .internal_context("reading current DNS")?;
+        .internal_context("reading current DNS (internal)")?;
+    let external_dns_config_current = datastore
+        .dns_config_read(opctx, DnsGroup::External)
+        .await
+        .internal_context("reading current DNS (external)")?;
 
     // We could check here that the DNS version we found isn't newer than when
     // the blueprint was generated.  But we have to check later when we try to
@@ -55,18 +58,83 @@ pub(crate) async fn deploy_dns(
     // we know it's being hit when we exercise this condition.
 
     // Next, construct the DNS config represented by the blueprint.
-    let dns_config_blueprint = blueprint_dns_config(blueprint, sleds_by_id);
+    let internal_dns_config_blueprint =
+        blueprint_internal_dns_config(blueprint, sleds_by_id, overrides)
+            .map_err(|e| {
+                Error::internal_error(&format!(
+                    "error constructing internal DNS config: {:#}",
+                    e
+                ))
+            })?;
+    let silos = datastore
+        .silo_list_all_batched(opctx, Discoverability::All)
+        .await
+        .internal_context("listing Silos (for configuring external DNS)")?
+        .into_iter()
+        // We do not generate a DNS name for the "default" Silo.
+        .filter(|silo| silo.id() != *SILO_ID)
+        .collect::<Vec<_>>();
+
+    let (nexus_external_ips, nexus_external_dns_zones) =
+        datastore.nexus_external_addresses(opctx, Some(blueprint)).await?;
+    let nexus_external_dns_zone_names = nexus_external_dns_zones
+        .into_iter()
+        .map(|z| z.zone_name)
+        .collect::<Vec<_>>();
+    let external_dns_config_blueprint = blueprint_external_dns_config(
+        blueprint,
+        &nexus_external_ips,
+        &silos,
+        &nexus_external_dns_zone_names,
+    );
+
+    // Deploy the changes.
+    deploy_dns_one(
+        opctx,
+        datastore,
+        creator.clone(),
+        blueprint,
+        &internal_dns_config_current,
+        &internal_dns_config_blueprint,
+        DnsGroup::Internal,
+    )
+    .await?;
+    deploy_dns_one(
+        opctx,
+        datastore,
+        creator,
+        blueprint,
+        &external_dns_config_current,
+        &external_dns_config_blueprint,
+        DnsGroup::External,
+    )
+    .await?;
+    Ok(())
+}
+
+pub(crate) async fn deploy_dns_one(
+    opctx: &OpContext,
+    datastore: &DataStore,
+    creator: String,
+    blueprint: &Blueprint,
+    dns_config_current: &DnsConfigParams,
+    dns_config_blueprint: &DnsConfigParams,
+    dns_group: DnsGroup,
+) -> Result<(), Error> {
+    let log = opctx
+        .log
+        .new(o!("blueprint_execution" => format!("dns {:?}", dns_group)));
 
     // Looking at the current contents of DNS, prepare an update that will make
     // it match what it should be.
-    let log = opctx.log.new(o!("blueprint_execution" => "DNS"));
     let comment = format!("blueprint {} ({})", blueprint.id, blueprint.comment);
     let maybe_update = dns_compute_update(
         &log,
+        dns_group,
         comment,
         creator,
-        &dns_config_current,
-        &dns_config_blueprint,
+        dns_config_current,
+        dns_config_blueprint,
     )?;
     let Some(update) = maybe_update else {
         // Nothing to do.
@@ -81,12 +149,11 @@ pub(crate) async fn deploy_dns(
     // executing a newer target blueprint.
     //
     // To avoid this problem, before generating a blueprint, Nexus fetches the
-    // current internal DNS generation and stores that into the blueprint
-    // itself.  Here, when we execute the blueprint, we make our database update
-    // conditional on that still being the current internal DNS generation.
-    // If some other instance has already come along and updated the database,
-    // whether for this same blueprint or a newer one, our attempt to update the
-    // database will fail.
+    // current DNS generation and stores that into the blueprint itself.  Here,
+    // when we execute the blueprint, we make our database update conditional on
+    // that still being the current DNS generation.  If some other instance has
+    // already come along and updated the database, whether for this same
+    // blueprint or a newer one, our attempt to update the database will fail.
     //
     // Let's look at a tricky example.  Suppose:
     //
@@ -100,7 +167,7 @@ pub(crate) async fn deploy_dns(
     //    that's still the current version in DNS.  B3 is made the current
     //    target.
     //
-    //    Assume B2 and B3 specify different internal DNS contents (e.g., have a
+    //    Assume B2 and B3 specify different DNS contents (e.g., have a
     //    different set of Omicron zones in them).
     //
     // 4. Nexus instance N1 finds B2 to be the current target and starts
@@ -121,21 +188,21 @@ pub(crate) async fn deploy_dns(
     // Now, one of two things could happen:
     //
     // 1. N1 wins.  Its database update applies successfully.  In the database,
-    //    the internal DNS version becomes version 4.  In this case, N2 loses.
-    //    Its database operation fails altogether.  At this point, any
-    //    subsequent attempt to execute blueprint B3 will fail because any DNS
-    //    update will be conditional on the database having version 3.  The only
-    //    way out of this is for the planner to generate a new blueprint B4
-    //    that's exactly equivalent to B3 except that the stored internal DNS
-    //    version is 4.  Then we'll be able to execute that.
+    //    the DNS version becomes version 4.  In this case, N2 loses.  Its
+    //    database operation fails altogether.  At this point, any subsequent
+    //    attempt to execute blueprint B3 will fail because any DNS update will
+    //    be conditional on the database having version 3.  The only way out of
+    //    this is for the planner to generate a new blueprint B4 that's exactly
+    //    equivalent to B3 except that the stored DNS version is 4.  Then we'll
+    //    be able to execute that.
     //
     // 2. N2 wins.  Its database update applies successfully.  In the database,
-    //    the internal DNS version becomes version 4.  In this case, N1 loses.
-    //    Its database operation fails altogether.  At this point, any
-    //    subsequent attempt to execute blueprint B3 will fail because any DNS
-    //    update will be conditional on the databae having version 3.  No
-    //    further action is needed, though, because we've successfully executed
-    //    the latest target blueprint.
+    //    the DNS version becomes version 4.  In this case, N1 loses.  Its
+    //    database operation fails altogether.  At this point, any subsequent
+    //    attempt to execute blueprint B3 will fail because any DNS update will
+    //    be conditional on the databae having version 3.  No further action is
+    //    needed, though, because we've successfully executed the latest target
+    //    blueprint.
     //
     // In both cases, the system will (1) converge to having successfully
     // executed the target blueprint, and (2) never have rolled any changes back
@@ -149,7 +216,7 @@ pub(crate) async fn deploy_dns(
     let generation_u32 =
         u32::try_from(dns_config_current.generation).map_err(|e| {
             Error::internal_error(&format!(
-                "internal DNS generation got too large: {}",
+                "DNS generation got too large: {}",
                 e,
             ))
         })?;
@@ -159,10 +226,11 @@ pub(crate) async fn deploy_dns(
 }
 
 /// Returns the expected contents of internal DNS based on the given blueprint
-pub fn blueprint_dns_config(
+pub fn blueprint_internal_dns_config(
     blueprint: &Blueprint,
     sleds_by_id: &BTreeMap<Uuid, Sled>,
-) -> DnsConfigParams {
+    overrides: &Overridables,
+) -> Result<DnsConfigParams, anyhow::Error> {
     // The DNS names configured here should match what RSS configures for the
     // same zones.  It's tricky to have RSS share the same code because it uses
     // Sled Agent's _internal_ `OmicronZoneConfig` (and friends), whereas we're
@@ -171,52 +239,72 @@ pub fn blueprint_dns_config(
     // the details.
     let mut dns_builder = DnsConfigBuilder::new();
 
-    // The code below assumes that all zones are using the default port numbers.
-    // That should be true, as those are the only ports ever used today.
-    // In an ideal world, the correct port would be pulled out of the
-    // `OmicronZoneType` variant instead.  Although that information is present,
-    // it's irritatingly non-trivial to do right now because SocketAddrs are
-    // represented as strings, so we'd need to parse all of them and handle all
-    // the errors, even though they should never happen.
-    // See oxidecomputer/omicron#4988.
+    // It's annoying that we have to parse this because it really should be
+    // valid already.  See oxidecomputer/omicron#4988.
+    fn parse_port(address: &str) -> Result<u16, anyhow::Error> {
+        address
+            .parse::<SocketAddrV6>()
+            .with_context(|| format!("parsing socket address {:?}", address))
+            .map(|addr| addr.port())
+    }
+
     for (_, omicron_zone) in blueprint.all_omicron_zones() {
         if !blueprint.zones_in_service.contains(&omicron_zone.id) {
             continue;
         }
 
-        let (service_name, port) = match omicron_zone.zone_type {
-            OmicronZoneType::BoundaryNtp { .. } => {
-                (ServiceName::BoundaryNtp, NTP_PORT)
+        let context = || {
+            format!(
+                "parsing {} zone with id {}",
+                omicron_zone.zone_type.label(),
+                omicron_zone.id
+            )
+        };
+        let (service_name, port) = match &omicron_zone.zone_type {
+            OmicronZoneType::BoundaryNtp { address, .. } => {
+                let port = parse_port(&address).with_context(context)?;
+                (ServiceName::BoundaryNtp, port)
             }
-            OmicronZoneType::InternalNtp { .. } => {
-                (ServiceName::InternalNtp, NTP_PORT)
+            OmicronZoneType::InternalNtp { address, .. } => {
+                let port = parse_port(&address).with_context(context)?;
+                (ServiceName::InternalNtp, port)
             }
-            OmicronZoneType::Clickhouse { .. } => {
-                (ServiceName::Clickhouse, CLICKHOUSE_PORT)
+            OmicronZoneType::Clickhouse { address, .. } => {
+                let port = parse_port(&address).with_context(context)?;
+                (ServiceName::Clickhouse, port)
             }
-            OmicronZoneType::ClickhouseKeeper { .. } => {
-                (ServiceName::ClickhouseKeeper, CLICKHOUSE_KEEPER_PORT)
+            OmicronZoneType::ClickhouseKeeper { address, .. } => {
+                let port = parse_port(&address).with_context(context)?;
+                (ServiceName::ClickhouseKeeper, port)
             }
-            OmicronZoneType::CockroachDb { .. } => {
-                (ServiceName::Cockroach, COCKROACH_PORT)
+            OmicronZoneType::CockroachDb { address, .. } => {
+                let port = parse_port(&address).with_context(context)?;
+                (ServiceName::Cockroach, port)
             }
-            OmicronZoneType::Nexus { .. } => {
-                (ServiceName::Nexus, NEXUS_INTERNAL_PORT)
+            OmicronZoneType::Nexus { internal_address, .. } => {
+                let port =
+                    parse_port(internal_address).with_context(context)?;
+                (ServiceName::Nexus, port)
             }
-            OmicronZoneType::Crucible { .. } => {
-                (ServiceName::Crucible(omicron_zone.id), CRUCIBLE_PORT)
+            OmicronZoneType::Crucible { address, .. } => {
+                let port = parse_port(address).with_context(context)?;
+                (ServiceName::Crucible(omicron_zone.id), port)
             }
-            OmicronZoneType::CruciblePantry { .. } => {
-                (ServiceName::CruciblePantry, CRUCIBLE_PANTRY_PORT)
+            OmicronZoneType::CruciblePantry { address } => {
+                let port = parse_port(address).with_context(context)?;
+                (ServiceName::CruciblePantry, port)
             }
-            OmicronZoneType::Oximeter { .. } => {
-                (ServiceName::Oximeter, OXIMETER_PORT)
+            OmicronZoneType::Oximeter { address } => {
+                let port = parse_port(address).with_context(context)?;
+                (ServiceName::Oximeter, port)
             }
-            OmicronZoneType::ExternalDns { .. } => {
-                (ServiceName::ExternalDns, DNS_HTTP_PORT)
+            OmicronZoneType::ExternalDns { http_address, .. } => {
+                let port = parse_port(http_address).with_context(context)?;
+                (ServiceName::ExternalDns, port)
             }
-            OmicronZoneType::InternalDns { .. } => {
-                (ServiceName::InternalDns, DNS_HTTP_PORT)
+            OmicronZoneType::InternalDns { http_address, .. } => {
+                let port = parse_port(http_address).with_context(context)?;
+                (ServiceName::InternalDns, port)
             }
         };
 
@@ -235,15 +323,15 @@ pub fn blueprint_dns_config(
     let scrimlets = sleds_by_id.values().filter(|sled| sled.is_scrimlet);
     for scrimlet in scrimlets {
         let sled_subnet = scrimlet.subnet();
-        let switch_zone_ip = get_switch_zone_address(sled_subnet);
+        let switch_zone_ip = overrides.switch_zone_ip(scrimlet.id, sled_subnet);
         // unwrap(): see above.
         dns_builder
             .host_zone_switch(
                 scrimlet.id,
                 switch_zone_ip,
-                DENDRITE_PORT,
-                MGS_PORT,
-                MGD_PORT,
+                overrides.dendrite_port(scrimlet.id),
+                overrides.mgs_port(scrimlet.id),
+                overrides.mgd_port(scrimlet.id),
             )
             .unwrap();
     }
@@ -252,18 +340,52 @@ pub fn blueprint_dns_config(
     // whatever it was when this blueprint was generated.  This will only be
     // used if the generated DNS contents are different from what's current.
     dns_builder.generation(blueprint.internal_dns_version.next());
-    dns_builder.build()
+    Ok(dns_builder.build())
+}
+
+pub fn blueprint_external_dns_config(
+    blueprint: &Blueprint,
+    nexus_external_ips: &[IpAddr],
+    silos: &[Silo],
+    external_dns_zone_names: &[String],
+) -> DnsConfigParams {
+    let dns_records: Vec<DnsRecord> = nexus_external_ips
+        .into_iter()
+        .map(|addr| match addr {
+            IpAddr::V4(addr) => DnsRecord::A(*addr),
+            IpAddr::V6(addr) => DnsRecord::Aaaa(*addr),
+        })
+        .collect();
+
+    let records = silos
+        .into_iter()
+        .map(|silo| (silo_dns_name(&silo.name()), dns_records.clone()))
+        .collect::<HashMap<String, Vec<DnsRecord>>>();
+
+    let zones = external_dns_zone_names
+        .into_iter()
+        .map(|zone_name| DnsConfigZone {
+            zone_name: zone_name.to_owned(),
+            records: records.clone(),
+        })
+        .collect();
+
+    DnsConfigParams {
+        generation: u64::from(blueprint.external_dns_version.next()),
+        time_created: chrono::Utc::now(),
+        zones,
+    }
 }
 
 fn dns_compute_update(
     log: &slog::Logger,
+    dns_group: DnsGroup,
     comment: String,
     creator: String,
     current_config: &DnsConfigParams,
     new_config: &DnsConfigParams,
 ) -> Result<Option<DnsVersionUpdateBuilder>, Error> {
-    let mut update =
-        DnsVersionUpdateBuilder::new(DnsGroup::Internal, comment, creator);
+    let mut update = DnsVersionUpdateBuilder::new(dns_group, comment, creator);
 
     let diff = DnsDiff::new(&current_config, &new_config)
         .map_err(|e| Error::internal_error(&format!("{:#}", e)))?;
@@ -276,7 +398,7 @@ fn dns_compute_update(
         debug!(
             log,
             "adding name";
-            "name" => name,
+            "dns_name" => name,
             "new_records" => ?new_records,
         );
         update.add_name(
@@ -289,7 +411,7 @@ fn dns_compute_update(
         debug!(
             log,
             "removing name";
-            "name" => name,
+            "dns_name" => name,
             "old_records" => ?old_records,
         );
         update.remove_name(name.to_string())?;
@@ -299,7 +421,7 @@ fn dns_compute_update(
         debug!(
             log,
             "updating name";
-            "name" => name,
+            "dns_name" => name,
             "old_records" => ?old_records,
             "new_records" => ?new_records,
         );
@@ -313,42 +435,83 @@ fn dns_compute_update(
     Ok(Some(update))
 }
 
+/// Returns the (relative) DNS name for this Silo's API and console endpoints
+/// _within_ the external DNS zone (i.e., without that zone's suffix)
+///
+/// This specific naming scheme is determined under RFD 357.
+pub fn silo_dns_name(name: &omicron_common::api::external::Name) -> String {
+    // RFD 4 constrains resource names (including Silo names) to DNS-safe
+    // strings, which is why it's safe to directly put the name of the
+    // resource into the DNS name rather than doing any kind of escaping.
+    format!("{}.sys", name)
+}
+
 #[cfg(test)]
 mod test {
-    use super::blueprint_dns_config;
-    use super::dns_compute_update;
+    use super::*;
+    use crate::overridables::Overridables;
     use crate::Sled;
+    use dns_service_client::DnsDiff;
     use internal_dns::ServiceName;
     use internal_dns::DNS_ZONE;
+    use nexus_db_model::DnsGroup;
+    use nexus_db_model::Silo;
+    use nexus_db_queries::authn;
+    use nexus_db_queries::authz;
+    use nexus_db_queries::context::OpContext;
+    use nexus_db_queries::db::DataStore;
     use nexus_inventory::CollectionBuilder;
     use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder;
+    use nexus_reconfigurator_planning::blueprint_builder::EnsureMultiple;
+    use nexus_reconfigurator_planning::example::example;
+    use nexus_reconfigurator_preparation::policy_from_db;
+    use nexus_test_utils::resource_helpers::create_silo;
+    use nexus_test_utils_macros::nexus_test;
     use nexus_types::deployment::Blueprint;
+    use nexus_types::deployment::BlueprintTarget;
     use nexus_types::deployment::OmicronZoneConfig;
     use nexus_types::deployment::OmicronZoneType;
     use nexus_types::deployment::Policy;
     use nexus_types::deployment::SledResources;
     use nexus_types::deployment::ZpoolName;
+    use nexus_types::external_api::params;
+    use nexus_types::external_api::shared;
     use nexus_types::external_api::views::SledPolicy;
     use nexus_types::external_api::views::SledState;
+    use nexus_types::identity::Resource;
     use nexus_types::internal_api::params::DnsConfigParams;
     use nexus_types::internal_api::params::DnsConfigZone;
     use nexus_types::internal_api::params::DnsRecord;
+    use nexus_types::internal_api::params::Srv;
     use omicron_common::address::get_sled_address;
     use omicron_common::address::get_switch_zone_address;
+    use omicron_common::address::IpRange;
     use omicron_common::address::Ipv6Subnet;
+    use omicron_common::address::NEXUS_REDUNDANCY;
     use omicron_common::address::RACK_PREFIX;
     use omicron_common::address::SLED_PREFIX;
+    use omicron_common::api::external::Error;
     use omicron_common::api::external::Generation;
+    use omicron_common::api::external::IdentityMetadataCreateParams;
+    use omicron_test_utils::dev::poll::wait_for_condition;
+    use omicron_test_utils::dev::poll::CondCheckError;
     use omicron_test_utils::dev::test_setup_log;
+    use slog::{debug, info};
     use std::collections::BTreeMap;
     use std::collections::BTreeSet;
     use std::collections::HashMap;
+    use std::net::IpAddr;
     use std::net::Ipv4Addr;
     use std::net::Ipv6Addr;
     use std::net::SocketAddrV6;
     use std::str::FromStr;
+    use std::sync::Arc;
+    use std::time::Duration;
     use uuid::Uuid;
 
+    type ControlPlaneTestContext =
+        nexus_test_utils::ControlPlaneTestContext<omicron_nexus::Server>;
+
     fn blueprint_empty() -> Blueprint {
         let builder = CollectionBuilder::new("test-suite");
         let collection = builder.build();
@@ -360,6 +523,7 @@ mod test {
         BlueprintBuilder::build_initial_from_collection(
             &collection,
             Generation::new(),
+            Generation::new(),
             &policy,
             "test-suite",
         )
@@ -377,11 +541,16 @@ mod test {
         }
     }
 
-    /// test blueprint_dns_config(): trivial case of an empty blueprint
+    /// test blueprint_internal_dns_config(): trivial case of an empty blueprint
     #[test]
-    fn test_blueprint_dns_empty() {
+    fn test_blueprint_internal_dns_empty() {
         let blueprint = blueprint_empty();
-        let blueprint_dns = blueprint_dns_config(&blueprint, &BTreeMap::new());
+        let blueprint_dns = blueprint_internal_dns_config(
+            &blueprint,
+            &BTreeMap::new(),
+            &Default::default(),
+        )
+        .unwrap();
         assert!(blueprint_dns.sole_zone().unwrap().records.is_empty());
     }
 
@@ -389,7 +558,7 @@ mod test {
     /// - one of each type of zone in service
     /// - some zones not in service
     #[test]
-    fn test_blueprint_dns_basic() {
+    fn test_blueprint_internal_dns_basic() {
         // We'll use the standard representative inventory collection to build a
         // blueprint.  The main thing we care about here is that we have at
         // least one zone of each type.  Later, we'll mark a couple of the sleds
@@ -434,6 +603,7 @@ mod test {
         let mut blueprint = BlueprintBuilder::build_initial_from_collection(
             &collection,
             initial_dns_generation,
+            Generation::new(),
             &policy,
             "test-suite",
         )
@@ -477,8 +647,12 @@ mod test {
             })
             .collect();
 
-        let dns_config_blueprint =
-            blueprint_dns_config(&blueprint, &sleds_by_id);
+        let dns_config_blueprint = blueprint_internal_dns_config(
+            &blueprint,
+            &sleds_by_id,
+            &Default::default(),
+        )
+        .unwrap();
         assert_eq!(
             dns_config_blueprint.generation,
             u64::from(initial_dns_generation.next())
@@ -655,6 +829,143 @@ mod test {
         assert!(srv_kinds_expected.is_empty());
     }
 
+    #[tokio::test]
+    async fn test_blueprint_external_dns_basic() {
+        static TEST_NAME: &str = "test_blueprint_external_dns_basic";
+        let logctx = test_setup_log(TEST_NAME);
+        let (collection, policy) = example(&logctx.log, TEST_NAME, 5);
+        let initial_external_dns_generation = Generation::new();
+        let blueprint = BlueprintBuilder::build_initial_from_collection(
+            &collection,
+            Generation::new(),
+            initial_external_dns_generation,
+            &policy,
+            "test suite",
+        )
+        .expect("failed to generate initial blueprint");
+
+        let my_silo = Silo::new(params::SiloCreate {
+            identity: IdentityMetadataCreateParams {
+                name: "my-silo".parse().unwrap(),
+                description: String::new(),
+            },
+            quotas: params::SiloQuotasCreate::empty(),
+            discoverable: false,
+            identity_mode: shared::SiloIdentityMode::SamlJit,
+            admin_group_name: None,
+            tls_certificates: vec![],
+            mapped_fleet_roles: Default::default(),
+        })
+        .unwrap();
+
+        let nexus_external_ips: Vec<_> = blueprint
+            .all_omicron_zones()
+            .filter_map(|(_, z)| match &z.zone_type {
+                OmicronZoneType::Nexus { external_ip, .. } => {
+                    Some(*external_ip)
+                }
+                _ => None,
+            })
+            .collect();
+
+        // It shouldn't ever be possible to have no Silos at all, but at least
+        // make sure we don't panic.
+        let external_dns_config = blueprint_external_dns_config(
+            &blueprint,
+            &nexus_external_ips,
+            &[],
+            &[String::from("oxide.test")],
+        );
+        assert_eq!(
+            external_dns_config.generation,
+            u64::from(initial_external_dns_generation.next())
+        );
+        assert_eq!(external_dns_config.zones.len(), 1);
+        assert_eq!(external_dns_config.zones[0].zone_name, "oxide.test");
+        assert!(external_dns_config.zones[0].records.is_empty());
+
+        // Same with external DNS zones.
+        let external_dns_config = blueprint_external_dns_config(
+            &blueprint,
+            &nexus_external_ips,
+            std::slice::from_ref(&my_silo),
+            &[],
+        );
+        assert_eq!(
+            external_dns_config.generation,
+            u64::from(initial_external_dns_generation.next())
+        );
+        assert!(external_dns_config.zones.is_empty());
+
+        // Same with external IPs.
+        let external_dns_config = blueprint_external_dns_config(
+            &blueprint,
+            &[],
+            std::slice::from_ref(&my_silo),
+            &[String::from("oxide.test")],
+        );
+        assert_eq!(
+            external_dns_config.generation,
+            u64::from(initial_external_dns_generation.next())
+        );
+
+        // Now check a more typical case.  (Although we wouldn't normally have
+        // more than one external DNS zone, it's a more general case and pretty
+        // easy to test.)
+        let external_dns_config = blueprint_external_dns_config(
+            &blueprint,
+            &nexus_external_ips,
+            std::slice::from_ref(&my_silo),
+            &[String::from("oxide1.test"), String::from("oxide2.test")],
+        );
+        assert_eq!(
+            external_dns_config.generation,
+            u64::from(initial_external_dns_generation.next())
+        );
+        assert_eq!(external_dns_config.zones.len(), 2);
+        assert_eq!(
+            external_dns_config.zones[0].records,
+            external_dns_config.zones[1].records
+        );
+        assert_eq!(
+            external_dns_config.zones[0].zone_name,
+            String::from("oxide1.test"),
+        );
+        assert_eq!(
+            external_dns_config.zones[1].zone_name,
+            String::from("oxide2.test"),
+        );
+        let records = &external_dns_config.zones[0].records;
+        assert_eq!(records.len(), 1);
+        let silo_records = records
+            .get(&silo_dns_name(my_silo.name()))
+            .expect("missing silo DNS records");
+
+        // Here we're hardcoding the contents of the example blueprint.  It
+        // currently puts one Nexus zone on each sled.  If we change the example
+        // blueprint, change the expected set of IPs here.
+        let mut silo_record_ips: Vec<_> = silo_records
+            .into_iter()
+            .map(|record| match record {
+                DnsRecord::A(v) => IpAddr::V4(*v),
+                DnsRecord::Aaaa(v) => IpAddr::V6(*v),
+                DnsRecord::Srv(_) => panic!("unexpected SRV record"),
+            })
+            .collect();
+        silo_record_ips.sort();
+        assert_eq!(
+            silo_record_ips,
+            &[
+                "192.0.2.2".parse::<IpAddr>().unwrap(),
+                "192.0.2.3".parse::<IpAddr>().unwrap(),
+                "192.0.2.4".parse::<IpAddr>().unwrap(),
+                "192.0.2.5".parse::<IpAddr>().unwrap(),
+                "192.0.2.6".parse::<IpAddr>().unwrap(),
+            ]
+        );
+        logctx.cleanup_successful();
+    }
+
     #[test]
     fn test_dns_compute_update() {
         let logctx = test_setup_log("dns_compute_update");
@@ -664,6 +975,7 @@ mod test {
         let dns_empty = dns_config_empty();
         match dns_compute_update(
             &logctx.log,
+            DnsGroup::Internal,
             "test-suite".to_string(),
             "test-suite".to_string(),
             &dns_empty,
@@ -717,6 +1029,7 @@ mod test {
 
         let update = dns_compute_update(
             &logctx.log,
+            DnsGroup::Internal,
             "test-suite".to_string(),
             "test-suite".to_string(),
             &dns_config1,
@@ -742,6 +1055,544 @@ mod test {
             ]
         );
 
+        // Test the difference between two configs whose SRV records differ.
+        let mut dns_config1 = dns_config1.clone();
+        dns_config1.zones[0].records.insert(
+            String::from("_nexus._tcp"),
+            vec![
+                DnsRecord::Srv(Srv {
+                    port: 123,
+                    prio: 1,
+                    target: String::from("ex1.my-zone"),
+                    weight: 2,
+                }),
+                DnsRecord::Srv(Srv {
+                    port: 123,
+                    prio: 1,
+                    target: String::from("ex2.my-zone"),
+                    weight: 2,
+                }),
+            ],
+        );
+        // A clone of the same one should of course be the same as the original.
+        let mut dns_config2 = dns_config1.clone();
+        let update = dns_compute_update(
+            &logctx.log,
+            DnsGroup::Internal,
+            "test-suite".to_string(),
+            "test-suite".to_string(),
+            &dns_config1,
+            &dns_config2,
+        )
+        .expect("failed to compute update");
+        assert!(update.is_none());
+
+        // If we shift the order of the items, it should still reflect no
+        // changes.
+        let records =
+            dns_config2.zones[0].records.get_mut("_nexus._tcp").unwrap();
+        records.rotate_left(1);
+        assert!(
+            records != dns_config1.zones[0].records.get("_nexus._tcp").unwrap()
+        );
+        let update = dns_compute_update(
+            &logctx.log,
+            DnsGroup::Internal,
+            "test-suite".to_string(),
+            "test-suite".to_string(),
+            &dns_config1,
+            &dns_config2,
+        )
+        .expect("failed to compute update");
+        assert!(update.is_none());
+
+        // If we add another record, there should indeed be a new update.
+        let records =
+            dns_config2.zones[0].records.get_mut("_nexus._tcp").unwrap();
+        records.push(DnsRecord::Srv(Srv {
+            port: 123,
+            prio: 1,
+            target: String::from("ex3.my-zone"),
+            weight: 2,
+        }));
+        let final_records = records.clone();
+
+        let update = dns_compute_update(
+            &logctx.log,
+            DnsGroup::Internal,
+            "test-suite".to_string(),
+            "test-suite".to_string(),
+            &dns_config1,
+            &dns_config2,
+        )
+        .expect("failed to compute update")
+        .expect("expected an update");
+
+        assert_eq!(
+            update.names_removed().collect::<Vec<_>>(),
+            &["_nexus._tcp"]
+        );
+        assert_eq!(
+            update.names_added().collect::<Vec<_>>(),
+            &[("_nexus._tcp", final_records.as_slice())]
+        );
+
         logctx.cleanup_successful();
     }
+
+    // Tests end-to-end DNS behavior:
+    //
+    // - If we create a blueprint matching the current system, and then apply
+    //   it, there are no changes to either internal or external DNS
+    //
+    // - If we create a Silo, DNS will be updated.  If we then re-execute the
+    //   previous blueprint, again, there will be no new changes to DNS.
+    //
+    // - If we then generate a blueprint with a Nexus zone and execute the DNS
+    //   part of that, then:
+    //
+    //   - internal DNS SRV record for _nexus._tcp is added
+    //   - internal DNS AAAA record for the new zone is added
+    //   - external DNS gets a A record for the new zone's external IP
+    //
+    // - If we subsequently create a new Silo, the new Silo's DNS record
+    //   reflects the Nexus zone that was added.
+    #[nexus_test]
+    async fn test_silos_external_dns_end_to_end(
+        cptestctx: &ControlPlaneTestContext,
+    ) {
+        let nexus = &cptestctx.server.apictx().nexus;
+        let datastore = nexus.datastore();
+        let log = &cptestctx.logctx.log;
+        let opctx = OpContext::for_background(
+            log.clone(),
+            Arc::new(authz::Authz::new(log)),
+            authn::Context::internal_api(),
+            datastore.clone(),
+        );
+
+        // First, wait until Nexus has successfully completed an inventory
+        // collection.
+        let collection = wait_for_condition(
+            || async {
+                let result =
+                    datastore.inventory_get_latest_collection(&opctx).await;
+                let log_result = match &result {
+                    Ok(Some(_)) => Ok("found"),
+                    Ok(None) => Ok("not found"),
+                    Err(error) => Err(error),
+                };
+                debug!(
+                    log,
+                    "attempt to fetch latest inventory collection";
+                    "result" => ?log_result,
+                );
+
+                match result {
+                    Ok(None) => Err(CondCheckError::NotYet),
+                    Ok(Some(c)) => Ok(c),
+                    Err(Error::ServiceUnavailable { .. }) => {
+                        Err(CondCheckError::NotYet)
+                    }
+                    Err(error) => Err(CondCheckError::Failed(error)),
+                }
+            },
+            &Duration::from_millis(50),
+            &Duration::from_secs(30),
+        )
+        .await
+        .expect("expected to find inventory collection");
+
+        // Fetch the initial contents of internal and external DNS.
+        let dns_initial_internal = datastore
+            .dns_config_read(&opctx, DnsGroup::Internal)
+            .await
+            .expect("fetching initial internal DNS");
+        let dns_initial_external = datastore
+            .dns_config_read(&opctx, DnsGroup::External)
+            .await
+            .expect("fetching initial external DNS");
+
+        // Now, use the collection to construct an initial blueprint.
+        // This stores it into the database, too.
+        info!(log, "using collection"; "collection_id" => %collection.id);
+        let blueprint = nexus
+            .blueprint_generate_from_collection(&opctx, collection.id)
+            .await
+            .expect("failed to generate initial blueprint");
+        eprintln!("blueprint: {:?}", blueprint);
+
+        // Set it as the current target.  We'll need this later.
+        datastore
+            .blueprint_target_set_current(
+                &opctx,
+                BlueprintTarget {
+                    target_id: blueprint.id,
+                    enabled: false,
+                    time_made_target: chrono::Utc::now(),
+                },
+            )
+            .await
+            .expect("failed to set blueprint as target");
+
+        // Now, execute the blueprint.
+        let overrides = Overridables::for_test(cptestctx);
+        crate::realize_blueprint_with_overrides(
+            &opctx,
+            datastore,
+            &blueprint,
+            "test-suite",
+            &overrides,
+        )
+        .await
+        .expect("failed to execute initial blueprint");
+
+        // DNS ought not to have changed.
+        verify_dns_unchanged(
+            &opctx,
+            datastore,
+            &dns_initial_internal,
+            &dns_initial_external,
+        )
+        .await;
+
+        // Create a Silo.  Make sure that external DNS is updated (and that
+        // internal DNS is not).  Then make sure that if we execute the same
+        // blueprint again, DNS does not change again (i.e., that it does not
+        // revert somehow).
+        let dns_latest_external = create_silo_and_verify_dns(
+            cptestctx,
+            &opctx,
+            datastore,
+            &blueprint,
+            &overrides,
+            "squidport",
+            &dns_initial_internal,
+            &dns_initial_external,
+        )
+        .await;
+
+        // Now, go through the motions of provisioning a new Nexus zone.
+        // We do this directly with BlueprintBuilder to avoid the planner
+        // deciding to make other unrelated changes.
+        let sled_rows = datastore.sled_list_all_batched(&opctx).await.unwrap();
+        let zpool_rows =
+            datastore.zpool_list_all_external_batched(&opctx).await.unwrap();
+        let ip_pool_range_rows = {
+            let (authz_service_ip_pool, _) =
+                datastore.ip_pools_service_lookup(&opctx).await.unwrap();
+            datastore
+                .ip_pool_list_ranges_batched(&opctx, &authz_service_ip_pool)
+                .await
+                .unwrap()
+        };
+        let mut policy = policy_from_db(
+            &sled_rows,
+            &zpool_rows,
+            &ip_pool_range_rows,
+            // This is not used because we're not actually going through the
+            // planner.
+            NEXUS_REDUNDANCY,
+        )
+        .unwrap();
+        // We'll need another (fake) external IP for this new Nexus.
+        policy
+            .service_ip_pool_ranges
+            .push(IpRange::from(IpAddr::V4(Ipv4Addr::LOCALHOST)));
+        let mut builder = BlueprintBuilder::new_based_on(
+            &log,
+            &blueprint,
+            Generation::from(
+                u32::try_from(dns_initial_internal.generation).unwrap(),
+            ),
+            Generation::from(
+                u32::try_from(dns_latest_external.generation).unwrap(),
+            ),
+            &policy,
+            "test suite",
+        )
+        .unwrap();
+        let sled_id =
+            blueprint.sleds().next().expect("expected at least one sled");
+        let nalready = builder.sled_num_nexus_zones(sled_id);
+        let rv = builder
+            .sled_ensure_zone_multiple_nexus(sled_id, nalready + 1)
+            .unwrap();
+        assert_eq!(rv, EnsureMultiple::Added(1));
+        let blueprint2 = builder.build();
+        eprintln!("blueprint2: {:?}", blueprint2);
+        // Figure out the id of the new zone.
+        let zones_before = blueprint
+            .all_omicron_zones()
+            .filter_map(|(_, z)| z.zone_type.is_nexus().then_some(z.id))
+            .collect::<BTreeSet<_>>();
+        let zones_after = blueprint2
+            .all_omicron_zones()
+            .filter_map(|(_, z)| z.zone_type.is_nexus().then_some(z.id))
+            .collect::<BTreeSet<_>>();
+        let new_zones: Vec<_> = zones_after.difference(&zones_before).collect();
+        assert_eq!(new_zones.len(), 1);
+        let new_zone_id = *new_zones[0];
+
+        // Set this blueprint as the current target.  We set it to disabled
+        // because we're controlling the execution directly here.  But we need
+        // to do this so that silo creation sees the change.
+        datastore
+            .blueprint_insert(&opctx, &blueprint2)
+            .await
+            .expect("failed to save blueprint to database");
+        datastore
+            .blueprint_target_set_current(
+                &opctx,
+                BlueprintTarget {
+                    target_id: blueprint2.id,
+                    enabled: false,
+                    time_made_target: chrono::Utc::now(),
+                },
+            )
+            .await
+            .expect("failed to set blueprint as target");
+
+        crate::realize_blueprint_with_overrides(
+            &opctx,
+            datastore,
+            &blueprint2,
+            "test-suite",
+            &overrides,
+        )
+        .await
+        .expect("failed to execute second blueprint");
+
+        // Now fetch DNS again.  Both should have changed this time.
+        let dns_latest_internal = datastore
+            .dns_config_read(&opctx, DnsGroup::Internal)
+            .await
+            .expect("fetching latest internal DNS");
+
+        assert_eq!(
+            dns_latest_internal.generation,
+            dns_initial_internal.generation + 1,
+        );
+
+        let diff =
+            DnsDiff::new(&dns_initial_internal, &dns_latest_internal).unwrap();
+        // There should be one new AAAA record for the zone itself.
+        let new_records: Vec<_> = diff.names_added().collect();
+        let (new_name, &[DnsRecord::Aaaa(_)]) = new_records[0] else {
+            panic!("did not find expected AAAA record for new Nexus zone");
+        };
+        let new_zone_host = internal_dns::config::Host::for_zone(
+            new_zone_id,
+            internal_dns::config::ZoneVariant::Other,
+        );
+        assert!(new_zone_host.fqdn().starts_with(new_name));
+
+        // Nothing was removed.
+        assert!(diff.names_removed().next().is_none());
+
+        // The SRV record for Nexus itself ought to have changed, growing one
+        // more record -- for the new AAAA record above.
+        let changed: Vec<_> = diff.names_changed().collect();
+        assert_eq!(changed.len(), 1);
+        let (name, old_records, new_records) = changed[0];
+        assert_eq!(name, ServiceName::Nexus.dns_name());
+        let new_srv = subset_plus_one(old_records, new_records);
+        let DnsRecord::Srv(new_srv) = new_srv else {
+            panic!("expected SRV record, found {:?}", new_srv);
+        };
+        assert_eq!(new_srv.target, new_zone_host.fqdn());
+
+        // As for external DNS: all existing names ought to have been changed,
+        // gaining a new A record for the new host.
+        let dns_previous_external = dns_latest_external;
+        let dns_latest_external = datastore
+            .dns_config_read(&opctx, DnsGroup::External)
+            .await
+            .expect("fetching latest external DNS");
+        assert_eq!(
+            dns_latest_external.generation,
+            dns_previous_external.generation + 1,
+        );
+        let diff =
+            DnsDiff::new(&dns_previous_external, &dns_latest_external).unwrap();
+        assert!(diff.names_added().next().is_none());
+        assert!(diff.names_removed().next().is_none());
+        let changed: Vec<_> = diff.names_changed().collect();
+        for (name, old_records, new_records) in changed {
+            // These are Silo names and end with ".sys".
+            assert!(name.ends_with(".sys"));
+            // We can't really tell which one points to what, especially in the
+            // test suite where all Nexus zones use localhost for their external
+            // IP.  All we can tell is that there's one new one.
+            assert_eq!(old_records.len() + 1, new_records.len());
+        }
+
+        // If we execute it again, we should see no more changes.
+        crate::realize_blueprint_with_overrides(
+            &opctx,
+            datastore,
+            &blueprint2,
+            "test-suite",
+            &overrides,
+        )
+        .await
+        .expect("failed to execute second blueprint again");
+        verify_dns_unchanged(
+            &opctx,
+            datastore,
+            &dns_latest_internal,
+            &dns_latest_external,
+        )
+        .await;
+
+        // Now create another Silo and verify the changes to DNS.
+        // This ensures that the "create Silo" path picks up Nexus instances
+        // that exist only in Reconfigurator, not the services table.
+        let dns_latest_external = create_silo_and_verify_dns(
+            &cptestctx,
+            &opctx,
+            datastore,
+            &blueprint2,
+            &overrides,
+            "tickety-boo",
+            &dns_latest_internal,
+            &dns_latest_external,
+        )
+        .await;
+
+        // One more time, make sure that executing the blueprint does not do
+        // anything.
+        crate::realize_blueprint_with_overrides(
+            &opctx,
+            datastore,
+            &blueprint2,
+            "test-suite",
+            &overrides,
+        )
+        .await
+        .expect("failed to execute second blueprint again");
+        verify_dns_unchanged(
+            &opctx,
+            datastore,
+            &dns_latest_internal,
+            &dns_latest_external,
+        )
+        .await;
+    }
+
+    fn subset_plus_one<'a, T: std::fmt::Debug + Ord + Eq>(
+        list1: &'a [T],
+        list2: &'a [T],
+    ) -> &'a T {
+        let set: BTreeSet<_> = list1.into_iter().collect();
+        let mut extra = Vec::with_capacity(1);
+        for item in list2 {
+            if !set.contains(&item) {
+                extra.push(item);
+            }
+        }
+
+        if extra.len() != 1 {
+            panic!(
+                "expected list2 to have one extra element:\n\
+                list1: {:?}\n\
+                list2: {:?}\n
+                extra: {:?}\n",
+                list1, list2, extra
+            );
+        }
+
+        extra.into_iter().next().unwrap()
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    async fn create_silo_and_verify_dns(
+        cptestctx: &ControlPlaneTestContext,
+        opctx: &OpContext,
+        datastore: &DataStore,
+        blueprint: &Blueprint,
+        overrides: &Overridables,
+        silo_name: &str,
+        old_internal: &DnsConfigParams,
+        old_external: &DnsConfigParams,
+    ) -> DnsConfigParams {
+        // Create a Silo.  Make sure that external DNS is updated (and that
+        // internal DNS is not).  This is tested elsewhere already but really we
+        // want to make sure that if we then execute the blueprint again, DNS
+        // does not change _again_ (i.e., does not somehow revert).
+        let silo = create_silo(
+            &cptestctx.external_client,
+            silo_name,
+            false,
+            shared::SiloIdentityMode::SamlJit,
+        )
+        .await;
+
+        let dns_latest_internal = datastore
+            .dns_config_read(&opctx, DnsGroup::Internal)
+            .await
+            .expect("fetching latest internal DNS");
+        assert_eq!(old_internal.generation, dns_latest_internal.generation);
+        let dns_latest_external = datastore
+            .dns_config_read(&opctx, DnsGroup::External)
+            .await
+            .expect("fetching latest external DNS");
+        assert_eq!(old_external.generation + 1, dns_latest_external.generation);
+
+        // Specifically, there should be one new name (for the new Silo).
+        let diff = DnsDiff::new(&old_external, &dns_latest_external).unwrap();
+        assert!(diff.names_removed().next().is_none());
+        assert!(diff.names_changed().next().is_none());
+        let added = diff.names_added().collect::<Vec<_>>();
+        assert_eq!(added.len(), 1);
+        let (new_name, new_records) = added[0];
+        assert_eq!(new_name, silo_dns_name(&silo.identity.name));
+        // And it should have the same IP addresses as all of the other Silos.
+        assert_eq!(
+            new_records,
+            old_external.zones[0].records.values().next().unwrap()
+        );
+
+        // If we execute the blueprint, DNS should not be changed.
+        crate::realize_blueprint_with_overrides(
+            &opctx,
+            datastore,
+            &blueprint,
+            "test-suite",
+            &overrides,
+        )
+        .await
+        .expect("failed to execute blueprint");
+        let dns_latest_internal = datastore
+            .dns_config_read(&opctx, DnsGroup::Internal)
+            .await
+            .expect("fetching latest internal DNS");
+        let dns_latest_external = datastore
+            .dns_config_read(&opctx, DnsGroup::External)
+            .await
+            .expect("fetching latest external DNS");
+        assert_eq!(old_internal.generation, dns_latest_internal.generation);
+        assert_eq!(old_external.generation + 1, dns_latest_external.generation);
+
+        dns_latest_external
+    }
+
+    async fn verify_dns_unchanged(
+        opctx: &OpContext,
+        datastore: &DataStore,
+        old_internal: &DnsConfigParams,
+        old_external: &DnsConfigParams,
+    ) {
+        let dns_latest_internal = datastore
+            .dns_config_read(&opctx, DnsGroup::Internal)
+            .await
+            .expect("fetching latest internal DNS");
+        let dns_latest_external = datastore
+            .dns_config_read(&opctx, DnsGroup::External)
+            .await
+            .expect("fetching latest external DNS");
+
+        assert_eq!(old_internal.generation, dns_latest_internal.generation);
+        assert_eq!(old_external.generation, dns_latest_external.generation);
+    }
 }
diff --git a/nexus/reconfigurator/execution/src/lib.rs b/nexus/reconfigurator/execution/src/lib.rs
index 386e9dc1c8..8da4e7d44c 100644
--- a/nexus/reconfigurator/execution/src/lib.rs
+++ b/nexus/reconfigurator/execution/src/lib.rs
@@ -13,15 +13,19 @@ use nexus_types::deployment::Blueprint;
 use nexus_types::identity::Asset;
 use omicron_common::address::Ipv6Subnet;
 use omicron_common::address::SLED_PREFIX;
+use overridables::Overridables;
 use slog::info;
 use slog_error_chain::InlineErrorChain;
 use std::collections::BTreeMap;
 use std::net::SocketAddrV6;
 use uuid::Uuid;
 
+pub use dns::silo_dns_name;
+
 mod datasets;
 mod dns;
 mod omicron_zones;
+mod overridables;
 mod resource_allocation;
 
 struct Sled {
@@ -57,6 +61,26 @@ pub async fn realize_blueprint<S>(
     blueprint: &Blueprint,
     nexus_label: S,
 ) -> Result<(), Vec<anyhow::Error>>
+where
+    String: From<S>,
+{
+    realize_blueprint_with_overrides(
+        opctx,
+        datastore,
+        blueprint,
+        nexus_label,
+        &Default::default(),
+    )
+    .await
+}
+
+pub async fn realize_blueprint_with_overrides<S>(
+    opctx: &OpContext,
+    datastore: &DataStore,
+    blueprint: &Blueprint,
+    nexus_label: S,
+    overrides: &Overridables,
+) -> Result<(), Vec<anyhow::Error>>
 where
     String: From<S>,
 {
@@ -123,6 +147,7 @@ where
         String::from(nexus_label),
         blueprint,
         &sleds_by_id,
+        &overrides,
     )
     .await
     .map_err(|e| vec![anyhow!("{}", InlineErrorChain::new(&e))])?;
diff --git a/nexus/reconfigurator/execution/src/omicron_zones.rs b/nexus/reconfigurator/execution/src/omicron_zones.rs
index 336e40df16..6054a40f8a 100644
--- a/nexus/reconfigurator/execution/src/omicron_zones.rs
+++ b/nexus/reconfigurator/execution/src/omicron_zones.rs
@@ -108,6 +108,7 @@ mod test {
                 zones_in_service: BTreeSet::new(),
                 parent_blueprint_id: None,
                 internal_dns_version: Generation::new(),
+                external_dns_version: Generation::new(),
                 time_created: chrono::Utc::now(),
                 creator: "test".to_string(),
                 comment: "test blueprint".to_string(),
diff --git a/nexus/reconfigurator/execution/src/overridables.rs b/nexus/reconfigurator/execution/src/overridables.rs
new file mode 100644
index 0000000000..5c4ce7dc6f
--- /dev/null
+++ b/nexus/reconfigurator/execution/src/overridables.rs
@@ -0,0 +1,121 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use omicron_common::address::get_switch_zone_address;
+use omicron_common::address::Ipv6Subnet;
+use omicron_common::address::DENDRITE_PORT;
+use omicron_common::address::MGD_PORT;
+use omicron_common::address::MGS_PORT;
+use omicron_common::address::SLED_PREFIX;
+use std::collections::BTreeMap;
+use std::net::Ipv6Addr;
+use uuid::Uuid;
+
+/// Override values used during blueprint execution
+///
+/// Blueprint execution assumes certain values about production systems that
+/// differ in the simulated testing environment and cannot be easily derived
+/// from anything else in the environment.  To accommodate this, this structure
+/// provides access to these values.  Everywhere except the test suite, this
+/// structure is empty and returns the default (production) values.  The test
+/// suite overrides these values.
+#[derive(Debug, Default)]
+pub struct Overridables {
+    /// map: sled id -> TCP port on which that sled's Dendrite is listening
+    pub dendrite_ports: BTreeMap<Uuid, u16>,
+    /// map: sled id -> TCP port on which that sled's MGS is listening
+    pub mgs_ports: BTreeMap<Uuid, u16>,
+    /// map: sled id -> TCP port on which that sled's MGD is listening
+    pub mgd_ports: BTreeMap<Uuid, u16>,
+    /// map: sled id -> IP address of the sled's switch zone
+    pub switch_zone_ips: BTreeMap<Uuid, Ipv6Addr>,
+}
+
+impl Overridables {
+    /// Specify the TCP port on which this sled's Dendrite is listening
+    #[cfg(test)]
+    fn override_dendrite_port(&mut self, sled_id: Uuid, port: u16) {
+        self.dendrite_ports.insert(sled_id, port);
+    }
+
+    /// Returns the TCP port on which this sled's Dendrite is listening
+    pub fn dendrite_port(&self, sled_id: Uuid) -> u16 {
+        self.dendrite_ports.get(&sled_id).copied().unwrap_or(DENDRITE_PORT)
+    }
+
+    /// Specify the TCP port on which this sled's MGS is listening
+    #[cfg(test)]
+    fn override_mgs_port(&mut self, sled_id: Uuid, port: u16) {
+        self.mgs_ports.insert(sled_id, port);
+    }
+
+    /// Returns the TCP port on which this sled's MGS is listening
+    pub fn mgs_port(&self, sled_id: Uuid) -> u16 {
+        self.mgs_ports.get(&sled_id).copied().unwrap_or(MGS_PORT)
+    }
+
+    /// Specify the TCP port on which this sled's MGD is listening
+    #[cfg(test)]
+    fn override_mgd_port(&mut self, sled_id: Uuid, port: u16) {
+        self.mgd_ports.insert(sled_id, port);
+    }
+
+    /// Returns the TCP port on which this sled's MGD is listening
+    pub fn mgd_port(&self, sled_id: Uuid) -> u16 {
+        self.mgd_ports.get(&sled_id).copied().unwrap_or(MGD_PORT)
+    }
+
+    /// Specify the IP address of this switch zone
+    #[cfg(test)]
+    fn override_switch_zone_ip(&mut self, sled_id: Uuid, addr: Ipv6Addr) {
+        self.switch_zone_ips.insert(sled_id, addr);
+    }
+
+    /// Returns the IP address of this sled's switch zone
+    pub fn switch_zone_ip(
+        &self,
+        sled_id: Uuid,
+        sled_subnet: Ipv6Subnet<SLED_PREFIX>,
+    ) -> Ipv6Addr {
+        self.switch_zone_ips
+            .get(&sled_id)
+            .copied()
+            .unwrap_or_else(|| get_switch_zone_address(sled_subnet))
+    }
+
+    /// Generates a set of overrides describing the simulated test environment.
+    #[cfg(test)]
+    pub fn for_test(
+        cptestctx: &nexus_test_utils::ControlPlaneTestContext<
+            omicron_nexus::Server,
+        >,
+    ) -> Overridables {
+        use omicron_common::api::external::SwitchLocation;
+
+        let mut overrides = Overridables::default();
+        let scrimlets = [
+            (nexus_test_utils::SLED_AGENT_UUID, SwitchLocation::Switch0),
+            (nexus_test_utils::SLED_AGENT2_UUID, SwitchLocation::Switch1),
+        ];
+        for (id_str, switch_location) in scrimlets {
+            let sled_id = id_str.parse().unwrap();
+            let ip = Ipv6Addr::LOCALHOST;
+            let mgs_port = cptestctx
+                .gateway
+                .get(&switch_location)
+                .unwrap()
+                .client
+                .bind_address
+                .port();
+            let dendrite_port =
+                cptestctx.dendrite.get(&switch_location).unwrap().port;
+            let mgd_port = cptestctx.mgd.get(&switch_location).unwrap().port;
+            overrides.override_switch_zone_ip(sled_id, ip);
+            overrides.override_dendrite_port(sled_id, dendrite_port);
+            overrides.override_mgs_port(sled_id, mgs_port);
+            overrides.override_mgd_port(sled_id, mgd_port);
+        }
+        overrides
+    }
+}
diff --git a/nexus/reconfigurator/execution/src/resource_allocation.rs b/nexus/reconfigurator/execution/src/resource_allocation.rs
index 83a484baa4..5872cee0f9 100644
--- a/nexus/reconfigurator/execution/src/resource_allocation.rs
+++ b/nexus/reconfigurator/execution/src/resource_allocation.rs
@@ -93,6 +93,14 @@ impl<'a> ResourceAllocator<'a> {
         external_ip: IpAddr,
         port_range: Option<(u16, u16)>,
     ) -> anyhow::Result<bool> {
+        // localhost is used by many components in the test suite.  We can't use
+        // the normal path because normally a given external IP must only be
+        // used once.  Just treat localhost in the test suite as though it's
+        // already allocated.  We do the same in is_nic_already_allocated().
+        if cfg!(test) && external_ip.is_loopback() {
+            return Ok(true);
+        }
+
         let allocated_ips = self
             .datastore
             .service_lookup_external_ips(self.opctx, zone_id)
@@ -157,6 +165,11 @@ impl<'a> ResourceAllocator<'a> {
         zone_id: Uuid,
         nic: &NetworkInterface,
     ) -> anyhow::Result<bool> {
+        // See the comment in is_external_ip_already_allocated().
+        if cfg!(test) && nic.ip.is_loopback() {
+            return Ok(true);
+        }
+
         let allocated_nics = self
             .datastore
             .service_list_network_interfaces(self.opctx, zone_id)
diff --git a/nexus/reconfigurator/planning/Cargo.toml b/nexus/reconfigurator/planning/Cargo.toml
index 52111bab94..3c4ba12ee4 100644
--- a/nexus/reconfigurator/planning/Cargo.toml
+++ b/nexus/reconfigurator/planning/Cargo.toml
@@ -27,4 +27,3 @@ omicron-workspace-hack.workspace = true
 [dev-dependencies]
 expectorate.workspace = true
 omicron-test-utils.workspace = true
-sled-agent-client.workspace = true
diff --git a/nexus/reconfigurator/planning/src/blueprint_builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder.rs
index 7a99343bf9..8b0d440d26 100644
--- a/nexus/reconfigurator/planning/src/blueprint_builder.rs
+++ b/nexus/reconfigurator/planning/src/blueprint_builder.rs
@@ -110,6 +110,7 @@ pub struct BlueprintBuilder<'a> {
     /// previous blueprint, on which this one will be based
     parent_blueprint: &'a Blueprint,
     internal_dns_version: Generation,
+    external_dns_version: Generation,
 
     // These fields are used to allocate resources from sleds.
     policy: &'a Policy,
@@ -142,12 +143,14 @@ impl<'a> BlueprintBuilder<'a> {
     pub fn build_initial_from_collection(
         collection: &'a Collection,
         internal_dns_version: Generation,
+        external_dns_version: Generation,
         policy: &'a Policy,
         creator: &str,
     ) -> Result<Blueprint, Error> {
         Self::build_initial_impl(
             collection,
             internal_dns_version,
+            external_dns_version,
             policy,
             creator,
             BlueprintBuilderRng::new(),
@@ -159,6 +162,7 @@ impl<'a> BlueprintBuilder<'a> {
     pub fn build_initial_from_collection_seeded<H: Hash>(
         collection: &'a Collection,
         internal_dns_version: Generation,
+        external_dns_version: Generation,
         policy: &'a Policy,
         creator: &str,
         seed: H,
@@ -168,6 +172,7 @@ impl<'a> BlueprintBuilder<'a> {
         Self::build_initial_impl(
             collection,
             internal_dns_version,
+            external_dns_version,
             policy,
             creator,
             rng,
@@ -177,6 +182,7 @@ impl<'a> BlueprintBuilder<'a> {
     fn build_initial_impl(
         collection: &'a Collection,
         internal_dns_version: Generation,
+        external_dns_version: Generation,
         policy: &'a Policy,
         creator: &str,
         mut rng: BlueprintBuilderRng,
@@ -225,6 +231,7 @@ impl<'a> BlueprintBuilder<'a> {
             zones_in_service,
             parent_blueprint_id: None,
             internal_dns_version,
+            external_dns_version,
             time_created: now_db_precision(),
             creator: creator.to_owned(),
             comment: format!("from collection {}", collection.id),
@@ -237,6 +244,7 @@ impl<'a> BlueprintBuilder<'a> {
         log: &Logger,
         parent_blueprint: &'a Blueprint,
         internal_dns_version: Generation,
+        external_dns_version: Generation,
         policy: &'a Policy,
         creator: &str,
     ) -> anyhow::Result<BlueprintBuilder<'a>> {
@@ -298,7 +306,12 @@ impl<'a> BlueprintBuilder<'a> {
                 }
             }
             if let Some(external_ip) = z.zone_type.external_ip()? {
-                if !used_external_ips.insert(external_ip) {
+                // For the test suite, ignore localhost.  It gets reused many
+                // times and that's okay.  We don't expect to see localhost
+                // outside the test suite.
+                if !external_ip.is_loopback()
+                    && !used_external_ips.insert(external_ip)
+                {
                     bail!("duplicate external IP: {external_ip}");
                 }
             }
@@ -344,6 +357,7 @@ impl<'a> BlueprintBuilder<'a> {
             log,
             parent_blueprint,
             internal_dns_version,
+            external_dns_version,
             policy,
             sled_ip_allocators: BTreeMap::new(),
             zones: BlueprintZones::new(parent_blueprint),
@@ -369,6 +383,7 @@ impl<'a> BlueprintBuilder<'a> {
             zones_in_service: self.zones_in_service,
             parent_blueprint_id: Some(self.parent_blueprint.id),
             internal_dns_version: self.internal_dns_version,
+            external_dns_version: self.external_dns_version,
             time_created: now_db_precision(),
             creator: self.creator,
             comment: self.comments.join(", "),
@@ -754,8 +769,7 @@ impl UuidRng {
     }
 
     /// `extra` is a string that should be unique to the purpose of the UUIDs.
-    #[cfg(test)]
-    fn from_seed<H: Hash>(seed: H, extra: &'static str) -> Self {
+    pub(crate) fn from_seed<H: Hash>(seed: H, extra: &'static str) -> Self {
         let mut seeder = rand_seeder::Seeder::from((seed, extra));
         Self { rng: seeder.make_rng::<StdRng>() }
     }
@@ -872,145 +886,15 @@ impl<'a> BlueprintZones<'a> {
 #[cfg(test)]
 pub mod test {
     use super::*;
+    use crate::example::example;
+    use crate::example::ExampleSystem;
     use crate::system::SledBuilder;
-    use crate::system::SystemDescription;
     use omicron_common::address::IpRange;
     use omicron_test_utils::dev::test_setup_log;
-    use sled_agent_client::types::{
-        OmicronZoneConfig, OmicronZoneType, OmicronZonesConfig,
-    };
+    use sled_agent_client::types::{OmicronZoneConfig, OmicronZoneType};
 
     pub const DEFAULT_N_SLEDS: usize = 3;
 
-    pub struct ExampleSystem {
-        pub system: SystemDescription,
-        pub policy: Policy,
-        pub collection: Collection,
-        pub blueprint: Blueprint,
-        // If we add more types of RNGs than just sleds here, we'll need to
-        // expand this to be similar to BlueprintBuilderRng where a root RNG
-        // creates sub-RNGs.
-        pub(crate) sled_rng: UuidRng,
-    }
-
-    impl ExampleSystem {
-        pub fn new(
-            log: &slog::Logger,
-            test_name: &str,
-            nsleds: usize,
-        ) -> ExampleSystem {
-            let mut system = SystemDescription::new();
-            let mut sled_rng = UuidRng::from_seed(test_name, "ExampleSystem");
-
-            let sled_ids: Vec<_> =
-                (0..nsleds).map(|_| sled_rng.next_uuid()).collect();
-            for sled_id in &sled_ids {
-                let _ = system.sled(SledBuilder::new().id(*sled_id)).unwrap();
-            }
-
-            let policy = system.to_policy().expect("failed to make policy");
-            let mut inventory_builder = system
-                .to_collection_builder()
-                .expect("failed to build collection");
-
-            // For each sled, have it report 0 zones in the initial inventory.
-            // This will enable us to build a blueprint from the initial
-            // inventory, which we can then use to build new blueprints.
-            for sled_id in &sled_ids {
-                inventory_builder
-                    .found_sled_omicron_zones(
-                        "fake sled agent",
-                        *sled_id,
-                        OmicronZonesConfig {
-                            generation: Generation::new(),
-                            zones: vec![],
-                        },
-                    )
-                    .expect("recording Omicron zones");
-            }
-
-            let empty_zone_inventory = inventory_builder.build();
-            let initial_blueprint =
-                BlueprintBuilder::build_initial_from_collection_seeded(
-                    &empty_zone_inventory,
-                    Generation::new(),
-                    &policy,
-                    "test suite",
-                    (test_name, "ExampleSystem initial"),
-                )
-                .unwrap();
-
-            // Now make a blueprint and collection with some zones on each sled.
-            let mut builder = BlueprintBuilder::new_based_on(
-                &log,
-                &initial_blueprint,
-                Generation::new(),
-                &policy,
-                "test suite",
-            )
-            .unwrap();
-            builder.set_rng_seed((test_name, "ExampleSystem make_zones"));
-            for (sled_id, sled_resources) in &policy.sleds {
-                let _ = builder.sled_ensure_zone_ntp(*sled_id).unwrap();
-                let _ = builder
-                    .sled_ensure_zone_multiple_nexus_with_config(
-                        *sled_id,
-                        1,
-                        false,
-                        vec![],
-                    )
-                    .unwrap();
-                for pool_name in &sled_resources.zpools {
-                    let _ = builder
-                        .sled_ensure_zone_crucible(*sled_id, pool_name.clone())
-                        .unwrap();
-                }
-            }
-
-            let blueprint = builder.build();
-            let mut builder = system
-                .to_collection_builder()
-                .expect("failed to build collection");
-
-            for sled_id in blueprint.sleds() {
-                let Some(zones) = blueprint.omicron_zones.get(&sled_id) else {
-                    continue;
-                };
-                builder
-                    .found_sled_omicron_zones(
-                        "fake sled agent",
-                        sled_id,
-                        zones.clone(),
-                    )
-                    .unwrap();
-            }
-
-            ExampleSystem {
-                system,
-                policy,
-                collection: builder.build(),
-                blueprint,
-                sled_rng,
-            }
-        }
-    }
-
-    /// Returns a collection and policy describing a pretty simple system.
-    ///
-    /// The test name is used as the RNG seed.
-    ///
-    /// `n_sleds` is the number of sleds supported. Currently, this value can
-    /// be anywhere between 0 and 5. (More can be added in the future if
-    /// necessary.)
-    pub fn example(
-        log: &slog::Logger,
-        test_name: &str,
-        nsleds: usize,
-    ) -> (Collection, Policy) {
-        let example = ExampleSystem::new(log, test_name, nsleds);
-        (example.collection, example.policy)
-    }
-
     /// Checks various conditions that should be true for all blueprints
     pub fn verify_blueprint(blueprint: &Blueprint) {
         let mut underlay_ips: BTreeMap<Ipv6Addr, &OmicronZoneConfig> =
@@ -1042,6 +926,7 @@ pub mod test {
             BlueprintBuilder::build_initial_from_collection_seeded(
                 &collection,
                 Generation::new(),
+                Generation::new(),
                 &policy,
                 "the_test",
                 TEST_NAME,
@@ -1067,6 +952,7 @@ pub mod test {
             &logctx.log,
             &blueprint_initial,
             Generation::new(),
+            Generation::new(),
             &policy,
             "test_basic",
         )
@@ -1098,6 +984,7 @@ pub mod test {
             &logctx.log,
             blueprint1,
             Generation::new(),
+            Generation::new(),
             &example.policy,
             "test_basic",
         )
@@ -1135,6 +1022,7 @@ pub mod test {
             &logctx.log,
             &blueprint2,
             Generation::new(),
+            Generation::new(),
             &policy,
             "test_basic",
         )
@@ -1215,8 +1103,9 @@ pub mod test {
         let (mut collection, policy) =
             example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS);
 
-        // We don't care about the internal DNS version here.
+        // We don't care about the DNS versions here.
         let internal_dns_version = Generation::new();
+        let external_dns_version = Generation::new();
 
         // Adding a new Nexus zone currently requires copying settings from an
         // existing Nexus zone. If we remove all Nexus zones from the
@@ -1231,6 +1120,7 @@ pub mod test {
         let parent = BlueprintBuilder::build_initial_from_collection_seeded(
             &collection,
             internal_dns_version,
+            external_dns_version,
             &policy,
             "test",
             TEST_NAME,
@@ -1241,6 +1131,7 @@ pub mod test {
             &logctx.log,
             &parent,
             internal_dns_version,
+            external_dns_version,
             &policy,
             "test",
         )
@@ -1273,8 +1164,9 @@ pub mod test {
         let (mut collection, policy) =
             example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS);
 
-        // We don't care about the internal DNS version here.
+        // We don't care about the DNS versions here.
         let internal_dns_version = Generation::new();
+        let external_dns_version = Generation::new();
 
         // Remove the Nexus zone from one of the sleds so that
         // `sled_ensure_zone_nexus` can attempt to add a Nexus zone to
@@ -1297,6 +1189,7 @@ pub mod test {
         let parent = BlueprintBuilder::build_initial_from_collection_seeded(
             &collection,
             Generation::new(),
+            Generation::new(),
             &policy,
             "test",
             TEST_NAME,
@@ -1310,6 +1203,7 @@ pub mod test {
                 &logctx.log,
                 &parent,
                 internal_dns_version,
+                external_dns_version,
                 &policy,
                 "test",
             )
@@ -1329,6 +1223,7 @@ pub mod test {
                 &logctx.log,
                 &parent,
                 internal_dns_version,
+                external_dns_version,
                 &policy,
                 "test",
             )
@@ -1362,6 +1257,7 @@ pub mod test {
                 &logctx.log,
                 &parent,
                 internal_dns_version,
+                external_dns_version,
                 &policy,
                 "test",
             )
@@ -1424,6 +1320,7 @@ pub mod test {
         let parent = BlueprintBuilder::build_initial_from_collection_seeded(
             &collection,
             Generation::new(),
+            Generation::new(),
             &policy,
             "test",
             TEST_NAME,
@@ -1434,6 +1331,7 @@ pub mod test {
             &logctx.log,
             &parent,
             Generation::new(),
+            Generation::new(),
             &policy,
             "test",
         ) {
@@ -1482,6 +1380,7 @@ pub mod test {
         let parent = BlueprintBuilder::build_initial_from_collection_seeded(
             &collection,
             Generation::new(),
+            Generation::new(),
             &policy,
             "test",
             TEST_NAME,
@@ -1492,6 +1391,7 @@ pub mod test {
             &logctx.log,
             &parent,
             Generation::new(),
+            Generation::new(),
             &policy,
             "test",
         ) {
@@ -1540,6 +1440,7 @@ pub mod test {
         let parent = BlueprintBuilder::build_initial_from_collection_seeded(
             &collection,
             Generation::new(),
+            Generation::new(),
             &policy,
             "test",
             TEST_NAME,
@@ -1550,6 +1451,7 @@ pub mod test {
             &logctx.log,
             &parent,
             Generation::new(),
+            Generation::new(),
             &policy,
             "test",
         ) {
diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs
new file mode 100644
index 0000000000..c8c1bb380b
--- /dev/null
+++ b/nexus/reconfigurator/planning/src/example.rs
@@ -0,0 +1,147 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Example blueprints
+
+use crate::blueprint_builder::BlueprintBuilder;
+use crate::blueprint_builder::UuidRng;
+use crate::system::SledBuilder;
+use crate::system::SystemDescription;
+use nexus_types::deployment::Blueprint;
+use nexus_types::deployment::Policy;
+use nexus_types::inventory::Collection;
+use omicron_common::api::external::Generation;
+use sled_agent_client::types::OmicronZonesConfig;
+
+pub struct ExampleSystem {
+    pub system: SystemDescription,
+    pub policy: Policy,
+    pub collection: Collection,
+    pub blueprint: Blueprint,
+    // If we add more types of RNGs than just sleds here, we'll need to
+    // expand this to be similar to BlueprintBuilderRng where a root RNG
+    // creates sub-RNGs.
+    //
+    // This is currently only used for tests, so it looks unused in normal
+    // builds.  But in the future it could be used by other consumers, too.
+    #[allow(dead_code)]
+    pub(crate) sled_rng: UuidRng,
+}
+
+impl ExampleSystem {
+    pub fn new(
+        log: &slog::Logger,
+        test_name: &str,
+        nsleds: usize,
+    ) -> ExampleSystem {
+        let mut system = SystemDescription::new();
+        let mut sled_rng = UuidRng::from_seed(test_name, "ExampleSystem");
+        let sled_ids: Vec<_> =
+            (0..nsleds).map(|_| sled_rng.next_uuid()).collect();
+        for sled_id in &sled_ids {
+            let _ = system.sled(SledBuilder::new().id(*sled_id)).unwrap();
+        }
+
+        let policy = system.to_policy().expect("failed to make policy");
+        let mut inventory_builder =
+            system.to_collection_builder().expect("failed to build collection");
+
+        // For each sled, have it report 0 zones in the initial inventory.
+        // This will enable us to build a blueprint from the initial
+        // inventory, which we can then use to build new blueprints.
+        for sled_id in &sled_ids {
+            inventory_builder
+                .found_sled_omicron_zones(
+                    "fake sled agent",
+                    *sled_id,
+                    OmicronZonesConfig {
+                        generation: Generation::new(),
+                        zones: vec![],
+                    },
+                )
+                .expect("recording Omicron zones");
+        }
+
+        let empty_zone_inventory = inventory_builder.build();
+        let initial_blueprint =
+            BlueprintBuilder::build_initial_from_collection_seeded(
+                &empty_zone_inventory,
+                Generation::new(),
+                Generation::new(),
+                &policy,
+                "test suite",
+                (test_name, "ExampleSystem initial"),
+            )
+            .unwrap();
+
+        // Now make a blueprint and collection with some zones on each sled.
+        let mut builder = BlueprintBuilder::new_based_on(
+            &log,
+            &initial_blueprint,
+            Generation::new(),
+            Generation::new(),
+            &policy,
+            "test suite",
+        )
+        .unwrap();
+        builder.set_rng_seed((test_name, "ExampleSystem make_zones"));
+        for (sled_id, sled_resources) in &policy.sleds {
+            let _ = builder.sled_ensure_zone_ntp(*sled_id).unwrap();
+            let _ = builder
+                .sled_ensure_zone_multiple_nexus_with_config(
+                    *sled_id,
+                    1,
+                    false,
+                    vec![],
+                )
+                .unwrap();
+            for pool_name in &sled_resources.zpools {
+                let _ = builder
+                    .sled_ensure_zone_crucible(*sled_id, pool_name.clone())
+                    .unwrap();
+            }
+        }
+
+        let blueprint = builder.build();
+        let mut builder =
+            system.to_collection_builder().expect("failed to build collection");
+
+        for sled_id in blueprint.sleds() {
+            let Some(zones) = blueprint.omicron_zones.get(&sled_id) else {
+                continue;
+            };
+            builder
+                .found_sled_omicron_zones(
+                    "fake sled agent",
+                    sled_id,
+                    zones.clone(),
+                )
+                .unwrap();
+        }
+
+        ExampleSystem {
+            system,
+            policy,
+            collection: builder.build(),
+            blueprint,
+            sled_rng,
+        }
+    }
+}
+
+/// Returns a collection and policy describing a pretty simple system.
+///
+/// The test name is used as the RNG seed.
+///
+/// `n_sleds` is the number of sleds supported. Currently, this value can
+/// be anywhere between 0 and 5. (More can be added in the future if
+/// necessary.)
+pub fn example(
+    log: &slog::Logger,
+    test_name: &str,
+    nsleds: usize,
+) -> (Collection, Policy) {
+    let example = ExampleSystem::new(log, test_name, nsleds);
+    (example.collection, example.policy)
+}
diff --git a/nexus/reconfigurator/planning/src/lib.rs b/nexus/reconfigurator/planning/src/lib.rs
index e0a61826f0..3d6cc6a778 100644
--- a/nexus/reconfigurator/planning/src/lib.rs
+++ b/nexus/reconfigurator/planning/src/lib.rs
@@ -116,6 +116,7 @@
 //! updates, etc.
 
 pub mod blueprint_builder;
+pub mod example;
 mod ip_allocator;
 pub mod planner;
 pub mod system;
diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs
index 1882d62e22..1f79fda222 100644
--- a/nexus/reconfigurator/planning/src/planner.rs
+++ b/nexus/reconfigurator/planning/src/planner.rs
@@ -42,6 +42,7 @@ impl<'a> Planner<'a> {
         log: Logger,
         parent_blueprint: &'a Blueprint,
         internal_dns_version: Generation,
+        external_dns_version: Generation,
         policy: &'a Policy,
         creator: &str,
         // NOTE: Right now, we just assume that this is the latest inventory
@@ -52,6 +53,7 @@ impl<'a> Planner<'a> {
             &log,
             parent_blueprint,
             internal_dns_version,
+            external_dns_version,
             policy,
             creator,
         )?;
@@ -330,11 +332,11 @@ impl<'a> Planner<'a> {
 #[cfg(test)]
 mod test {
     use super::Planner;
-    use crate::blueprint_builder::test::example;
     use crate::blueprint_builder::test::verify_blueprint;
-    use crate::blueprint_builder::test::ExampleSystem;
     use crate::blueprint_builder::test::DEFAULT_N_SLEDS;
     use crate::blueprint_builder::BlueprintBuilder;
+    use crate::example::example;
+    use crate::example::ExampleSystem;
     use crate::system::SledBuilder;
     use expectorate::assert_contents;
     use nexus_inventory::now_db_precision;
@@ -352,8 +354,9 @@ mod test {
         static TEST_NAME: &str = "planner_basic_add_sled";
         let logctx = test_setup_log(TEST_NAME);
 
-        // For our purposes, we don't care about the internal DNS generation.
+        // For our purposes, we don't care about the DNS generations.
         let internal_dns_version = Generation::new();
+        let external_dns_version = Generation::new();
 
         // Use our example inventory collection.
         let mut example =
@@ -365,6 +368,7 @@ mod test {
             BlueprintBuilder::build_initial_from_collection_seeded(
                 &example.collection,
                 internal_dns_version,
+                external_dns_version,
                 &example.policy,
                 "the_test",
                 (TEST_NAME, "bp1"),
@@ -379,6 +383,7 @@ mod test {
             logctx.log.clone(),
             &blueprint1,
             internal_dns_version,
+            external_dns_version,
             &example.policy,
             "no-op?",
             &example.collection,
@@ -406,6 +411,7 @@ mod test {
             logctx.log.clone(),
             &blueprint2,
             internal_dns_version,
+            external_dns_version,
             &policy,
             "test: add NTP?",
             &example.collection,
@@ -447,6 +453,7 @@ mod test {
             logctx.log.clone(),
             &blueprint3,
             internal_dns_version,
+            external_dns_version,
             &policy,
             "test: add nothing more",
             &example.collection,
@@ -486,6 +493,7 @@ mod test {
             logctx.log.clone(),
             &blueprint3,
             internal_dns_version,
+            external_dns_version,
             &policy,
             "test: add Crucible zones?",
             &collection,
@@ -527,6 +535,7 @@ mod test {
             logctx.log.clone(),
             &blueprint5,
             internal_dns_version,
+            external_dns_version,
             &policy,
             "test: no-op?",
             &collection,
@@ -553,8 +562,9 @@ mod test {
         static TEST_NAME: &str = "planner_add_multiple_nexus_to_one_sled";
         let logctx = test_setup_log(TEST_NAME);
 
-        // For our purposes, we don't care about the internal DNS generation.
+        // For our purposes, we don't care about the DNS generations.
         let internal_dns_version = Generation::new();
+        let external_dns_version = Generation::new();
 
         // Use our example inventory collection as a starting point, but strip
         // it down to just one sled.
@@ -580,6 +590,7 @@ mod test {
             BlueprintBuilder::build_initial_from_collection_seeded(
                 &collection,
                 internal_dns_version,
+                external_dns_version,
                 &policy,
                 "the_test",
                 (TEST_NAME, "bp1"),
@@ -608,6 +619,7 @@ mod test {
             logctx.log.clone(),
             &blueprint1,
             internal_dns_version,
+            external_dns_version,
             &policy,
             "add more Nexus",
             &collection,
@@ -655,6 +667,7 @@ mod test {
             BlueprintBuilder::build_initial_from_collection_seeded(
                 &collection,
                 Generation::new(),
+                Generation::new(),
                 &policy,
                 "the_test",
                 (TEST_NAME, "bp1"),
@@ -680,6 +693,7 @@ mod test {
             logctx.log.clone(),
             &blueprint1,
             Generation::new(),
+            Generation::new(),
             &policy,
             "add more Nexus",
             &collection,
@@ -744,6 +758,7 @@ mod test {
             BlueprintBuilder::build_initial_from_collection_seeded(
                 &collection,
                 Generation::new(),
+                Generation::new(),
                 &policy,
                 "the_test",
                 (TEST_NAME, "bp1"),
@@ -800,6 +815,7 @@ mod test {
             logctx.log.clone(),
             &blueprint1,
             Generation::new(),
+            Generation::new(),
             &policy,
             "add more Nexus",
             &collection,
diff --git a/nexus/src/app/background/blueprint_execution.rs b/nexus/src/app/background/blueprint_execution.rs
index 7024f8be75..c65a49ec24 100644
--- a/nexus/src/app/background/blueprint_execution.rs
+++ b/nexus/src/app/background/blueprint_execution.rs
@@ -132,7 +132,7 @@ mod test {
 
     fn create_blueprint(
         omicron_zones: BTreeMap<Uuid, OmicronZonesConfig>,
-        internal_dns_version: Generation,
+        dns_version: Generation,
     ) -> (BlueprintTarget, Blueprint) {
         let id = Uuid::new_v4();
         (
@@ -146,7 +146,8 @@ mod test {
                 omicron_zones,
                 zones_in_service: BTreeSet::new(),
                 parent_blueprint_id: None,
-                internal_dns_version,
+                internal_dns_version: dns_version,
+                external_dns_version: dns_version,
                 time_created: chrono::Utc::now(),
                 creator: "test".to_string(),
                 comment: "test blueprint".to_string(),
diff --git a/nexus/src/app/background/blueprint_load.rs b/nexus/src/app/background/blueprint_load.rs
index 603bc6b7d2..b593354099 100644
--- a/nexus/src/app/background/blueprint_load.rs
+++ b/nexus/src/app/background/blueprint_load.rs
@@ -232,6 +232,7 @@ mod test {
                 zones_in_service: BTreeSet::new(),
                 parent_blueprint_id,
                 internal_dns_version: Generation::new(),
+                external_dns_version: Generation::new(),
                 time_created: now_db_precision(),
                 creator: "test".to_string(),
                 comment: "test blueprint".to_string(),
diff --git a/nexus/src/app/deployment.rs b/nexus/src/app/deployment.rs
index 4f66f2af78..48ed844f12 100644
--- a/nexus/src/app/deployment.rs
+++ b/nexus/src/app/deployment.rs
@@ -35,6 +35,7 @@ struct PlanningContext {
     creator: String,
     inventory: Option<Collection>,
     internal_dns_version: Generation,
+    external_dns_version: Generation,
 }
 
 impl super::Nexus {
@@ -168,21 +169,28 @@ impl super::Nexus {
                 "fetching latest inventory collection for blueprint planner",
             )?;
 
-        // Fetch the current internal DNS version.  This could be made part of
+        // Fetch the current DNS versions.  This could be made part of
         // inventory, but it's enough of a one-off that there's no particular
         // advantage to doing that work now.
-        let dns_version = datastore
+        let internal_dns_version = datastore
             .dns_group_latest_version(opctx, DnsGroup::Internal)
             .await
             .internal_context(
                 "fetching internal DNS version for blueprint planning",
             )?;
+        let external_dns_version = datastore
+            .dns_group_latest_version(opctx, DnsGroup::External)
+            .await
+            .internal_context(
+                "fetching external DNS version for blueprint planning",
+            )?;
 
         Ok(PlanningContext {
             creator,
             policy,
             inventory,
-            internal_dns_version: *dns_version.version,
+            internal_dns_version: *internal_dns_version.version,
+            external_dns_version: *external_dns_version.version,
         })
     }
 
@@ -207,6 +215,7 @@ impl super::Nexus {
         let blueprint = BlueprintBuilder::build_initial_from_collection(
             &collection,
             planning_context.internal_dns_version,
+            planning_context.external_dns_version,
             &planning_context.policy,
             &planning_context.creator,
         )
@@ -242,6 +251,7 @@ impl super::Nexus {
             opctx.log.clone(),
             &parent_blueprint,
             planning_context.internal_dns_version,
+            planning_context.external_dns_version,
             &planning_context.policy,
             &planning_context.creator,
             &inventory,
diff --git a/nexus/src/app/external_endpoints.rs b/nexus/src/app/external_endpoints.rs
index bcfec667ce..25a9dd4e6c 100644
--- a/nexus/src/app/external_endpoints.rs
+++ b/nexus/src/app/external_endpoints.rs
@@ -26,7 +26,6 @@
 //! "certificate resolver" object that impls
 //! [`rustls::server::ResolvesServerCert`].  See [`NexusCertResolver`].
 
-use super::silo::silo_dns_name;
 use crate::ServerContext;
 use anyhow::anyhow;
 use anyhow::bail;
@@ -39,6 +38,7 @@ use nexus_db_queries::db::datastore::Discoverability;
 use nexus_db_queries::db::fixed_data::silo::SILO_ID;
 use nexus_db_queries::db::model::ServiceKind;
 use nexus_db_queries::db::DataStore;
+use nexus_reconfigurator_execution::silo_dns_name;
 use nexus_types::identity::Resource;
 use omicron_common::api::external::http_pagination::PaginatedBy;
 use omicron_common::api::external::DataPageParams;
diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs
index 4030fce31d..4ba31bb0fe 100644
--- a/nexus/src/app/rack.rs
+++ b/nexus/src/app/rack.rs
@@ -4,7 +4,6 @@
 
 //! Rack management
 
-use super::silo::silo_dns_name;
 use crate::external_api::params;
 use crate::external_api::params::CertificateCreate;
 use crate::external_api::shared::ServiceUsingCertificate;
@@ -20,6 +19,7 @@ use nexus_db_queries::db;
 use nexus_db_queries::db::datastore::DnsVersionUpdateBuilder;
 use nexus_db_queries::db::datastore::RackInit;
 use nexus_db_queries::db::lookup::LookupPath;
+use nexus_reconfigurator_execution::silo_dns_name;
 use nexus_types::external_api::params::Address;
 use nexus_types::external_api::params::AddressConfig;
 use nexus_types::external_api::params::AddressLotBlockCreate;
diff --git a/nexus/src/app/silo.rs b/nexus/src/app/silo.rs
index 8461be015a..487af96aab 100644
--- a/nexus/src/app/silo.rs
+++ b/nexus/src/app/silo.rs
@@ -16,6 +16,7 @@ use nexus_db_queries::db::identity::{Asset, Resource};
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_db_queries::db::{self, lookup};
 use nexus_db_queries::{authn, authz};
+use nexus_reconfigurator_execution::silo_dns_name;
 use nexus_types::internal_api::params::DnsRecord;
 use omicron_common::api::external::http_pagination::PaginatedBy;
 use omicron_common::api::external::ListResultVec;
@@ -95,8 +96,13 @@ impl super::Nexus {
 
         // Set up an external DNS name for this Silo's API and console
         // endpoints (which are the same endpoint).
+        let target_blueprint = datastore
+            .blueprint_target_get_current_full(opctx)
+            .await
+            .internal_context("loading target blueprint")?;
+        let target = target_blueprint.as_ref().map(|(_, blueprint)| blueprint);
         let (nexus_external_ips, nexus_external_dns_zones) =
-            datastore.nexus_external_addresses(nexus_opctx).await?;
+            datastore.nexus_external_addresses(nexus_opctx, target).await?;
         let dns_records: Vec<DnsRecord> = nexus_external_ips
             .into_iter()
             .map(|addr| match addr {
@@ -886,16 +892,3 @@ impl super::Nexus {
         LookupPath::new(opctx, &self.db_datastore).silo_group_id(*group_id)
     }
 }
-
-/// Returns the (relative) DNS name for this Silo's API and console endpoints
-/// _within_ the external DNS zone (i.e., without that zone's suffix)
-///
-/// This specific naming scheme is determined under RFD 357.
-pub(crate) fn silo_dns_name(
-    name: &omicron_common::api::external::Name,
-) -> String {
-    // RFD 4 constrains resource names (including Silo names) to DNS-safe
-    // strings, which is why it's safe to directly put the name of the
-    // resource into the DNS name rather than doing any kind of escaping.
-    format!("{}.sys", name)
-}
diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs
index 7054a3ea15..a6b53272eb 100644
--- a/nexus/types/src/deployment.rs
+++ b/nexus/types/src/deployment.rs
@@ -151,9 +151,13 @@ pub struct Blueprint {
     pub parent_blueprint_id: Option<Uuid>,
 
     /// internal DNS version when this blueprint was created
-    // See blueprint generation for more on this.
+    // See blueprint execution for more on this.
     pub internal_dns_version: Generation,
 
+    /// external DNS version when thi blueprint was created
+    // See blueprint execution for more on this.
+    pub external_dns_version: Generation,
+
     /// when this blueprint was generated (for debugging)
     pub time_created: chrono::DateTime<chrono::Utc>,
     /// identity of the component that generated the blueprint (for debugging)
@@ -305,6 +309,8 @@ pub struct BlueprintMetadata {
     pub parent_blueprint_id: Option<Uuid>,
     /// internal DNS version when this blueprint was created
     pub internal_dns_version: Generation,
+    /// external DNS version when this blueprint was created
+    pub external_dns_version: Generation,
 
     /// when this blueprint was generated (for debugging)
     pub time_created: chrono::DateTime<chrono::Utc>,
diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json
index c67fe27652..09b5f1e5ab 100644
--- a/openapi/nexus-internal.json
+++ b/openapi/nexus-internal.json
@@ -2341,6 +2341,14 @@
             "description": "identity of the component that generated the blueprint (for debugging) This would generally be the Uuid of a Nexus instance.",
             "type": "string"
           },
+          "external_dns_version": {
+            "description": "external DNS version when thi blueprint was created",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Generation"
+              }
+            ]
+          },
           "id": {
             "description": "unique identifier for this blueprint",
             "type": "string",
@@ -2385,6 +2393,7 @@
         "required": [
           "comment",
           "creator",
+          "external_dns_version",
           "id",
           "internal_dns_version",
           "omicron_zones",
@@ -2404,6 +2413,14 @@
             "description": "identity of the component that generated the blueprint (for debugging) This would generally be the Uuid of a Nexus instance.",
             "type": "string"
           },
+          "external_dns_version": {
+            "description": "external DNS version when this blueprint was created",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Generation"
+              }
+            ]
+          },
           "id": {
             "description": "unique identifier for this blueprint",
             "type": "string",
@@ -2432,6 +2449,7 @@
         "required": [
           "comment",
           "creator",
+          "external_dns_version",
           "id",
           "internal_dns_version",
           "time_created"
diff --git a/schema/crdb/42.0.0/up1.sql b/schema/crdb/42.0.0/up1.sql
new file mode 100644
index 0000000000..e8ae49c7c8
--- /dev/null
+++ b/schema/crdb/42.0.0/up1.sql
@@ -0,0 +1,8 @@
+-- Add the "external_dns_version" column to the "blueprint" table.
+-- This query will end up setting the external DNS version for any existing
+-- blueprints to 1.  This is always safe because it's the smallest possible
+-- value and if a value is too small, the end result is simply needing to
+-- regenerate the blueprint in order to be able to execute it.  (On the other
+-- hand, using a value that's too large could cause corruption.)
+ALTER TABLE omicron.public.blueprint
+    ADD COLUMN IF NOT EXISTS external_dns_version INT8 NOT NULL DEFAULT 1;
diff --git a/schema/crdb/42.0.0/up2.sql b/schema/crdb/42.0.0/up2.sql
new file mode 100644
index 0000000000..cc89117e1d
--- /dev/null
+++ b/schema/crdb/42.0.0/up2.sql
@@ -0,0 +1,2 @@
+ALTER TABLE omicron.public.blueprint
+    ALTER COLUMN external_dns_version DROP DEFAULT;
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 7c3b8fcc34..9895464fb5 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -3159,7 +3159,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.blueprint (
     comment TEXT NOT NULL,
 
     -- identifies the latest internal DNS version when blueprint planning began
-    internal_dns_version INT8 NOT NULL
+    internal_dns_version INT8 NOT NULL,
+    -- identifies the latest external DNS version when blueprint planning began
+    external_dns_version INT8 NOT NULL
 );
 
 -- table describing both the current and historical target blueprints of the
@@ -3584,7 +3586,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '41.0.0', NULL)
+    ( TRUE, NOW(), NOW(), '42.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;

From bd46b6b3daea2f18fb8e18d275001b2a238a26ca Mon Sep 17 00:00:00 2001
From: "oxide-renovate[bot]"
 <146848827+oxide-renovate[bot]@users.noreply.github.com>
Date: Thu, 14 Mar 2024 06:42:18 +0000
Subject: [PATCH 115/157] chore(deps): update taiki-e/install-action digest to
 3d5321a (#5261)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [taiki-e/install-action](https://togithub.com/taiki-e/install-action)
| action | digest | [`34c5068` ->
`3d5321a`](https://togithub.com/taiki-e/install-action/compare/34c5068...3d5321a)
|

---

### Configuration

📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone
America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone
America/Los_Angeles.

🚦 **Automerge**: Enabled.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the
rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update
again.

---

- [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check
this box

---

This PR has been generated by [Renovate
Bot](https://togithub.com/renovatebot/renovate).

<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzNy4yNDQuMCIsInVwZGF0ZWRJblZlciI6IjM3LjI0NC4wIiwidGFyZ2V0QnJhbmNoIjoibWFpbiJ9-->

Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com>
---
 .github/workflows/hakari.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml
index ae4622290d..5194f2d28a 100644
--- a/.github/workflows/hakari.yml
+++ b/.github/workflows/hakari.yml
@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@34c5068afd12dd215738693ffa556632845e203d # v2
+        uses: taiki-e/install-action@3d5321a5e3ceb69232a18ca12966908a643cbce3 # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

From e6029bbd98122bed366776e1c6d14ac0321aee8f Mon Sep 17 00:00:00 2001
From: David Pacheco <dap@oxidecomputer.com>
Date: Thu, 14 Mar 2024 08:26:07 -0700
Subject: [PATCH 116/157] rename SILO_ID -> DEFAULT_SILO_ID (#5264)

---
 nexus/db-queries/src/db/datastore/mod.rs      | 16 +++++++-------
 nexus/db-queries/src/db/fixed_data/silo.rs    |  4 ++--
 .../db-queries/src/db/fixed_data/silo_user.rs |  6 +++---
 nexus/reconfigurator/execution/src/dns.rs     |  4 ++--
 nexus/src/app/external_endpoints.rs           |  4 ++--
 nexus/tests/integration_tests/authn_http.rs   |  4 ++--
 nexus/tests/integration_tests/disks.rs        | 10 ++++-----
 nexus/tests/integration_tests/instances.rs    |  4 ++--
 nexus/tests/integration_tests/metrics.rs      | 21 ++++++++++++-------
 nexus/tests/integration_tests/silo_users.rs   |  9 +++++---
 nexus/tests/integration_tests/silos.rs        | 12 +++++------
 11 files changed, 52 insertions(+), 42 deletions(-)

diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs
index 475fb27df1..b2b564cc09 100644
--- a/nexus/db-queries/src/db/datastore/mod.rs
+++ b/nexus/db-queries/src/db/datastore/mod.rs
@@ -380,7 +380,7 @@ mod test {
     };
     use crate::db::explain::ExplainableAsync;
     use crate::db::fixed_data::silo::DEFAULT_SILO;
-    use crate::db::fixed_data::silo::SILO_ID;
+    use crate::db::fixed_data::silo::DEFAULT_SILO_ID;
     use crate::db::identity::Asset;
     use crate::db::lookup::LookupPath;
     use crate::db::model::{
@@ -499,8 +499,8 @@ mod test {
         // Associate silo with user
         let authz_silo = authz::Silo::new(
             authz::FLEET,
-            *SILO_ID,
-            LookupType::ById(*SILO_ID),
+            *DEFAULT_SILO_ID,
+            LookupType::ById(*DEFAULT_SILO_ID),
         );
         datastore
             .silo_user_create(
@@ -519,7 +519,7 @@ mod test {
             .fetch()
             .await
             .unwrap();
-        assert_eq!(*SILO_ID, db_silo_user.silo_id);
+        assert_eq!(*DEFAULT_SILO_ID, db_silo_user.silo_id);
 
         // fetch the one we just created
         let (.., fetched) = LookupPath::new(&opctx, &datastore)
@@ -577,7 +577,7 @@ mod test {
             Arc::new(authz::Authz::new(&logctx.log)),
             authn::Context::for_test_user(
                 silo_user_id,
-                *SILO_ID,
+                *DEFAULT_SILO_ID,
                 SiloAuthnPolicy::try_from(&*DEFAULT_SILO).unwrap(),
             ),
             Arc::clone(&datastore),
@@ -1381,8 +1381,8 @@ mod test {
         // Create a new Silo user so that we can lookup their keys.
         let authz_silo = authz::Silo::new(
             authz::FLEET,
-            *SILO_ID,
-            LookupType::ById(*SILO_ID),
+            *DEFAULT_SILO_ID,
+            LookupType::ById(*DEFAULT_SILO_ID),
         );
         let silo_user_id = Uuid::new_v4();
         datastore
@@ -1432,7 +1432,7 @@ mod test {
                 .fetch()
                 .await
                 .unwrap();
-        assert_eq!(authz_silo.id(), *SILO_ID);
+        assert_eq!(authz_silo.id(), *DEFAULT_SILO_ID);
         assert_eq!(authz_silo_user.id(), silo_user_id);
         assert_eq!(found.silo_user_id, ssh_key.silo_user_id);
         assert_eq!(found.public_key, ssh_key.public_key);
diff --git a/nexus/db-queries/src/db/fixed_data/silo.rs b/nexus/db-queries/src/db/fixed_data/silo.rs
index 62bcc61c1e..dc5f19fc2f 100644
--- a/nexus/db-queries/src/db/fixed_data/silo.rs
+++ b/nexus/db-queries/src/db/fixed_data/silo.rs
@@ -7,7 +7,7 @@ use nexus_types::external_api::{params, shared};
 use omicron_common::api::external::IdentityMetadataCreateParams;
 use once_cell::sync::Lazy;
 
-pub static SILO_ID: Lazy<uuid::Uuid> = Lazy::new(|| {
+pub static DEFAULT_SILO_ID: Lazy<uuid::Uuid> = Lazy::new(|| {
     "001de000-5110-4000-8000-000000000000"
         .parse()
         .expect("invalid uuid for builtin silo id")
@@ -19,7 +19,7 @@ pub static SILO_ID: Lazy<uuid::Uuid> = Lazy::new(|| {
 /// remove it per omicron#2305.
 pub static DEFAULT_SILO: Lazy<db::model::Silo> = Lazy::new(|| {
     db::model::Silo::new_with_id(
-        *SILO_ID,
+        *DEFAULT_SILO_ID,
         params::SiloCreate {
             identity: IdentityMetadataCreateParams {
                 name: "default-silo".parse().unwrap(),
diff --git a/nexus/db-queries/src/db/fixed_data/silo_user.rs b/nexus/db-queries/src/db/fixed_data/silo_user.rs
index b5253b68e3..eb49093152 100644
--- a/nexus/db-queries/src/db/fixed_data/silo_user.rs
+++ b/nexus/db-queries/src/db/fixed_data/silo_user.rs
@@ -15,7 +15,7 @@ use once_cell::sync::Lazy;
 // not automatically at Nexus startup.  See omicron#2305.
 pub static USER_TEST_PRIVILEGED: Lazy<db::model::SiloUser> = Lazy::new(|| {
     db::model::SiloUser::new(
-        *db::fixed_data::silo::SILO_ID,
+        *db::fixed_data::silo::DEFAULT_SILO_ID,
         // "4007" looks a bit like "root".
         "001de000-05e4-4000-8000-000000004007".parse().unwrap(),
         "privileged".into(),
@@ -39,7 +39,7 @@ pub static ROLE_ASSIGNMENTS_PRIVILEGED: Lazy<Vec<db::model::RoleAssignment>> =
                 db::model::IdentityType::SiloUser,
                 USER_TEST_PRIVILEGED.id(),
                 role_builtin::SILO_ADMIN.resource_type,
-                *db::fixed_data::silo::SILO_ID,
+                *db::fixed_data::silo::DEFAULT_SILO_ID,
                 role_builtin::SILO_ADMIN.role_name,
             ),
         ]
@@ -52,7 +52,7 @@ pub static ROLE_ASSIGNMENTS_PRIVILEGED: Lazy<Vec<db::model::RoleAssignment>> =
 pub static USER_TEST_UNPRIVILEGED: Lazy<db::model::SiloUser> =
     Lazy::new(|| {
         db::model::SiloUser::new(
-            *db::fixed_data::silo::SILO_ID,
+            *db::fixed_data::silo::DEFAULT_SILO_ID,
             // 60001 is the decimal uid for "nobody" on Helios.
             "001de000-05e4-4000-8000-000000060001".parse().unwrap(),
             "unprivileged".into(),
diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs
index 9e0434c59f..6133da6428 100644
--- a/nexus/reconfigurator/execution/src/dns.rs
+++ b/nexus/reconfigurator/execution/src/dns.rs
@@ -15,7 +15,7 @@ use nexus_db_model::Silo;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db::datastore::Discoverability;
 use nexus_db_queries::db::datastore::DnsVersionUpdateBuilder;
-use nexus_db_queries::db::fixed_data::silo::SILO_ID;
+use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID;
 use nexus_db_queries::db::DataStore;
 use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::OmicronZoneType;
@@ -72,7 +72,7 @@ pub(crate) async fn deploy_dns(
         .internal_context("listing Silos (for configuring external DNS)")?
         .into_iter()
         // We do not generate a DNS name for the "default" Silo.
-        .filter(|silo| silo.id() != *SILO_ID)
+        .filter(|silo| silo.id() != *DEFAULT_SILO_ID)
         .collect::<Vec<_>>();
 
     let (nexus_external_ips, nexus_external_dns_zones) =
diff --git a/nexus/src/app/external_endpoints.rs b/nexus/src/app/external_endpoints.rs
index 25a9dd4e6c..db87632bbf 100644
--- a/nexus/src/app/external_endpoints.rs
+++ b/nexus/src/app/external_endpoints.rs
@@ -35,7 +35,7 @@ use nexus_db_model::Certificate;
 use nexus_db_model::DnsGroup;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db::datastore::Discoverability;
-use nexus_db_queries::db::fixed_data::silo::SILO_ID;
+use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID;
 use nexus_db_queries::db::model::ServiceKind;
 use nexus_db_queries::db::DataStore;
 use nexus_reconfigurator_execution::silo_dns_name;
@@ -225,7 +225,7 @@ impl ExternalEndpoints {
             .filter(|s| {
                 // Ignore the built-in Silo, which people are not supposed to
                 // log into.
-                s.id() != *SILO_ID
+                s.id() != *DEFAULT_SILO_ID
             })
             .find(|s| s.authentication_mode == AuthenticationMode::Local)
             .and_then(|s| {
diff --git a/nexus/tests/integration_tests/authn_http.rs b/nexus/tests/integration_tests/authn_http.rs
index 1b1286c7a6..51bfa08ce2 100644
--- a/nexus/tests/integration_tests/authn_http.rs
+++ b/nexus/tests/integration_tests/authn_http.rs
@@ -26,7 +26,7 @@ use nexus_db_queries::authn::external::spoof::SPOOF_SCHEME_NAME;
 use nexus_db_queries::authn::external::AuthenticatorContext;
 use nexus_db_queries::authn::external::HttpAuthnScheme;
 use nexus_db_queries::authn::external::SiloUserSilo;
-use nexus_db_queries::db::fixed_data::silo::SILO_ID;
+use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID;
 use std::collections::HashMap;
 use std::sync::Mutex;
 use uuid::Uuid;
@@ -335,7 +335,7 @@ impl SiloUserSilo for WhoamiServerState {
             silo_user_id.to_string(),
             "7f927c86-3371-4295-c34a-e3246a4b9c02"
         );
-        Ok(*SILO_ID)
+        Ok(*DEFAULT_SILO_ID)
     }
 }
 
diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs
index 379042c849..5464d7e589 100644
--- a/nexus/tests/integration_tests/disks.rs
+++ b/nexus/tests/integration_tests/disks.rs
@@ -11,7 +11,7 @@ use dropshot::HttpErrorResponseBody;
 use http::method::Method;
 use http::StatusCode;
 use nexus_db_queries::context::OpContext;
-use nexus_db_queries::db::fixed_data::{silo::SILO_ID, FLEET_ID};
+use nexus_db_queries::db::fixed_data::{silo::DEFAULT_SILO_ID, FLEET_ID};
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_test_utils::http_testing::AuthnMode;
 use nexus_test_utils::http_testing::Collection;
@@ -1073,7 +1073,7 @@ async fn test_disk_virtual_provisioning_collection(
         0
     );
     let virtual_provisioning_collection = datastore
-        .virtual_provisioning_collection_get(&opctx, *SILO_ID)
+        .virtual_provisioning_collection_get(&opctx, *DEFAULT_SILO_ID)
         .await
         .unwrap();
     assert_eq!(
@@ -1137,7 +1137,7 @@ async fn test_disk_virtual_provisioning_collection(
         0
     );
     let virtual_provisioning_collection = datastore
-        .virtual_provisioning_collection_get(&opctx, *SILO_ID)
+        .virtual_provisioning_collection_get(&opctx, *DEFAULT_SILO_ID)
         .await
         .unwrap();
     assert_eq!(
@@ -1194,7 +1194,7 @@ async fn test_disk_virtual_provisioning_collection(
         disk_size
     );
     let virtual_provisioning_collection = datastore
-        .virtual_provisioning_collection_get(&opctx, *SILO_ID)
+        .virtual_provisioning_collection_get(&opctx, *DEFAULT_SILO_ID)
         .await
         .unwrap();
     assert_eq!(
@@ -1232,7 +1232,7 @@ async fn test_disk_virtual_provisioning_collection(
         0
     );
     let virtual_provisioning_collection = datastore
-        .virtual_provisioning_collection_get(&opctx, *SILO_ID)
+        .virtual_provisioning_collection_get(&opctx, *DEFAULT_SILO_ID)
         .await
         .unwrap();
     assert_eq!(
diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs
index 0df2b83008..309036256f 100644
--- a/nexus/tests/integration_tests/instances.rs
+++ b/nexus/tests/integration_tests/instances.rs
@@ -14,7 +14,7 @@ use http::StatusCode;
 use itertools::Itertools;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO;
-use nexus_db_queries::db::fixed_data::silo::SILO_ID;
+use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID;
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_test_interface::NexusServer;
 use nexus_test_utils::http_testing::AuthnMode;
@@ -1031,7 +1031,7 @@ async fn assert_metrics(
             ram
         );
     }
-    for id in &[None, Some(*SILO_ID)] {
+    for id in &[None, Some(*DEFAULT_SILO_ID)] {
         assert_eq!(
             get_latest_system_metric(
                 cptestctx,
diff --git a/nexus/tests/integration_tests/metrics.rs b/nexus/tests/integration_tests/metrics.rs
index 5f517d49e0..73f11ce49a 100644
--- a/nexus/tests/integration_tests/metrics.rs
+++ b/nexus/tests/integration_tests/metrics.rs
@@ -6,7 +6,7 @@ use chrono::Utc;
 use dropshot::test_util::ClientTestContext;
 use dropshot::ResultsPage;
 use http::{Method, StatusCode};
-use nexus_db_queries::db::fixed_data::silo::SILO_ID;
+use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID;
 use nexus_test_utils::http_testing::{AuthnMode, NexusRequest, RequestBuilder};
 use nexus_test_utils::resource_helpers::{
     create_default_ip_pool, create_disk, create_instance, create_project,
@@ -173,7 +173,7 @@ async fn test_metrics(
 
     // silo metrics start out zero
     assert_system_metrics(&cptestctx, None, 0, 0, 0).await;
-    assert_system_metrics(&cptestctx, Some(*SILO_ID), 0, 0, 0).await;
+    assert_system_metrics(&cptestctx, Some(*DEFAULT_SILO_ID), 0, 0, 0).await;
     assert_silo_metrics(&cptestctx, None, 0, 0, 0).await;
 
     let project1_id = create_project(&client, "p-1").await.identity.id;
@@ -189,7 +189,7 @@ async fn test_metrics(
         "/v1/metrics/cpus_provisioned?start_time={:?}&end_time={:?}&order=descending&limit=1&project={}", 
         cptestctx.start_time,
         Utc::now(),
-        *SILO_ID,
+        *DEFAULT_SILO_ID,
     );
     assert_404(&cptestctx, &bad_silo_metrics_url).await;
     let bad_system_metrics_url = format!(
@@ -205,14 +205,15 @@ async fn test_metrics(
     assert_silo_metrics(&cptestctx, Some(project1_id), 0, 4, GIB).await;
     assert_silo_metrics(&cptestctx, None, 0, 4, GIB).await;
     assert_system_metrics(&cptestctx, None, 0, 4, GIB).await;
-    assert_system_metrics(&cptestctx, Some(*SILO_ID), 0, 4, GIB).await;
+    assert_system_metrics(&cptestctx, Some(*DEFAULT_SILO_ID), 0, 4, GIB).await;
 
     // create disk in project 1
     create_disk(&client, "p-1", "d-1").await;
     assert_silo_metrics(&cptestctx, Some(project1_id), GIB, 4, GIB).await;
     assert_silo_metrics(&cptestctx, None, GIB, 4, GIB).await;
     assert_system_metrics(&cptestctx, None, GIB, 4, GIB).await;
-    assert_system_metrics(&cptestctx, Some(*SILO_ID), GIB, 4, GIB).await;
+    assert_system_metrics(&cptestctx, Some(*DEFAULT_SILO_ID), GIB, 4, GIB)
+        .await;
 
     // project 2 metrics still empty
     assert_silo_metrics(&cptestctx, Some(project2_id), 0, 0, 0).await;
@@ -225,8 +226,14 @@ async fn test_metrics(
     // both instances show up in silo and fleet metrics
     assert_silo_metrics(&cptestctx, None, 2 * GIB, 8, 2 * GIB).await;
     assert_system_metrics(&cptestctx, None, 2 * GIB, 8, 2 * GIB).await;
-    assert_system_metrics(&cptestctx, Some(*SILO_ID), 2 * GIB, 8, 2 * GIB)
-        .await;
+    assert_system_metrics(
+        &cptestctx,
+        Some(*DEFAULT_SILO_ID),
+        2 * GIB,
+        8,
+        2 * GIB,
+    )
+    .await;
 
     // project 1 unaffected by project 2's resources
     assert_silo_metrics(&cptestctx, Some(project1_id), GIB, 4, GIB).await;
diff --git a/nexus/tests/integration_tests/silo_users.rs b/nexus/tests/integration_tests/silo_users.rs
index 9357c28063..099a186a2c 100644
--- a/nexus/tests/integration_tests/silo_users.rs
+++ b/nexus/tests/integration_tests/silo_users.rs
@@ -7,7 +7,7 @@ use http::{method::Method, StatusCode};
 use nexus_db_queries::authn::USER_TEST_UNPRIVILEGED;
 use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
-use nexus_db_queries::db::fixed_data::silo::SILO_ID;
+use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID;
 use nexus_test_utils::assert_same_items;
 use nexus_test_utils::http_testing::{AuthnMode, NexusRequest};
 use nexus_test_utils::resource_helpers::objects_list_page_authz;
@@ -44,8 +44,11 @@ async fn test_silo_group_users(cptestctx: &ControlPlaneTestContext) {
         objects_list_page_authz::<views::User>(client, &"/v1/groups").await;
     assert_eq!(groups.items.len(), 0);
 
-    let authz_silo =
-        authz::Silo::new(authz::FLEET, *SILO_ID, LookupType::ById(*SILO_ID));
+    let authz_silo = authz::Silo::new(
+        authz::FLEET,
+        *DEFAULT_SILO_ID,
+        LookupType::ById(*DEFAULT_SILO_ID),
+    );
 
     // create a group
     let group_name = "group1".to_string();
diff --git a/nexus/tests/integration_tests/silos.rs b/nexus/tests/integration_tests/silos.rs
index 86bf01062f..6dfddb12e1 100644
--- a/nexus/tests/integration_tests/silos.rs
+++ b/nexus/tests/integration_tests/silos.rs
@@ -11,7 +11,7 @@ use nexus_db_queries::authn::{USER_TEST_PRIVILEGED, USER_TEST_UNPRIVILEGED};
 use nexus_db_queries::authz::{self};
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db;
-use nexus_db_queries::db::fixed_data::silo::{DEFAULT_SILO, SILO_ID};
+use nexus_db_queries::db::fixed_data::silo::{DEFAULT_SILO, DEFAULT_SILO_ID};
 use nexus_db_queries::db::identity::Asset;
 use nexus_db_queries::db::lookup::LookupPath;
 use nexus_test_utils::http_testing::{AuthnMode, NexusRequest, RequestBuilder};
@@ -918,12 +918,12 @@ async fn test_silo_users_list(cptestctx: &ControlPlaneTestContext) {
             views::User {
                 id: USER_TEST_PRIVILEGED.id(),
                 display_name: USER_TEST_PRIVILEGED.external_id.clone(),
-                silo_id: *SILO_ID,
+                silo_id: *DEFAULT_SILO_ID,
             },
             views::User {
                 id: USER_TEST_UNPRIVILEGED.id(),
                 display_name: USER_TEST_UNPRIVILEGED.external_id.clone(),
-                silo_id: *SILO_ID,
+                silo_id: *DEFAULT_SILO_ID,
             },
         ]
     );
@@ -952,17 +952,17 @@ async fn test_silo_users_list(cptestctx: &ControlPlaneTestContext) {
             views::User {
                 id: new_silo_user_id,
                 display_name: new_silo_user_external_id.into(),
-                silo_id: *SILO_ID,
+                silo_id: *DEFAULT_SILO_ID,
             },
             views::User {
                 id: USER_TEST_PRIVILEGED.id(),
                 display_name: USER_TEST_PRIVILEGED.external_id.clone(),
-                silo_id: *SILO_ID,
+                silo_id: *DEFAULT_SILO_ID,
             },
             views::User {
                 id: USER_TEST_UNPRIVILEGED.id(),
                 display_name: USER_TEST_UNPRIVILEGED.external_id.clone(),
-                silo_id: *SILO_ID,
+                silo_id: *DEFAULT_SILO_ID,
             },
         ]
     );

From 2406d9d93258cc33b57d03c7abcd90a95ca487c4 Mon Sep 17 00:00:00 2001
From: James MacMahon <james@oxide.computer>
Date: Thu, 14 Mar 2024 12:44:49 -0400
Subject: [PATCH 117/157] Accept notifications from Crucible (#5135)

Allow any Upstairs to notify Nexus about the start or completion (plus
status) of live repairs. The motivation for this was to be used in the
final stage of region replacement to notify Nexus that the replacement
has finished, but more generally this can be used to keep track of how
many times repair occurs for each region.

Also accept notifications for:

- when a downstairs client is requested to stop
- when a downstairs client stops

These will be used as breadcrumbs to determine when downstairs were
having problems, why repairs started in the first place, and more.

Fixes #5120
---
 Cargo.lock                                    |   3 +
 clients/nexus-client/Cargo.toml               |   1 +
 clients/nexus-client/src/lib.rs               |   4 +
 common/Cargo.toml                             |   1 +
 common/src/api/internal/nexus.rs              |  83 ++
 common/src/lib.rs                             |  81 ++
 nexus/Cargo.toml                              |   1 +
 nexus/db-model/src/downstairs.rs              | 133 +++
 nexus/db-model/src/lib.rs                     |   4 +
 nexus/db-model/src/schema.rs                  |  46 +-
 nexus/db-model/src/upstairs_repair.rs         | 154 ++++
 nexus/db-queries/src/db/datastore/volume.rs   | 249 ++++++
 nexus/db-queries/src/db/pool_connection.rs    |   4 +
 nexus/src/app/instance_network.rs             |   2 +-
 nexus/src/app/sagas/common_storage.rs         |   2 +-
 .../src/app/sagas/loopback_address_create.rs  |   2 +-
 .../src/app/sagas/loopback_address_delete.rs  |   2 +-
 nexus/src/app/sagas/mod.rs                    |  82 --
 nexus/src/app/sagas/snapshot_create.rs        |   2 +-
 .../app/sagas/switch_port_settings_apply.rs   |   2 +-
 .../app/sagas/switch_port_settings_clear.rs   |   2 +-
 nexus/src/app/volume.rs                       | 162 ++++
 nexus/src/internal_api/http_entrypoints.rs    | 180 ++++
 .../integration_tests/volume_management.rs    | 768 ++++++++++++++++++
 openapi/nexus-internal.json                   | 401 +++++++++
 schema/crdb/43.0.0/up01.sql                   |   5 +
 schema/crdb/43.0.0/up02.sql                   |   4 +
 schema/crdb/43.0.0/up03.sql                   |  21 +
 schema/crdb/43.0.0/up04.sql                   |   8 +
 schema/crdb/43.0.0/up05.sql                   |  11 +
 schema/crdb/43.0.0/up06.sql                   |   8 +
 schema/crdb/43.0.0/up07.sql                   |  11 +
 schema/crdb/43.0.0/up08.sql                   |   8 +
 schema/crdb/dbinit.sql                        |  86 +-
 uuid-kinds/src/lib.rs                         |   5 +
 35 files changed, 2447 insertions(+), 91 deletions(-)
 create mode 100644 nexus/db-model/src/downstairs.rs
 create mode 100644 nexus/db-model/src/upstairs_repair.rs
 create mode 100644 schema/crdb/43.0.0/up01.sql
 create mode 100644 schema/crdb/43.0.0/up02.sql
 create mode 100644 schema/crdb/43.0.0/up03.sql
 create mode 100644 schema/crdb/43.0.0/up04.sql
 create mode 100644 schema/crdb/43.0.0/up05.sql
 create mode 100644 schema/crdb/43.0.0/up06.sql
 create mode 100644 schema/crdb/43.0.0/up07.sql
 create mode 100644 schema/crdb/43.0.0/up08.sql

diff --git a/Cargo.lock b/Cargo.lock
index 88c78a83d4..84f365e2f0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4369,6 +4369,7 @@ dependencies = [
  "nexus-types",
  "omicron-common",
  "omicron-passwords",
+ "omicron-uuid-kinds",
  "omicron-workspace-hack",
  "progenitor",
  "regress 0.9.0",
@@ -5057,6 +5058,7 @@ dependencies = [
  "once_cell",
  "parse-display",
  "progenitor",
+ "progenitor-client",
  "proptest",
  "rand 0.8.5",
  "regress 0.9.0",
@@ -5216,6 +5218,7 @@ dependencies = [
  "omicron-rpaths",
  "omicron-sled-agent",
  "omicron-test-utils",
+ "omicron-uuid-kinds",
  "omicron-workspace-hack",
  "once_cell",
  "openapi-lint",
diff --git a/clients/nexus-client/Cargo.toml b/clients/nexus-client/Cargo.toml
index 965e2a7dfb..fd6df6919f 100644
--- a/clients/nexus-client/Cargo.toml
+++ b/clients/nexus-client/Cargo.toml
@@ -20,3 +20,4 @@ serde_json.workspace = true
 slog.workspace = true
 uuid.workspace = true
 omicron-workspace-hack.workspace = true
+omicron-uuid-kinds.workspace = true
diff --git a/clients/nexus-client/src/lib.rs b/clients/nexus-client/src/lib.rs
index 85c67ddbfd..ad8269e675 100644
--- a/clients/nexus-client/src/lib.rs
+++ b/clients/nexus-client/src/lib.rs
@@ -35,6 +35,10 @@ progenitor::generate_api!(
         NewPasswordHash = omicron_passwords::NewPasswordHash,
         NetworkInterface = omicron_common::api::internal::shared::NetworkInterface,
         NetworkInterfaceKind = omicron_common::api::internal::shared::NetworkInterfaceKind,
+        TypedUuidForDownstairsKind = omicron_uuid_kinds::TypedUuid<omicron_uuid_kinds::DownstairsKind>,
+        TypedUuidForUpstairsKind = omicron_uuid_kinds::TypedUuid<omicron_uuid_kinds::UpstairsKind>,
+        TypedUuidForUpstairsRepairKind = omicron_uuid_kinds::TypedUuid<omicron_uuid_kinds::UpstairsRepairKind>,
+        TypedUuidForUpstairsSessionKind = omicron_uuid_kinds::TypedUuid<omicron_uuid_kinds::UpstairsSessionKind>,
     },
     patch = {
         SledAgentInfo = { derives = [PartialEq, Eq] },
diff --git a/common/Cargo.toml b/common/Cargo.toml
index 4451d92bdb..b16415b828 100644
--- a/common/Cargo.toml
+++ b/common/Cargo.toml
@@ -41,6 +41,7 @@ tokio = { workspace = true, features = ["full"] }
 uuid.workspace = true
 parse-display.workspace = true
 progenitor.workspace = true
+progenitor-client.workspace = true
 omicron-workspace-hack.workspace = true
 once_cell.workspace = true
 regress.workspace = true
diff --git a/common/src/api/internal/nexus.rs b/common/src/api/internal/nexus.rs
index 3972e011cf..24ef9a16aa 100644
--- a/common/src/api/internal/nexus.rs
+++ b/common/src/api/internal/nexus.rs
@@ -9,6 +9,10 @@ use crate::api::external::{
     InstanceState, IpNet, SemverVersion, Vni,
 };
 use chrono::{DateTime, Utc};
+use omicron_uuid_kinds::DownstairsRegionKind;
+use omicron_uuid_kinds::TypedUuid;
+use omicron_uuid_kinds::UpstairsRepairKind;
+use omicron_uuid_kinds::UpstairsSessionKind;
 use parse_display::{Display, FromStr};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
@@ -251,3 +255,82 @@ pub enum HostIdentifier {
     Ip(IpNet),
     Vpc(Vni),
 }
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone, Copy)]
+#[serde(rename_all = "snake_case")]
+pub enum UpstairsRepairType {
+    Live,
+    Reconciliation,
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+pub struct DownstairsUnderRepair {
+    pub region_uuid: TypedUuid<DownstairsRegionKind>,
+    pub target_addr: std::net::SocketAddrV6,
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+pub struct RepairStartInfo {
+    pub time: DateTime<Utc>,
+    pub session_id: TypedUuid<UpstairsSessionKind>,
+    pub repair_id: TypedUuid<UpstairsRepairKind>,
+    pub repair_type: UpstairsRepairType,
+    pub repairs: Vec<DownstairsUnderRepair>,
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+pub struct RepairFinishInfo {
+    pub time: DateTime<Utc>,
+    pub session_id: TypedUuid<UpstairsSessionKind>,
+    pub repair_id: TypedUuid<UpstairsRepairKind>,
+    pub repair_type: UpstairsRepairType,
+    pub repairs: Vec<DownstairsUnderRepair>,
+    pub aborted: bool,
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+pub struct RepairProgress {
+    pub time: DateTime<Utc>,
+    pub current_item: i64,
+    pub total_items: i64,
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+#[serde(rename_all = "snake_case")]
+pub enum DownstairsClientStopRequestReason {
+    Replacing,
+    Disabled,
+    FailedReconcile,
+    IOError,
+    BadNegotiationOrder,
+    Incompatible,
+    FailedLiveRepair,
+    TooManyOutstandingJobs,
+    Deactivated,
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+pub struct DownstairsClientStopRequest {
+    pub time: DateTime<Utc>,
+    pub reason: DownstairsClientStopRequestReason,
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+#[serde(rename_all = "snake_case")]
+pub enum DownstairsClientStoppedReason {
+    ConnectionTimeout,
+    ConnectionFailed,
+    Timeout,
+    WriteFailed,
+    ReadFailed,
+    RequestedStop,
+    Finished,
+    QueueClosed,
+    ReceiveTaskCancelled,
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+pub struct DownstairsClientStopped {
+    pub time: DateTime<Utc>,
+    pub reason: DownstairsClientStoppedReason,
+}
diff --git a/common/src/lib.rs b/common/src/lib.rs
index 411bc3e426..24fa4dfba0 100644
--- a/common/src/lib.rs
+++ b/common/src/lib.rs
@@ -77,3 +77,84 @@ impl slog::KV for FileKv {
 }
 
 pub const OMICRON_DPD_TAG: &str = "omicron";
+
+use futures::Future;
+use slog::warn;
+
+/// Retry a progenitor client operation until a known result is returned.
+///
+/// Saga execution relies on the outcome of an external call being known: since
+/// they are idempotent, reissue the external call until a known result comes
+/// back. Retry if a communication error is seen, or if another retryable error
+/// is seen.
+///
+/// Note that retrying is only valid if the call itself is idempotent.
+pub async fn retry_until_known_result<F, T, E, Fut>(
+    log: &slog::Logger,
+    mut f: F,
+) -> Result<T, progenitor_client::Error<E>>
+where
+    F: FnMut() -> Fut,
+    Fut: Future<Output = Result<T, progenitor_client::Error<E>>>,
+    E: std::fmt::Debug,
+{
+    backoff::retry_notify(
+        backoff::retry_policy_internal_service(),
+        move || {
+            let fut = f();
+            async move {
+                match fut.await {
+                    Err(progenitor_client::Error::CommunicationError(e)) => {
+                        warn!(
+                            log,
+                            "saw transient communication error {}, retrying...",
+                            e,
+                        );
+
+                        Err(backoff::BackoffError::transient(
+                            progenitor_client::Error::CommunicationError(e),
+                        ))
+                    }
+
+                    Err(progenitor_client::Error::ErrorResponse(
+                        response_value,
+                    )) => {
+                        match response_value.status() {
+                            // Retry on 503 or 429
+                            http::StatusCode::SERVICE_UNAVAILABLE
+                            | http::StatusCode::TOO_MANY_REQUESTS => {
+                                Err(backoff::BackoffError::transient(
+                                    progenitor_client::Error::ErrorResponse(
+                                        response_value,
+                                    ),
+                                ))
+                            }
+
+                            // Anything else is a permanent error
+                            _ => Err(backoff::BackoffError::Permanent(
+                                progenitor_client::Error::ErrorResponse(
+                                    response_value,
+                                ),
+                            )),
+                        }
+                    }
+
+                    Err(e) => {
+                        warn!(log, "saw permanent error {}, aborting", e,);
+
+                        Err(backoff::BackoffError::Permanent(e))
+                    }
+
+                    Ok(v) => Ok(v),
+                }
+            }
+        },
+        |error: progenitor_client::Error<_>, delay| {
+            warn!(
+                log,
+                "failed external call ({:?}), will retry in {:?}", error, delay,
+            );
+        },
+    )
+    .await
+}
diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml
index 2b8845f6f5..57d929d44d 100644
--- a/nexus/Cargo.toml
+++ b/nexus/Cargo.toml
@@ -97,6 +97,7 @@ rustls = { workspace = true }
 rustls-pemfile = { workspace = true }
 update-common.workspace = true
 omicron-workspace-hack.workspace = true
+omicron-uuid-kinds.workspace = true
 
 [dev-dependencies]
 async-bb8-diesel.workspace = true
diff --git a/nexus/db-model/src/downstairs.rs b/nexus/db-model/src/downstairs.rs
new file mode 100644
index 0000000000..6aaeadc810
--- /dev/null
+++ b/nexus/db-model/src/downstairs.rs
@@ -0,0 +1,133 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use super::impl_enum_type;
+use crate::schema::downstairs_client_stop_request_notification;
+use crate::schema::downstairs_client_stopped_notification;
+use crate::typed_uuid::DbTypedUuid;
+use chrono::{DateTime, Utc};
+use omicron_common::api::internal;
+use omicron_uuid_kinds::DownstairsKind;
+use omicron_uuid_kinds::UpstairsKind;
+use serde::{Deserialize, Serialize};
+
+// Types for stop request notification
+
+impl_enum_type!(
+    #[derive(SqlType, Debug, QueryId)]
+    #[diesel(postgres_type(name = "downstairs_client_stop_request_reason_type", schema = "public"))]
+    pub struct DownstairsClientStopRequestReasonEnum;
+
+    #[derive(Copy, Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)]
+    #[diesel(sql_type = DownstairsClientStopRequestReasonEnum)]
+    pub enum DownstairsClientStopRequestReason;
+
+    // Reason types
+    Replacing => b"replacing"
+    Disabled => b"disabled"
+    FailedReconcile => b"failed_reconcile"
+    IOError => b"io_error"
+    BadNegotiationOrder => b"bad_negotiation_order"
+    Incompatible => b"incompatible"
+    FailedLiveRepair => b"failed_live_repair"
+    TooManyOutstandingJobs => b"too_many_outstanding_jobs"
+    Deactivated => b"deactivated"
+);
+
+impl From<internal::nexus::DownstairsClientStopRequestReason>
+    for DownstairsClientStopRequestReason
+{
+    fn from(
+        v: internal::nexus::DownstairsClientStopRequestReason,
+    ) -> DownstairsClientStopRequestReason {
+        match v {
+            internal::nexus::DownstairsClientStopRequestReason::Replacing => DownstairsClientStopRequestReason::Replacing,
+            internal::nexus::DownstairsClientStopRequestReason::Disabled => DownstairsClientStopRequestReason::Disabled,
+            internal::nexus::DownstairsClientStopRequestReason::FailedReconcile => DownstairsClientStopRequestReason::FailedReconcile,
+            internal::nexus::DownstairsClientStopRequestReason::IOError => DownstairsClientStopRequestReason::IOError,
+            internal::nexus::DownstairsClientStopRequestReason::BadNegotiationOrder => DownstairsClientStopRequestReason::BadNegotiationOrder,
+            internal::nexus::DownstairsClientStopRequestReason::Incompatible => DownstairsClientStopRequestReason::Incompatible,
+            internal::nexus::DownstairsClientStopRequestReason::FailedLiveRepair => DownstairsClientStopRequestReason::FailedLiveRepair,
+            internal::nexus::DownstairsClientStopRequestReason::TooManyOutstandingJobs => DownstairsClientStopRequestReason::TooManyOutstandingJobs,
+            internal::nexus::DownstairsClientStopRequestReason::Deactivated => DownstairsClientStopRequestReason::Deactivated,
+        }
+    }
+}
+
+/// A Record of when an Upstairs requested a Downstairs client task stop
+#[derive(Queryable, Insertable, Debug, Clone, Selectable)]
+#[diesel(table_name = downstairs_client_stop_request_notification)]
+pub struct DownstairsClientStopRequestNotification {
+    // Importantly, this is client time, not Nexus' time that it received the
+    // notification.
+    pub time: DateTime<Utc>,
+
+    // Which Upstairs sent this notification?
+    pub upstairs_id: DbTypedUuid<UpstairsKind>,
+
+    // Which Downstairs client was requested to stop?
+    pub downstairs_id: DbTypedUuid<DownstairsKind>,
+
+    pub reason: DownstairsClientStopRequestReason,
+}
+
+// Types for stopped notification
+
+impl_enum_type!(
+    #[derive(SqlType, Debug, QueryId)]
+    #[diesel(postgres_type(name = "downstairs_client_stopped_reason_type", schema = "public"))]
+    pub struct DownstairsClientStoppedReasonEnum;
+
+    #[derive(Copy, Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)]
+    #[diesel(sql_type = DownstairsClientStoppedReasonEnum)]
+    pub enum DownstairsClientStoppedReason;
+
+    // Reason types
+    ConnectionTimeout => b"connection_timeout"
+    ConnectionFailed => b"connection_failed"
+    Timeout => b"timeout"
+    WriteFailed => b"write_failed"
+    ReadFailed => b"read_failed"
+    RequestedStop => b"requested_stop"
+    Finished => b"finished"
+    QueueClosed => b"queue_closed"
+    ReceiveTaskCancelled => b"receive_task_cancelled"
+);
+
+impl From<internal::nexus::DownstairsClientStoppedReason>
+    for DownstairsClientStoppedReason
+{
+    fn from(
+        v: internal::nexus::DownstairsClientStoppedReason,
+    ) -> DownstairsClientStoppedReason {
+        match v {
+            internal::nexus::DownstairsClientStoppedReason::ConnectionTimeout => DownstairsClientStoppedReason::ConnectionTimeout,
+            internal::nexus::DownstairsClientStoppedReason::ConnectionFailed => DownstairsClientStoppedReason::ConnectionFailed,
+            internal::nexus::DownstairsClientStoppedReason::Timeout => DownstairsClientStoppedReason::Timeout,
+            internal::nexus::DownstairsClientStoppedReason::WriteFailed => DownstairsClientStoppedReason::WriteFailed,
+            internal::nexus::DownstairsClientStoppedReason::ReadFailed => DownstairsClientStoppedReason::ReadFailed,
+            internal::nexus::DownstairsClientStoppedReason::RequestedStop => DownstairsClientStoppedReason::RequestedStop,
+            internal::nexus::DownstairsClientStoppedReason::Finished => DownstairsClientStoppedReason::Finished,
+            internal::nexus::DownstairsClientStoppedReason::QueueClosed => DownstairsClientStoppedReason::QueueClosed,
+            internal::nexus::DownstairsClientStoppedReason::ReceiveTaskCancelled => DownstairsClientStoppedReason::ReceiveTaskCancelled,
+        }
+    }
+}
+
+/// A Record of when a Downstairs client task stopped
+#[derive(Queryable, Insertable, Debug, Clone, Selectable)]
+#[diesel(table_name = downstairs_client_stopped_notification)]
+pub struct DownstairsClientStoppedNotification {
+    // Importantly, this is client time, not Nexus' time that it received the
+    // notification.
+    pub time: DateTime<Utc>,
+
+    // Which Upstairs sent this notification?
+    pub upstairs_id: DbTypedUuid<UpstairsKind>,
+
+    // Which Downstairs client was stopped?
+    pub downstairs_id: DbTypedUuid<DownstairsKind>,
+
+    pub reason: DownstairsClientStoppedReason,
+}
diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs
index 5c89134b78..d2b676a3da 100644
--- a/nexus/db-model/src/lib.rs
+++ b/nexus/db-model/src/lib.rs
@@ -26,6 +26,7 @@ mod digest;
 mod disk;
 mod disk_state;
 mod dns;
+mod downstairs;
 mod external_ip;
 mod generation;
 mod identity_provider;
@@ -85,6 +86,7 @@ mod switch;
 mod tuf_repo;
 mod typed_uuid;
 mod unsigned;
+mod upstairs_repair;
 mod user_builtin;
 mod utilization;
 mod virtual_provisioning_collection;
@@ -127,6 +129,7 @@ pub use digest::*;
 pub use disk::*;
 pub use disk_state::*;
 pub use dns::*;
+pub use downstairs::*;
 pub use external_ip::*;
 pub use generation::*;
 pub use identity_provider::*;
@@ -176,6 +179,7 @@ pub use switch_interface::*;
 pub use switch_port::*;
 pub use tuf_repo::*;
 pub use typed_uuid::to_db_typed_uuid;
+pub use upstairs_repair::*;
 pub use user_builtin::*;
 pub use utilization::*;
 pub use virtual_provisioning_collection::*;
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index bcbf7fa88f..c31ce16775 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(42, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(43, 0, 0);
 
 table! {
     disk (id) {
@@ -1549,6 +1549,50 @@ table! {
     }
 }
 
+table! {
+    upstairs_repair_notification (repair_id, upstairs_id, session_id, region_id, notification_type) {
+        time -> Timestamptz,
+
+        repair_id -> Uuid,
+        repair_type -> crate::UpstairsRepairTypeEnum,
+        upstairs_id -> Uuid,
+        session_id -> Uuid,
+
+        region_id -> Uuid,
+        target_ip -> Inet,
+        target_port -> Int4,
+
+        notification_type -> crate::UpstairsRepairNotificationTypeEnum,
+    }
+}
+
+table! {
+    upstairs_repair_progress (repair_id, time, current_item, total_items) {
+        repair_id -> Uuid,
+        time -> Timestamptz,
+        current_item -> Int8,
+        total_items -> Int8,
+    }
+}
+
+table! {
+    downstairs_client_stop_request_notification (time, upstairs_id, downstairs_id, reason) {
+        time -> Timestamptz,
+        upstairs_id -> Uuid,
+        downstairs_id -> Uuid,
+        reason -> crate::DownstairsClientStopRequestReasonEnum,
+    }
+}
+
+table! {
+    downstairs_client_stopped_notification (time, upstairs_id, downstairs_id, reason) {
+        time -> Timestamptz,
+        upstairs_id -> Uuid,
+        downstairs_id -> Uuid,
+        reason -> crate::DownstairsClientStoppedReasonEnum,
+    }
+}
+
 table! {
     db_metadata (singleton) {
         singleton -> Bool,
diff --git a/nexus/db-model/src/upstairs_repair.rs b/nexus/db-model/src/upstairs_repair.rs
new file mode 100644
index 0000000000..311592f8e4
--- /dev/null
+++ b/nexus/db-model/src/upstairs_repair.rs
@@ -0,0 +1,154 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use super::impl_enum_type;
+use crate::ipv6;
+use crate::schema::upstairs_repair_notification;
+use crate::schema::upstairs_repair_progress;
+use crate::typed_uuid::DbTypedUuid;
+use crate::SqlU16;
+use chrono::{DateTime, Utc};
+use omicron_common::api::internal;
+use omicron_uuid_kinds::DownstairsRegionKind;
+use omicron_uuid_kinds::TypedUuid;
+use omicron_uuid_kinds::UpstairsKind;
+use omicron_uuid_kinds::UpstairsRepairKind;
+use omicron_uuid_kinds::UpstairsSessionKind;
+use serde::{Deserialize, Serialize};
+use std::net::SocketAddrV6; // internal::nexus::UpstairsRepairType;
+
+impl_enum_type!(
+    #[derive(SqlType, Debug, QueryId)]
+    #[diesel(postgres_type(name = "upstairs_repair_notification_type", schema = "public"))]
+    pub struct UpstairsRepairNotificationTypeEnum;
+
+    #[derive(Copy, Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)]
+    #[diesel(sql_type = UpstairsRepairNotificationTypeEnum)]
+    pub enum UpstairsRepairNotificationType;
+
+    // Notification types
+    Started => b"started"
+    Succeeded => b"succeeded"
+    Failed => b"failed"
+);
+
+impl_enum_type!(
+    #[derive(SqlType, Debug, QueryId)]
+    #[diesel(postgres_type(name = "upstairs_repair_type", schema = "public"))]
+    pub struct UpstairsRepairTypeEnum;
+
+    #[derive(Copy, Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq, Eq, Hash)]
+    #[diesel(sql_type = UpstairsRepairTypeEnum)]
+    pub enum UpstairsRepairType;
+
+    // Types of repair a Crucible Upstairs can do
+    Live => b"live"
+    Reconciliation => b"reconciliation"
+);
+
+impl From<internal::nexus::UpstairsRepairType> for UpstairsRepairType {
+    fn from(v: internal::nexus::UpstairsRepairType) -> UpstairsRepairType {
+        match v {
+            internal::nexus::UpstairsRepairType::Live => {
+                UpstairsRepairType::Live
+            }
+            internal::nexus::UpstairsRepairType::Reconciliation => {
+                UpstairsRepairType::Reconciliation
+            }
+        }
+    }
+}
+
+/// A record of Crucible Upstairs repair notifications: when a repair started,
+/// succeeded, failed, etc.
+///
+/// Each repair attempt is uniquely identified by the repair ID, upstairs ID,
+/// session ID, and region ID. How those change tells Nexus about what is going
+/// on:
+///
+/// - if all IDs are the same for different requests, Nexus knows that the
+///   client is retrying the notification.
+///
+/// - if the upstairs ID, session ID, and region ID are all the same, but the
+///   repair ID is different, then the same Upstairs is trying to repair that
+///   region again. This could be due to a failed first attempt, or that
+///   downstairs may have been kicked out again.
+///
+/// - if the upstairs ID and region ID are the same, but the session ID and
+///   repair ID are different, then a different session of the same Upstairs is
+///   trying to repair that Downstairs. Session IDs change each time the
+///   Upstairs is created, so it could have crashed, or it could have been
+///   migrated and the destination Propolis' Upstairs is attempting to repair
+///   the same region.
+#[derive(Queryable, Insertable, Debug, Clone, Selectable)]
+#[diesel(table_name = upstairs_repair_notification)]
+pub struct UpstairsRepairNotification {
+    // Importantly, this is client time, not Nexus' time that it received the
+    // notification.
+    pub time: DateTime<Utc>,
+
+    pub repair_id: DbTypedUuid<UpstairsRepairKind>,
+
+    // There's a difference between the live repairs and reconciliation: the
+    // Upstairs can go through reconciliation without there being any error from
+    // a downstairs, or any region replacement request from Nexus. One example
+    // is if the rack power is pulled: if everything is powered back up again
+    // reconciliation could be required but this isn't the fault of any problem
+    // with a physical disk, or any error that was returned.
+    //
+    // Alternatively any record of a live repair means that there was a problem:
+    // Currently, either an Upstairs kicked out a Downstairs (or two) due to
+    // some error or because it lagged behind the others, or Nexus has
+    // instructed an Upstairs to perform a region replacement.
+    pub repair_type: UpstairsRepairType,
+
+    pub upstairs_id: DbTypedUuid<UpstairsKind>,
+    pub session_id: DbTypedUuid<UpstairsSessionKind>,
+
+    pub region_id: DbTypedUuid<DownstairsRegionKind>,
+    pub target_ip: ipv6::Ipv6Addr,
+    pub target_port: SqlU16,
+
+    pub notification_type: UpstairsRepairNotificationType,
+}
+
+impl UpstairsRepairNotification {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        time: DateTime<Utc>,
+        repair_id: TypedUuid<UpstairsRepairKind>,
+        repair_type: UpstairsRepairType,
+        upstairs_id: TypedUuid<UpstairsKind>,
+        session_id: TypedUuid<UpstairsSessionKind>,
+        region_id: TypedUuid<DownstairsRegionKind>,
+        target_addr: SocketAddrV6,
+        notification_type: UpstairsRepairNotificationType,
+    ) -> Self {
+        Self {
+            time,
+            repair_id: repair_id.into(),
+            repair_type,
+            upstairs_id: upstairs_id.into(),
+            session_id: session_id.into(),
+            region_id: region_id.into(),
+            target_ip: target_addr.ip().into(),
+            target_port: target_addr.port().into(),
+            notification_type,
+        }
+    }
+
+    pub fn address(&self) -> SocketAddrV6 {
+        SocketAddrV6::new(*self.target_ip, *self.target_port, 0, 0)
+    }
+}
+
+/// A record of Crucible Upstairs repair progress.
+#[derive(Queryable, Insertable, Debug, Clone, Selectable)]
+#[diesel(table_name = upstairs_repair_progress)]
+pub struct UpstairsRepairProgress {
+    pub repair_id: DbTypedUuid<UpstairsRepairKind>,
+    pub time: DateTime<Utc>,
+    pub current_item: i64,
+    pub total_items: i64,
+}
diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs
index 374ef2cf73..a9646b9ef6 100644
--- a/nexus/db-queries/src/db/datastore/volume.rs
+++ b/nexus/db-queries/src/db/datastore/volume.rs
@@ -6,12 +6,18 @@
 
 use super::DataStore;
 use crate::db;
+use crate::db::datastore::OpContext;
 use crate::db::error::public_error_from_diesel;
 use crate::db::error::ErrorHandler;
 use crate::db::identity::Asset;
 use crate::db::model::Dataset;
+use crate::db::model::DownstairsClientStopRequestNotification;
+use crate::db::model::DownstairsClientStoppedNotification;
 use crate::db::model::Region;
 use crate::db::model::RegionSnapshot;
+use crate::db::model::UpstairsRepairNotification;
+use crate::db::model::UpstairsRepairNotificationType;
+use crate::db::model::UpstairsRepairProgress;
 use crate::db::model::Volume;
 use crate::db::queries::volume::DecreaseCrucibleResourceCountAndSoftDeleteVolume;
 use crate::transaction_retry::OptionalError;
@@ -25,6 +31,13 @@ use omicron_common::api::external::Error;
 use omicron_common::api::external::ListResultVec;
 use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::ResourceType;
+use omicron_common::api::internal::nexus::DownstairsClientStopRequest;
+use omicron_common::api::internal::nexus::DownstairsClientStopped;
+use omicron_common::api::internal::nexus::RepairProgress;
+use omicron_uuid_kinds::DownstairsKind;
+use omicron_uuid_kinds::TypedUuid;
+use omicron_uuid_kinds::UpstairsKind;
+use omicron_uuid_kinds::UpstairsRepairKind;
 use serde::Deserialize;
 use serde::Deserializer;
 use serde::Serialize;
@@ -809,6 +822,242 @@ impl DataStore {
                 public_error_from_diesel(e, ErrorHandler::Server)
             })
     }
+
+    // An Upstairs is created as part of a Volume hierarchy if the Volume
+    // Construction Request includes a "Region" variant. This may be at any
+    // layer of the Volume, and some notifications will come from an Upstairs
+    // instead of the top level of the Volume. The following functions have an
+    // Upstairs ID instead of a Volume ID for this reason.
+
+    /// Record when an Upstairs notifies us about a repair. If that record
+    /// (uniquely identified by the four IDs passed in plus the notification
+    /// type) exists already, do nothing.
+    pub async fn upstairs_repair_notification(
+        &self,
+        opctx: &OpContext,
+        record: UpstairsRepairNotification,
+    ) -> Result<(), Error> {
+        use db::schema::upstairs_repair_notification::dsl;
+
+        let conn = self.pool_connection_authorized(opctx).await?;
+        let err = OptionalError::new();
+
+        self.transaction_retry_wrapper("upstairs_repair_notification")
+            .transaction(&conn, |conn| {
+                let record = record.clone();
+                let err = err.clone();
+
+                async move {
+                    // Return 409 if a repair ID does not match types
+                    let mismatched_record_type_count: usize =
+                        dsl::upstairs_repair_notification
+                            .filter(dsl::repair_id.eq(record.repair_id))
+                            .filter(dsl::repair_type.ne(record.repair_type))
+                            .execute_async(&conn)
+                            .await?;
+
+                    if mismatched_record_type_count > 0 {
+                        return Err(err.bail(Error::conflict(&format!(
+                            "existing repair type for id {} does not match {:?}!",
+                            record.repair_id,
+                            record.repair_type,
+                        ))));
+                    }
+
+                    match &record.notification_type {
+                        UpstairsRepairNotificationType::Started => {
+                            // Proceed - the insertion can succeed or fail below
+                            // based on the table's primary key
+                        }
+
+                        UpstairsRepairNotificationType::Succeeded
+                        | UpstairsRepairNotificationType::Failed => {
+                            // However, Nexus must accept only one "finished"
+                            // status - an Upstairs cannot change this and must
+                            // instead perform another repair with a new repair
+                            // ID.
+                            let maybe_existing_finish_record: Option<
+                                UpstairsRepairNotification,
+                            > = dsl::upstairs_repair_notification
+                                .filter(dsl::repair_id.eq(record.repair_id))
+                                .filter(dsl::upstairs_id.eq(record.upstairs_id))
+                                .filter(dsl::session_id.eq(record.session_id))
+                                .filter(dsl::region_id.eq(record.region_id))
+                                .filter(dsl::notification_type.eq_any(vec![
+                                    UpstairsRepairNotificationType::Succeeded,
+                                    UpstairsRepairNotificationType::Failed,
+                                ]))
+                                .get_result_async(&conn)
+                                .await
+                                .optional()?;
+
+                            if let Some(existing_finish_record) =
+                                maybe_existing_finish_record
+                            {
+                                if existing_finish_record.notification_type
+                                    != record.notification_type
+                                {
+                                    return Err(err.bail(Error::conflict(
+                                        "existing finish record does not match",
+                                    )));
+                                } else {
+                                    // inserting the same record, bypass
+                                    return Ok(());
+                                }
+                            }
+                        }
+                    }
+
+                    diesel::insert_into(dsl::upstairs_repair_notification)
+                        .values(record)
+                        .on_conflict((
+                            dsl::repair_id,
+                            dsl::upstairs_id,
+                            dsl::session_id,
+                            dsl::region_id,
+                            dsl::notification_type,
+                        ))
+                        .do_nothing()
+                        .execute_async(&conn)
+                        .await?;
+
+                    Ok(())
+                }
+            })
+            .await
+            .map_err(|e| {
+                if let Some(err) = err.take() {
+                    err
+                } else {
+                    public_error_from_diesel(e, ErrorHandler::Server)
+                }
+            })
+    }
+
+    /// Record Upstairs repair progress
+    pub async fn upstairs_repair_progress(
+        &self,
+        opctx: &OpContext,
+        upstairs_id: TypedUuid<UpstairsKind>,
+        repair_id: TypedUuid<UpstairsRepairKind>,
+        repair_progress: RepairProgress,
+    ) -> Result<(), Error> {
+        use db::schema::upstairs_repair_notification::dsl as notification_dsl;
+        use db::schema::upstairs_repair_progress::dsl;
+
+        let conn = self.pool_connection_authorized(opctx).await?;
+        let err = OptionalError::new();
+
+        self.transaction_retry_wrapper("upstairs_repair_progress")
+            .transaction(&conn, |conn| {
+                let repair_progress = repair_progress.clone();
+                let err = err.clone();
+
+                async move {
+                    // Check that there is a repair id for the upstairs id
+                    let matching_repair: Option<UpstairsRepairNotification> =
+                        notification_dsl::upstairs_repair_notification
+                            .filter(notification_dsl::repair_id.eq(nexus_db_model::to_db_typed_uuid(repair_id)))
+                            .filter(notification_dsl::upstairs_id.eq(nexus_db_model::to_db_typed_uuid(upstairs_id)))
+                            .filter(notification_dsl::notification_type.eq(UpstairsRepairNotificationType::Started))
+                            .get_result_async(&conn)
+                            .await
+                            .optional()?;
+
+                    if matching_repair.is_none() {
+                        return Err(err.bail(Error::non_resourcetype_not_found(&format!(
+                            "upstairs {upstairs_id} repair {repair_id} not found"
+                        ))));
+                    }
+
+                    diesel::insert_into(dsl::upstairs_repair_progress)
+                        .values(UpstairsRepairProgress {
+                            repair_id: repair_id.into(),
+                            time: repair_progress.time,
+                            current_item: repair_progress.current_item,
+                            total_items: repair_progress.total_items,
+                        })
+                        .execute_async(&conn)
+                        .await?;
+
+                    Ok(())
+                }
+            })
+            .await
+            .map_err(|e| {
+                if let Some(err) = err.take() {
+                    err
+                } else {
+                    public_error_from_diesel(e, ErrorHandler::Server)
+                }
+            })
+    }
+
+    /// Record when a Downstairs client is requested to stop, and why
+    pub async fn downstairs_client_stop_request_notification(
+        &self,
+        opctx: &OpContext,
+        upstairs_id: TypedUuid<UpstairsKind>,
+        downstairs_id: TypedUuid<DownstairsKind>,
+        downstairs_client_stop_request: DownstairsClientStopRequest,
+    ) -> Result<(), Error> {
+        use db::schema::downstairs_client_stop_request_notification::dsl;
+
+        let conn = self.pool_connection_authorized(opctx).await?;
+
+        diesel::insert_into(dsl::downstairs_client_stop_request_notification)
+            .values(DownstairsClientStopRequestNotification {
+                time: downstairs_client_stop_request.time,
+                upstairs_id: upstairs_id.into(),
+                downstairs_id: downstairs_id.into(),
+                reason: downstairs_client_stop_request.reason.into(),
+            })
+            .on_conflict((
+                dsl::time,
+                dsl::upstairs_id,
+                dsl::downstairs_id,
+                dsl::reason,
+            ))
+            .do_nothing()
+            .execute_async(&*conn)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        Ok(())
+    }
+
+    /// Record when a Downstairs client is stopped, and why
+    pub async fn downstairs_client_stopped_notification(
+        &self,
+        opctx: &OpContext,
+        upstairs_id: TypedUuid<UpstairsKind>,
+        downstairs_id: TypedUuid<DownstairsKind>,
+        downstairs_client_stopped: DownstairsClientStopped,
+    ) -> Result<(), Error> {
+        use db::schema::downstairs_client_stopped_notification::dsl;
+
+        let conn = self.pool_connection_authorized(opctx).await?;
+
+        diesel::insert_into(dsl::downstairs_client_stopped_notification)
+            .values(DownstairsClientStoppedNotification {
+                time: downstairs_client_stopped.time,
+                upstairs_id: upstairs_id.into(),
+                downstairs_id: downstairs_id.into(),
+                reason: downstairs_client_stopped.reason.into(),
+            })
+            .on_conflict((
+                dsl::time,
+                dsl::upstairs_id,
+                dsl::downstairs_id,
+                dsl::reason,
+            ))
+            .do_nothing()
+            .execute_async(&*conn)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        Ok(())
+    }
 }
 
 #[derive(Default, Clone, Debug, Serialize, Deserialize)]
diff --git a/nexus/db-queries/src/db/pool_connection.rs b/nexus/db-queries/src/db/pool_connection.rs
index f419ba6852..0331a3a103 100644
--- a/nexus/db-queries/src/db/pool_connection.rs
+++ b/nexus/db-queries/src/db/pool_connection.rs
@@ -44,6 +44,8 @@ static CUSTOM_TYPE_KEYS: &'static [&'static str] = &[
     "caboose_which",
     "dataset_kind",
     "dns_group",
+    "downstairs_client_stop_request_reason_type",
+    "downstairs_client_stopped_reason_type",
     "hw_power_state",
     "hw_rot_slot",
     "identity_type",
@@ -69,6 +71,8 @@ static CUSTOM_TYPE_KEYS: &'static [&'static str] = &[
     "switch_link_fec",
     "switch_link_speed",
     "switch_port_geometry",
+    "upstairs_repair_notification_type",
+    "upstairs_repair_type",
     "user_provision_type",
     "vpc_firewall_rule_action",
     "vpc_firewall_rule_direction",
diff --git a/nexus/src/app/instance_network.rs b/nexus/src/app/instance_network.rs
index eb5f83470f..741b5b8b6d 100644
--- a/nexus/src/app/instance_network.rs
+++ b/nexus/src/app/instance_network.rs
@@ -4,7 +4,6 @@
 
 //! Routines that manage instance-related networking state.
 
-use crate::app::sagas::retry_until_known_result;
 use ipnetwork::IpNetwork;
 use ipnetwork::Ipv6Network;
 use nexus_db_model::ExternalIp;
@@ -24,6 +23,7 @@ use omicron_common::api::external::Ipv6Net;
 use omicron_common::api::internal::nexus;
 use omicron_common::api::internal::shared::NetworkInterface;
 use omicron_common::api::internal::shared::SwitchLocation;
+use omicron_common::retry_until_known_result;
 use sled_agent_client::types::DeleteVirtualNetworkInterfaceHost;
 use sled_agent_client::types::SetVirtualNetworkInterfaceHost;
 use std::collections::HashSet;
diff --git a/nexus/src/app/sagas/common_storage.rs b/nexus/src/app/sagas/common_storage.rs
index a7350d91fd..3b590f6205 100644
--- a/nexus/src/app/sagas/common_storage.rs
+++ b/nexus/src/app/sagas/common_storage.rs
@@ -6,7 +6,6 @@
 
 use super::*;
 
-use crate::app::sagas::retry_until_known_result;
 use crate::Nexus;
 use anyhow::anyhow;
 use crucible_agent_client::{
@@ -22,6 +21,7 @@ use nexus_db_queries::db::identity::Asset;
 use nexus_db_queries::db::lookup::LookupPath;
 use omicron_common::api::external::Error;
 use omicron_common::backoff::{self, BackoffError};
+use omicron_common::retry_until_known_result;
 use slog::Logger;
 use std::net::SocketAddrV6;
 
diff --git a/nexus/src/app/sagas/loopback_address_create.rs b/nexus/src/app/sagas/loopback_address_create.rs
index a5d89f202c..c32a5f387d 100644
--- a/nexus/src/app/sagas/loopback_address_create.rs
+++ b/nexus/src/app/sagas/loopback_address_create.rs
@@ -3,7 +3,6 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 use super::{NexusActionContext, NEXUS_DPD_TAG};
-use crate::app::sagas::retry_until_known_result;
 use crate::app::sagas::{
     declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError,
 };
@@ -13,6 +12,7 @@ use nexus_db_queries::authn;
 use nexus_db_queries::authz;
 use nexus_db_queries::db::model::LoopbackAddress;
 use omicron_common::api::internal::shared::SwitchLocation;
+use omicron_common::retry_until_known_result;
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 use steno::ActionError;
diff --git a/nexus/src/app/sagas/loopback_address_delete.rs b/nexus/src/app/sagas/loopback_address_delete.rs
index a030178d27..822a360acf 100644
--- a/nexus/src/app/sagas/loopback_address_delete.rs
+++ b/nexus/src/app/sagas/loopback_address_delete.rs
@@ -3,7 +3,6 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 use super::NexusActionContext;
-use crate::app::sagas::retry_until_known_result;
 use crate::app::sagas::{
     declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError,
 };
@@ -14,6 +13,7 @@ use nexus_db_queries::authz;
 use nexus_db_queries::db::model::{LoopbackAddress, Name};
 use nexus_types::identity::Asset;
 use omicron_common::api::external::{IpNet, NameOrId};
+use omicron_common::retry_until_known_result;
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 use steno::ActionError;
diff --git a/nexus/src/app/sagas/mod.rs b/nexus/src/app/sagas/mod.rs
index aef8442090..01b01c4571 100644
--- a/nexus/src/app/sagas/mod.rs
+++ b/nexus/src/app/sagas/mod.rs
@@ -328,88 +328,6 @@ pub(crate) use __emit_action;
 pub(crate) use __stringify_ident;
 pub(crate) use declare_saga_actions;
 
-use futures::Future;
-
-/// Retry a progenitor client operation until a known result is returned.
-///
-/// Saga execution relies on the outcome of an external call being known: since
-/// they are idempotent, reissue the external call until a known result comes
-/// back. Retry if a communication error is seen, or if another retryable error
-/// is seen.
-///
-/// Note that retrying is only valid if the call itself is idempotent.
-pub(crate) async fn retry_until_known_result<F, T, E, Fut>(
-    log: &slog::Logger,
-    mut f: F,
-) -> Result<T, progenitor_client::Error<E>>
-where
-    F: FnMut() -> Fut,
-    Fut: Future<Output = Result<T, progenitor_client::Error<E>>>,
-    E: std::fmt::Debug,
-{
-    use omicron_common::backoff;
-
-    backoff::retry_notify(
-        backoff::retry_policy_internal_service(),
-        move || {
-            let fut = f();
-            async move {
-                match fut.await {
-                    Err(progenitor_client::Error::CommunicationError(e)) => {
-                        warn!(
-                            log,
-                            "saw transient communication error {}, retrying...",
-                            e,
-                        );
-
-                        Err(backoff::BackoffError::transient(
-                            progenitor_client::Error::CommunicationError(e),
-                        ))
-                    }
-
-                    Err(progenitor_client::Error::ErrorResponse(
-                        response_value,
-                    )) => {
-                        match response_value.status() {
-                            // Retry on 503 or 429
-                            http::StatusCode::SERVICE_UNAVAILABLE
-                            | http::StatusCode::TOO_MANY_REQUESTS => {
-                                Err(backoff::BackoffError::transient(
-                                    progenitor_client::Error::ErrorResponse(
-                                        response_value,
-                                    ),
-                                ))
-                            }
-
-                            // Anything else is a permanent error
-                            _ => Err(backoff::BackoffError::Permanent(
-                                progenitor_client::Error::ErrorResponse(
-                                    response_value,
-                                ),
-                            )),
-                        }
-                    }
-
-                    Err(e) => {
-                        warn!(log, "saw permanent error {}, aborting", e,);
-
-                        Err(backoff::BackoffError::Permanent(e))
-                    }
-
-                    Ok(v) => Ok(v),
-                }
-            }
-        },
-        |error: progenitor_client::Error<_>, delay| {
-            warn!(
-                log,
-                "failed external call ({:?}), will retry in {:?}", error, delay,
-            );
-        },
-    )
-    .await
-}
-
 /// Reliable persistent workflows can request that sagas be run as part of their
 /// activation by sending a SagaRequest through a supplied channel to Nexus.
 pub enum SagaRequest {
diff --git a/nexus/src/app/sagas/snapshot_create.rs b/nexus/src/app/sagas/snapshot_create.rs
index e017ab377b..f1d1a2bd02 100644
--- a/nexus/src/app/sagas/snapshot_create.rs
+++ b/nexus/src/app/sagas/snapshot_create.rs
@@ -99,7 +99,6 @@ use super::{
     ACTION_GENERATE_ID,
 };
 use crate::app::sagas::declare_saga_actions;
-use crate::app::sagas::retry_until_known_result;
 use crate::app::{authn, authz, db};
 use crate::external_api::params;
 use anyhow::anyhow;
@@ -109,6 +108,7 @@ use nexus_db_queries::db::identity::{Asset, Resource};
 use nexus_db_queries::db::lookup::LookupPath;
 use omicron_common::api::external;
 use omicron_common::api::external::Error;
+use omicron_common::retry_until_known_result;
 use rand::{rngs::StdRng, RngCore, SeedableRng};
 use serde::Deserialize;
 use serde::Serialize;
diff --git a/nexus/src/app/sagas/switch_port_settings_apply.rs b/nexus/src/app/sagas/switch_port_settings_apply.rs
index 9e2331f416..44f2f77ea1 100644
--- a/nexus/src/app/sagas/switch_port_settings_apply.rs
+++ b/nexus/src/app/sagas/switch_port_settings_apply.rs
@@ -3,7 +3,6 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 use super::{NexusActionContext, NEXUS_DPD_TAG};
-use crate::app::sagas::retry_until_known_result;
 use crate::app::sagas::switch_port_settings_common::{
     api_to_dpd_port_settings, ensure_switch_port_bgp_settings,
     ensure_switch_port_uplink, select_dendrite_client, select_mg_client,
@@ -24,6 +23,7 @@ use nexus_db_queries::db::datastore::UpdatePrecondition;
 use nexus_db_queries::{authn, db};
 use omicron_common::api::external::{self, NameOrId};
 use omicron_common::api::internal::shared::SwitchLocation;
+use omicron_common::retry_until_known_result;
 use serde::{Deserialize, Serialize};
 use std::net::IpAddr;
 use std::str::FromStr;
diff --git a/nexus/src/app/sagas/switch_port_settings_clear.rs b/nexus/src/app/sagas/switch_port_settings_clear.rs
index 15290dd75b..2e35530ef1 100644
--- a/nexus/src/app/sagas/switch_port_settings_clear.rs
+++ b/nexus/src/app/sagas/switch_port_settings_clear.rs
@@ -3,7 +3,6 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 use super::{NexusActionContext, NEXUS_DPD_TAG};
-use crate::app::sagas::retry_until_known_result;
 use crate::app::sagas::switch_port_settings_common::{
     api_to_dpd_port_settings, apply_bootstore_update, bootstore_update,
     ensure_switch_port_bgp_settings, ensure_switch_port_uplink,
@@ -23,6 +22,7 @@ use nexus_db_model::NETWORK_KEY;
 use nexus_db_queries::authn;
 use nexus_db_queries::db::datastore::UpdatePrecondition;
 use omicron_common::api::external::{self, NameOrId, SwitchLocation};
+use omicron_common::retry_until_known_result;
 use serde::{Deserialize, Serialize};
 use std::net::IpAddr;
 use std::str::FromStr;
diff --git a/nexus/src/app/volume.rs b/nexus/src/app/volume.rs
index c36c4524c1..8cfffdb686 100644
--- a/nexus/src/app/volume.rs
+++ b/nexus/src/app/volume.rs
@@ -5,9 +5,20 @@
 //! Volumes
 
 use crate::app::sagas;
+use nexus_db_model::UpstairsRepairNotification;
+use nexus_db_model::UpstairsRepairNotificationType;
 use nexus_db_queries::authn;
 use nexus_db_queries::context::OpContext;
 use omicron_common::api::external::DeleteResult;
+use omicron_common::api::internal::nexus::DownstairsClientStopRequest;
+use omicron_common::api::internal::nexus::DownstairsClientStopped;
+use omicron_common::api::internal::nexus::RepairFinishInfo;
+use omicron_common::api::internal::nexus::RepairProgress;
+use omicron_common::api::internal::nexus::RepairStartInfo;
+use omicron_uuid_kinds::DownstairsKind;
+use omicron_uuid_kinds::TypedUuid;
+use omicron_uuid_kinds::UpstairsKind;
+use omicron_uuid_kinds::UpstairsRepairKind;
 use std::sync::Arc;
 use uuid::Uuid;
 
@@ -30,4 +41,155 @@ impl super::Nexus {
 
         Ok(())
     }
+
+    /// An Upstairs is telling us when a repair is starting.
+    pub(crate) async fn upstairs_repair_start(
+        self: &Arc<Self>,
+        opctx: &OpContext,
+        upstairs_id: TypedUuid<UpstairsKind>,
+        repair_start_info: RepairStartInfo,
+    ) -> DeleteResult {
+        info!(
+            self.log,
+            "received upstairs_repair_start from upstairs {upstairs_id}: {:?}",
+            repair_start_info,
+        );
+
+        for repaired_downstairs in repair_start_info.repairs {
+            self.db_datastore
+                .upstairs_repair_notification(
+                    opctx,
+                    UpstairsRepairNotification::new(
+                        repair_start_info.time,
+                        repair_start_info.repair_id,
+                        repair_start_info.repair_type.into(),
+                        upstairs_id,
+                        repair_start_info.session_id,
+                        repaired_downstairs.region_uuid,
+                        repaired_downstairs.target_addr,
+                        UpstairsRepairNotificationType::Started,
+                    ),
+                )
+                .await?;
+        }
+
+        Ok(())
+    }
+
+    /// An Upstairs is telling us when a repair is finished, and the result.
+    pub(crate) async fn upstairs_repair_finish(
+        self: &Arc<Self>,
+        opctx: &OpContext,
+        upstairs_id: TypedUuid<UpstairsKind>,
+        repair_finish_info: RepairFinishInfo,
+    ) -> DeleteResult {
+        info!(
+            self.log,
+            "received upstairs_repair_finish from upstairs {upstairs_id}: {:?}",
+            repair_finish_info,
+        );
+
+        for repaired_downstairs in repair_finish_info.repairs {
+            self.db_datastore
+                .upstairs_repair_notification(
+                    opctx,
+                    UpstairsRepairNotification::new(
+                        repair_finish_info.time,
+                        repair_finish_info.repair_id,
+                        repair_finish_info.repair_type.into(),
+                        upstairs_id,
+                        repair_finish_info.session_id,
+                        repaired_downstairs.region_uuid,
+                        repaired_downstairs.target_addr,
+                        if repair_finish_info.aborted {
+                            UpstairsRepairNotificationType::Failed
+                        } else {
+                            UpstairsRepairNotificationType::Succeeded
+                        },
+                    ),
+                )
+                .await?;
+
+            if !repair_finish_info.aborted {
+                // TODO-followup if there's an active region replacement
+                // occurring, a successfully completed live repair can trigger a
+                // saga to destroy the original region.
+            }
+        }
+
+        Ok(())
+    }
+
+    /// An Upstairs is updating us with repair progress
+    pub(crate) async fn upstairs_repair_progress(
+        self: &Arc<Self>,
+        opctx: &OpContext,
+        upstairs_id: TypedUuid<UpstairsKind>,
+        repair_id: TypedUuid<UpstairsRepairKind>,
+        repair_progress: RepairProgress,
+    ) -> DeleteResult {
+        info!(
+            self.log,
+            "received upstairs_repair_progress from upstairs {upstairs_id} for repair {repair_id}: {:?}",
+            repair_progress,
+        );
+
+        self.db_datastore
+            .upstairs_repair_progress(
+                opctx,
+                upstairs_id,
+                repair_id,
+                repair_progress,
+            )
+            .await
+    }
+
+    /// An Upstairs is telling us that a Downstairs client task was requested to
+    /// stop
+    pub(crate) async fn downstairs_client_stop_request_notification(
+        self: &Arc<Self>,
+        opctx: &OpContext,
+        upstairs_id: TypedUuid<UpstairsKind>,
+        downstairs_id: TypedUuid<DownstairsKind>,
+        downstairs_client_stop_request: DownstairsClientStopRequest,
+    ) -> DeleteResult {
+        info!(
+            self.log,
+            "received downstairs_client_stop_request_notification from upstairs {upstairs_id} for downstairs {downstairs_id}: {:?}",
+            downstairs_client_stop_request,
+        );
+
+        self.db_datastore
+            .downstairs_client_stop_request_notification(
+                opctx,
+                upstairs_id,
+                downstairs_id,
+                downstairs_client_stop_request,
+            )
+            .await
+    }
+
+    /// An Upstairs is telling us that a Downstairs client task was stopped
+    pub(crate) async fn downstairs_client_stopped_notification(
+        self: &Arc<Self>,
+        opctx: &OpContext,
+        upstairs_id: TypedUuid<UpstairsKind>,
+        downstairs_id: TypedUuid<DownstairsKind>,
+        downstairs_client_stopped: DownstairsClientStopped,
+    ) -> DeleteResult {
+        info!(
+            self.log,
+            "received downstairs_client_stopped_notification from upstairs {upstairs_id} for downstairs {downstairs_id}: {:?}",
+            downstairs_client_stopped,
+        );
+
+        self.db_datastore
+            .downstairs_client_stopped_notification(
+                opctx,
+                upstairs_id,
+                downstairs_id,
+                downstairs_client_stopped,
+            )
+            .await
+    }
 }
diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs
index 9871c38117..0676ace70c 100644
--- a/nexus/src/internal_api/http_entrypoints.rs
+++ b/nexus/src/internal_api/http_entrypoints.rs
@@ -45,9 +45,18 @@ use omicron_common::api::external::http_pagination::ScanById;
 use omicron_common::api::external::http_pagination::ScanParams;
 use omicron_common::api::external::Error;
 use omicron_common::api::internal::nexus::DiskRuntimeState;
+use omicron_common::api::internal::nexus::DownstairsClientStopRequest;
+use omicron_common::api::internal::nexus::DownstairsClientStopped;
 use omicron_common::api::internal::nexus::ProducerEndpoint;
+use omicron_common::api::internal::nexus::RepairFinishInfo;
+use omicron_common::api::internal::nexus::RepairProgress;
+use omicron_common::api::internal::nexus::RepairStartInfo;
 use omicron_common::api::internal::nexus::SledInstanceState;
 use omicron_common::update::ArtifactId;
+use omicron_uuid_kinds::DownstairsKind;
+use omicron_uuid_kinds::TypedUuid;
+use omicron_uuid_kinds::UpstairsKind;
+use omicron_uuid_kinds::UpstairsRepairKind;
 use oximeter::types::ProducerResults;
 use oximeter_producer::{collect, ProducerIdPathParams};
 use schemars::JsonSchema;
@@ -78,6 +87,12 @@ pub(crate) fn internal_api() -> NexusApiDescription {
         api.register(cpapi_metrics_collect)?;
         api.register(cpapi_artifact_download)?;
 
+        api.register(cpapi_upstairs_repair_start)?;
+        api.register(cpapi_upstairs_repair_finish)?;
+        api.register(cpapi_upstairs_repair_progress)?;
+        api.register(cpapi_downstairs_client_stop_request)?;
+        api.register(cpapi_downstairs_client_stopped)?;
+
         api.register(saga_list)?;
         api.register(saga_view)?;
 
@@ -513,6 +528,171 @@ async fn cpapi_artifact_download(
     Ok(HttpResponseOk(Body::from(body).into()))
 }
 
+/// Path parameters for Upstairs requests (internal API)
+#[derive(Deserialize, JsonSchema)]
+struct UpstairsPathParam {
+    upstairs_id: TypedUuid<UpstairsKind>,
+}
+
+/// An Upstairs will notify this endpoint when a repair starts
+#[endpoint {
+     method = POST,
+     path = "/crucible/0/upstairs/{upstairs_id}/repair-start",
+ }]
+async fn cpapi_upstairs_repair_start(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    path_params: Path<UpstairsPathParam>,
+    repair_start_info: TypedBody<RepairStartInfo>,
+) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+    let apictx = rqctx.context();
+    let nexus = &apictx.nexus;
+    let path = path_params.into_inner();
+
+    let handler = async {
+        let opctx = crate::context::op_context_for_internal_api(&rqctx).await;
+        nexus
+            .upstairs_repair_start(
+                &opctx,
+                path.upstairs_id,
+                repair_start_info.into_inner(),
+            )
+            .await?;
+        Ok(HttpResponseUpdatedNoContent())
+    };
+    apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
+/// An Upstairs will notify this endpoint when a repair finishes.
+#[endpoint {
+     method = POST,
+     path = "/crucible/0/upstairs/{upstairs_id}/repair-finish",
+ }]
+async fn cpapi_upstairs_repair_finish(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    path_params: Path<UpstairsPathParam>,
+    repair_finish_info: TypedBody<RepairFinishInfo>,
+) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+    let apictx = rqctx.context();
+    let nexus = &apictx.nexus;
+    let path = path_params.into_inner();
+
+    let handler = async {
+        let opctx = crate::context::op_context_for_internal_api(&rqctx).await;
+        nexus
+            .upstairs_repair_finish(
+                &opctx,
+                path.upstairs_id,
+                repair_finish_info.into_inner(),
+            )
+            .await?;
+        Ok(HttpResponseUpdatedNoContent())
+    };
+    apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
+/// Path parameters for Upstairs requests (internal API)
+#[derive(Deserialize, JsonSchema)]
+struct UpstairsRepairPathParam {
+    upstairs_id: TypedUuid<UpstairsKind>,
+    repair_id: TypedUuid<UpstairsRepairKind>,
+}
+
+/// An Upstairs will update this endpoint with the progress of a repair
+#[endpoint {
+     method = POST,
+     path = "/crucible/0/upstairs/{upstairs_id}/repair/{repair_id}/progress",
+ }]
+async fn cpapi_upstairs_repair_progress(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    path_params: Path<UpstairsRepairPathParam>,
+    repair_progress: TypedBody<RepairProgress>,
+) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+    let apictx = rqctx.context();
+    let nexus = &apictx.nexus;
+    let path = path_params.into_inner();
+
+    let handler = async {
+        let opctx = crate::context::op_context_for_internal_api(&rqctx).await;
+        nexus
+            .upstairs_repair_progress(
+                &opctx,
+                path.upstairs_id,
+                path.repair_id,
+                repair_progress.into_inner(),
+            )
+            .await?;
+        Ok(HttpResponseUpdatedNoContent())
+    };
+    apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
+/// Path parameters for Downstairs requests (internal API)
+#[derive(Deserialize, JsonSchema)]
+struct UpstairsDownstairsPathParam {
+    upstairs_id: TypedUuid<UpstairsKind>,
+    downstairs_id: TypedUuid<DownstairsKind>,
+}
+
+/// An Upstairs will update this endpoint if a Downstairs client task is
+/// requested to stop
+#[endpoint {
+     method = POST,
+     path = "/crucible/0/upstairs/{upstairs_id}/downstairs/{downstairs_id}/stop-request",
+ }]
+async fn cpapi_downstairs_client_stop_request(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    path_params: Path<UpstairsDownstairsPathParam>,
+    downstairs_client_stop_request: TypedBody<DownstairsClientStopRequest>,
+) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+    let apictx = rqctx.context();
+    let nexus = &apictx.nexus;
+    let path = path_params.into_inner();
+
+    let handler = async {
+        let opctx = crate::context::op_context_for_internal_api(&rqctx).await;
+        nexus
+            .downstairs_client_stop_request_notification(
+                &opctx,
+                path.upstairs_id,
+                path.downstairs_id,
+                downstairs_client_stop_request.into_inner(),
+            )
+            .await?;
+        Ok(HttpResponseUpdatedNoContent())
+    };
+    apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
+/// An Upstairs will update this endpoint if a Downstairs client task stops for
+/// any reason (not just after being requested to)
+#[endpoint {
+     method = POST,
+     path = "/crucible/0/upstairs/{upstairs_id}/downstairs/{downstairs_id}/stopped",
+ }]
+async fn cpapi_downstairs_client_stopped(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    path_params: Path<UpstairsDownstairsPathParam>,
+    downstairs_client_stopped: TypedBody<DownstairsClientStopped>,
+) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+    let apictx = rqctx.context();
+    let nexus = &apictx.nexus;
+    let path = path_params.into_inner();
+
+    let handler = async {
+        let opctx = crate::context::op_context_for_internal_api(&rqctx).await;
+        nexus
+            .downstairs_client_stopped_notification(
+                &opctx,
+                path.upstairs_id,
+                path.downstairs_id,
+                downstairs_client_stopped.into_inner(),
+            )
+            .await?;
+        Ok(HttpResponseUpdatedNoContent())
+    };
+    apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
 // Sagas
 
 /// List sagas
diff --git a/nexus/tests/integration_tests/volume_management.rs b/nexus/tests/integration_tests/volume_management.rs
index 34f037ee8c..289446fe85 100644
--- a/nexus/tests/integration_tests/volume_management.rs
+++ b/nexus/tests/integration_tests/volume_management.rs
@@ -5,6 +5,7 @@
 //! Tests that Nexus properly manages and cleans up Crucible resources
 //! associated with Volumes
 
+use chrono::Utc;
 use dropshot::test_util::ClientTestContext;
 use http::method::Method;
 use http::StatusCode;
@@ -24,6 +25,13 @@ use omicron_common::api::external::ByteCount;
 use omicron_common::api::external::Disk;
 use omicron_common::api::external::IdentityMetadataCreateParams;
 use omicron_common::api::external::Name;
+use omicron_common::api::internal;
+use omicron_uuid_kinds::DownstairsKind;
+use omicron_uuid_kinds::DownstairsRegionKind;
+use omicron_uuid_kinds::TypedUuid;
+use omicron_uuid_kinds::UpstairsKind;
+use omicron_uuid_kinds::UpstairsRepairKind;
+use omicron_uuid_kinds::UpstairsSessionKind;
 use rand::prelude::SliceRandom;
 use rand::{rngs::StdRng, SeedableRng};
 use sled_agent_client::types::{CrucibleOpts, VolumeConstructionRequest};
@@ -2552,3 +2560,763 @@ async fn test_volume_hard_delete_idempotent(
     datastore.volume_hard_delete(volume_id).await.unwrap();
     datastore.volume_hard_delete(volume_id).await.unwrap();
 }
+
+// internal API related tests
+
+/// Test that an Upstairs can reissue live repair notifications
+#[nexus_test]
+async fn test_upstairs_repair_notify_idempotent(
+    cptestctx: &ControlPlaneTestContext,
+) {
+    let int_client = &cptestctx.internal_client;
+
+    let upstairs_id: TypedUuid<UpstairsKind> = TypedUuid::new_v4();
+    let session_id: TypedUuid<UpstairsSessionKind> = TypedUuid::new_v4();
+    let repair_id: TypedUuid<UpstairsRepairKind> = TypedUuid::new_v4();
+    let region_id: TypedUuid<DownstairsRegionKind> = TypedUuid::new_v4();
+
+    // Send the same start request.
+    let notify_url = format!("/crucible/0/upstairs/{upstairs_id}/repair-start");
+
+    let request = internal::nexus::RepairStartInfo {
+        time: Utc::now(),
+        session_id,
+        repair_id,
+        repair_type: internal::nexus::UpstairsRepairType::Live,
+        repairs: vec![internal::nexus::DownstairsUnderRepair {
+            region_uuid: region_id,
+            target_addr: "[fd00:1122:3344:101::8]:12345".parse().unwrap(),
+        }],
+    };
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_url,
+            Some(request.clone()),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_url,
+            Some(request),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    // Send the same finish request.
+    let notify_url =
+        format!("/crucible/0/upstairs/{upstairs_id}/repair-finish");
+
+    let request = internal::nexus::RepairFinishInfo {
+        time: Utc::now(),
+        session_id,
+        repair_id,
+        repair_type: internal::nexus::UpstairsRepairType::Live,
+        repairs: vec![internal::nexus::DownstairsUnderRepair {
+            region_uuid: region_id,
+            target_addr: "[fd00:1122:3344:101::8]:12345".parse().unwrap(),
+        }],
+        aborted: false,
+    };
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_url,
+            Some(request.clone()),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_url,
+            Some(request),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+}
+
+/// Test that an Upstairs cannot issue different finish statuses for the same
+/// repair.
+#[nexus_test]
+async fn test_upstairs_repair_notify_different_finish_status(
+    cptestctx: &ControlPlaneTestContext,
+) {
+    let int_client = &cptestctx.internal_client;
+
+    let upstairs_id: TypedUuid<UpstairsKind> = TypedUuid::new_v4();
+    let session_id: TypedUuid<UpstairsSessionKind> = TypedUuid::new_v4();
+    let repair_id: TypedUuid<UpstairsRepairKind> = TypedUuid::new_v4();
+    let region_id: TypedUuid<DownstairsRegionKind> = TypedUuid::new_v4();
+
+    let notify_url =
+        format!("/crucible/0/upstairs/{upstairs_id}/repair-finish");
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_url,
+            Some(internal::nexus::RepairFinishInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+                aborted: false, // live repair was ok
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_url,
+            Some(internal::nexus::RepairFinishInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+                aborted: true, // live repair failed?
+            }),
+            StatusCode::CONFLICT,
+        )
+        .await
+        .unwrap_err();
+}
+
+/// Test that the same Upstairs can rerun a repair again.
+#[nexus_test]
+async fn test_upstairs_repair_same_upstairs_retry(
+    cptestctx: &ControlPlaneTestContext,
+) {
+    let int_client = &cptestctx.internal_client;
+
+    let upstairs_id: TypedUuid<UpstairsKind> = TypedUuid::new_v4();
+    let session_id: TypedUuid<UpstairsSessionKind> = TypedUuid::new_v4();
+    let repair_id: TypedUuid<UpstairsRepairKind> = TypedUuid::new_v4();
+    let region_id: TypedUuid<DownstairsRegionKind> = TypedUuid::new_v4();
+
+    // Simulate one failed repair
+
+    let notify_start_url =
+        format!("/crucible/0/upstairs/{upstairs_id}/repair-start");
+    let notify_finish_url =
+        format!("/crucible/0/upstairs/{upstairs_id}/repair-finish");
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_start_url,
+            Some(internal::nexus::RepairStartInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_finish_url,
+            Some(internal::nexus::RepairFinishInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+                aborted: true,
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    // Simulate the same Upstairs restarting the repair, which passes this time
+
+    let repair_id: TypedUuid<UpstairsRepairKind> = TypedUuid::new_v4();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_start_url,
+            Some(internal::nexus::RepairStartInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_finish_url,
+            Some(internal::nexus::RepairFinishInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+                aborted: false,
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+}
+
+/// Test that a different Upstairs session can rerun a repair again.
+#[nexus_test]
+async fn test_upstairs_repair_different_upstairs_retry(
+    cptestctx: &ControlPlaneTestContext,
+) {
+    let int_client = &cptestctx.internal_client;
+
+    let upstairs_id: TypedUuid<UpstairsKind> = TypedUuid::new_v4();
+    let session_id: TypedUuid<UpstairsSessionKind> = TypedUuid::new_v4();
+    let repair_id: TypedUuid<UpstairsRepairKind> = TypedUuid::new_v4();
+    let region_id: TypedUuid<DownstairsRegionKind> = TypedUuid::new_v4();
+
+    // Simulate one failed repair by one Upstairs
+
+    let notify_start_url =
+        format!("/crucible/0/upstairs/{upstairs_id}/repair-start");
+    let notify_finish_url =
+        format!("/crucible/0/upstairs/{upstairs_id}/repair-finish");
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_start_url,
+            Some(internal::nexus::RepairStartInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_finish_url,
+            Some(internal::nexus::RepairFinishInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+                aborted: true,
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    // Simulate a different Upstairs session restarting the repair, which passes this time
+
+    let session_id: TypedUuid<UpstairsSessionKind> = TypedUuid::new_v4();
+    let repair_id: TypedUuid<UpstairsRepairKind> = TypedUuid::new_v4();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_start_url,
+            Some(internal::nexus::RepairStartInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_finish_url,
+            Some(internal::nexus::RepairFinishInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+                aborted: false,
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+}
+
+/// Test that a different Upstairs session can rerun an interrupted repair
+#[nexus_test]
+async fn test_upstairs_repair_different_upstairs_retry_interrupted(
+    cptestctx: &ControlPlaneTestContext,
+) {
+    let int_client = &cptestctx.internal_client;
+
+    let upstairs_id: TypedUuid<UpstairsKind> = TypedUuid::new_v4();
+    let session_id: TypedUuid<UpstairsSessionKind> = TypedUuid::new_v4();
+    let repair_id: TypedUuid<UpstairsRepairKind> = TypedUuid::new_v4();
+    let region_id: TypedUuid<DownstairsRegionKind> = TypedUuid::new_v4();
+
+    // Simulate one failed repair by one Upstairs, which was interrupted (which
+    // leads to no finish message).
+
+    let notify_start_url =
+        format!("/crucible/0/upstairs/{upstairs_id}/repair-start");
+    let notify_finish_url =
+        format!("/crucible/0/upstairs/{upstairs_id}/repair-finish");
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_start_url,
+            Some(internal::nexus::RepairStartInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    // Simulate a different Upstairs session restarting the interrupted repair,
+    // which passes this time
+
+    let session_id: TypedUuid<UpstairsSessionKind> = TypedUuid::new_v4();
+    let repair_id: TypedUuid<UpstairsRepairKind> = TypedUuid::new_v4();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_start_url,
+            Some(internal::nexus::RepairStartInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_finish_url,
+            Some(internal::nexus::RepairFinishInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+                aborted: false,
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+}
+
+/// Test that the same repair ID cannot be used for different repair types
+#[nexus_test]
+async fn test_upstairs_repair_repair_id_and_type_conflict(
+    cptestctx: &ControlPlaneTestContext,
+) {
+    let int_client = &cptestctx.internal_client;
+
+    let upstairs_id: TypedUuid<UpstairsKind> = TypedUuid::new_v4();
+    let session_id: TypedUuid<UpstairsSessionKind> = TypedUuid::new_v4();
+    let repair_id: TypedUuid<UpstairsRepairKind> = TypedUuid::new_v4();
+    let region_id: TypedUuid<DownstairsRegionKind> = TypedUuid::new_v4();
+
+    let notify_start_url =
+        format!("/crucible/0/upstairs/{upstairs_id}/repair-start");
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_start_url,
+            Some(internal::nexus::RepairStartInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_start_url,
+            Some(internal::nexus::RepairStartInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type:
+                    internal::nexus::UpstairsRepairType::Reconciliation,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+            }),
+            StatusCode::CONFLICT,
+        )
+        .await
+        .unwrap_err();
+}
+
+/// Test that an Upstairs can submit progress for a repair
+#[nexus_test]
+async fn test_upstairs_repair_submit_progress(
+    cptestctx: &ControlPlaneTestContext,
+) {
+    let int_client = &cptestctx.internal_client;
+
+    let upstairs_id: TypedUuid<UpstairsKind> = TypedUuid::new_v4();
+    let session_id: TypedUuid<UpstairsSessionKind> = TypedUuid::new_v4();
+    let repair_id: TypedUuid<UpstairsRepairKind> = TypedUuid::new_v4();
+    let region_id: TypedUuid<DownstairsRegionKind> = TypedUuid::new_v4();
+
+    // A repair must be started before progress can be submitted
+
+    let notify_start_url =
+        format!("/crucible/0/upstairs/{upstairs_id}/repair-start");
+
+    int_client
+        .make_request(
+            Method::POST,
+            &notify_start_url,
+            Some(internal::nexus::RepairStartInfo {
+                time: Utc::now(),
+                session_id,
+                repair_id,
+                repair_type: internal::nexus::UpstairsRepairType::Live,
+                repairs: vec![internal::nexus::DownstairsUnderRepair {
+                    region_uuid: region_id,
+                    target_addr: "[fd00:1122:3344:101::8]:12345"
+                        .parse()
+                        .unwrap(),
+                }],
+            }),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    let progress_url = format!(
+        "/crucible/0/upstairs/{upstairs_id}/repair/{repair_id}/progress"
+    );
+
+    for i in 0..100 {
+        int_client
+            .make_request(
+                Method::POST,
+                &progress_url,
+                Some(internal::nexus::RepairProgress {
+                    time: Utc::now(),
+                    current_item: i,
+                    total_items: 100,
+                }),
+                StatusCode::NO_CONTENT,
+            )
+            .await
+            .unwrap();
+    }
+}
+
+/// Test that an Upstairs can't submit progress unless a repair was started
+#[nexus_test]
+async fn test_upstairs_repair_reject_submit_progress_when_no_repair(
+    cptestctx: &ControlPlaneTestContext,
+) {
+    let int_client = &cptestctx.internal_client;
+
+    let upstairs_id: TypedUuid<UpstairsKind> = TypedUuid::new_v4();
+    let repair_id: TypedUuid<UpstairsRepairKind> = TypedUuid::new_v4();
+
+    let progress_url = format!(
+        "/crucible/0/upstairs/{upstairs_id}/repair/{repair_id}/progress"
+    );
+
+    int_client
+        .make_request(
+            Method::POST,
+            &progress_url,
+            Some(internal::nexus::RepairProgress {
+                time: Utc::now(),
+                current_item: 10,
+                total_items: 100,
+            }),
+            StatusCode::NOT_FOUND,
+        )
+        .await
+        .unwrap_err();
+}
+
+/// Test that an Upstairs can notify Nexus when a Downstairs client task is
+/// requested to stop
+#[nexus_test]
+async fn test_upstairs_notify_downstairs_client_stop_request(
+    cptestctx: &ControlPlaneTestContext,
+) {
+    let int_client = &cptestctx.internal_client;
+
+    let upstairs_id: TypedUuid<UpstairsKind> = TypedUuid::new_v4();
+    let downstairs_id: TypedUuid<DownstairsKind> = TypedUuid::new_v4();
+
+    let stop_request_url = format!(
+        "/crucible/0/upstairs/{upstairs_id}/downstairs/{downstairs_id}/stop-request"
+    );
+
+    // Make sure an Upstairs can re-send the notification
+
+    let request = internal::nexus::DownstairsClientStopRequest {
+        time: Utc::now(),
+        reason:
+            internal::nexus::DownstairsClientStopRequestReason::TooManyOutstandingJobs,
+    };
+
+    int_client
+        .make_request(
+            Method::POST,
+            &stop_request_url,
+            Some(request.clone()),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &stop_request_url,
+            Some(request),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    // The client can be requested to stop for the same reason a different time
+
+    let request = internal::nexus::DownstairsClientStopRequest {
+        time: Utc::now(),
+        reason:
+            internal::nexus::DownstairsClientStopRequestReason::TooManyOutstandingJobs,
+    };
+
+    int_client
+        .make_request(
+            Method::POST,
+            &stop_request_url,
+            Some(request),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    // The client can also be requested to stop for a different reason
+
+    let request = internal::nexus::DownstairsClientStopRequest {
+        time: Utc::now(),
+        reason: internal::nexus::DownstairsClientStopRequestReason::IOError,
+    };
+
+    int_client
+        .make_request(
+            Method::POST,
+            &stop_request_url,
+            Some(request),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+}
+
+/// Test that an Upstairs can notify Nexus when a Downstairs client task stops
+#[nexus_test]
+async fn test_upstairs_notify_downstairs_client_stops(
+    cptestctx: &ControlPlaneTestContext,
+) {
+    let int_client = &cptestctx.internal_client;
+
+    let upstairs_id: TypedUuid<UpstairsKind> = TypedUuid::new_v4();
+    let downstairs_id: TypedUuid<DownstairsKind> = TypedUuid::new_v4();
+
+    let stopped_url = format!(
+        "/crucible/0/upstairs/{upstairs_id}/downstairs/{downstairs_id}/stopped"
+    );
+
+    // Make sure an Upstairs can re-send the notification
+
+    let request = internal::nexus::DownstairsClientStopped {
+        time: Utc::now(),
+        reason: internal::nexus::DownstairsClientStoppedReason::ReadFailed,
+    };
+
+    int_client
+        .make_request(
+            Method::POST,
+            &stopped_url,
+            Some(request.clone()),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    int_client
+        .make_request(
+            Method::POST,
+            &stopped_url,
+            Some(request),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    // The client can stop for the same reason a different time
+
+    let request = internal::nexus::DownstairsClientStopped {
+        time: Utc::now(),
+        reason: internal::nexus::DownstairsClientStoppedReason::ReadFailed,
+    };
+
+    int_client
+        .make_request(
+            Method::POST,
+            &stopped_url,
+            Some(request),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+
+    // The client can also stop for a different reason
+
+    let request = internal::nexus::DownstairsClientStopped {
+        time: Utc::now(),
+        reason: internal::nexus::DownstairsClientStoppedReason::Timeout,
+    };
+
+    int_client
+        .make_request(
+            Method::POST,
+            &stopped_url,
+            Some(request),
+            StatusCode::NO_CONTENT,
+        )
+        .await
+        .unwrap();
+}
diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json
index 09b5f1e5ab..679597c453 100644
--- a/openapi/nexus-internal.json
+++ b/openapi/nexus-internal.json
@@ -125,6 +125,217 @@
         }
       }
     },
+    "/crucible/0/upstairs/{upstairs_id}/downstairs/{downstairs_id}/stop-request": {
+      "post": {
+        "summary": "An Upstairs will update this endpoint if a Downstairs client task is",
+        "description": "requested to stop",
+        "operationId": "cpapi_downstairs_client_stop_request",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "downstairs_id",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/TypedUuidForDownstairsKind"
+            }
+          },
+          {
+            "in": "path",
+            "name": "upstairs_id",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/TypedUuidForUpstairsKind"
+            }
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/DownstairsClientStopRequest"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "204": {
+            "description": "resource updated"
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
+    "/crucible/0/upstairs/{upstairs_id}/downstairs/{downstairs_id}/stopped": {
+      "post": {
+        "summary": "An Upstairs will update this endpoint if a Downstairs client task stops for",
+        "description": "any reason (not just after being requested to)",
+        "operationId": "cpapi_downstairs_client_stopped",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "downstairs_id",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/TypedUuidForDownstairsKind"
+            }
+          },
+          {
+            "in": "path",
+            "name": "upstairs_id",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/TypedUuidForUpstairsKind"
+            }
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/DownstairsClientStopped"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "204": {
+            "description": "resource updated"
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
+    "/crucible/0/upstairs/{upstairs_id}/repair/{repair_id}/progress": {
+      "post": {
+        "summary": "An Upstairs will update this endpoint with the progress of a repair",
+        "operationId": "cpapi_upstairs_repair_progress",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "repair_id",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/TypedUuidForUpstairsRepairKind"
+            }
+          },
+          {
+            "in": "path",
+            "name": "upstairs_id",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/TypedUuidForUpstairsKind"
+            }
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/RepairProgress"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "204": {
+            "description": "resource updated"
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
+    "/crucible/0/upstairs/{upstairs_id}/repair-finish": {
+      "post": {
+        "summary": "An Upstairs will notify this endpoint when a repair finishes.",
+        "operationId": "cpapi_upstairs_repair_finish",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "upstairs_id",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/TypedUuidForUpstairsKind"
+            }
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/RepairFinishInfo"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "204": {
+            "description": "resource updated"
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
+    "/crucible/0/upstairs/{upstairs_id}/repair-start": {
+      "post": {
+        "summary": "An Upstairs will notify this endpoint when a repair starts",
+        "operationId": "cpapi_upstairs_repair_start",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "upstairs_id",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/TypedUuidForUpstairsKind"
+            }
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/RepairStartInfo"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "204": {
+            "description": "resource updated"
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/deployment/blueprints/all": {
       "get": {
         "summary": "Lists blueprints",
@@ -3649,6 +3860,81 @@
           }
         ]
       },
+      "DownstairsClientStopRequest": {
+        "type": "object",
+        "properties": {
+          "reason": {
+            "$ref": "#/components/schemas/DownstairsClientStopRequestReason"
+          },
+          "time": {
+            "type": "string",
+            "format": "date-time"
+          }
+        },
+        "required": [
+          "reason",
+          "time"
+        ]
+      },
+      "DownstairsClientStopRequestReason": {
+        "type": "string",
+        "enum": [
+          "replacing",
+          "disabled",
+          "failed_reconcile",
+          "i_o_error",
+          "bad_negotiation_order",
+          "incompatible",
+          "failed_live_repair",
+          "too_many_outstanding_jobs",
+          "deactivated"
+        ]
+      },
+      "DownstairsClientStopped": {
+        "type": "object",
+        "properties": {
+          "reason": {
+            "$ref": "#/components/schemas/DownstairsClientStoppedReason"
+          },
+          "time": {
+            "type": "string",
+            "format": "date-time"
+          }
+        },
+        "required": [
+          "reason",
+          "time"
+        ]
+      },
+      "DownstairsClientStoppedReason": {
+        "type": "string",
+        "enum": [
+          "connection_timeout",
+          "connection_failed",
+          "timeout",
+          "write_failed",
+          "read_failed",
+          "requested_stop",
+          "finished",
+          "queue_closed",
+          "receive_task_cancelled"
+        ]
+      },
+      "DownstairsUnderRepair": {
+        "type": "object",
+        "properties": {
+          "region_uuid": {
+            "$ref": "#/components/schemas/TypedUuidForDownstairsRegionKind"
+          },
+          "target_addr": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "region_uuid",
+          "target_addr"
+        ]
+      },
       "Duration": {
         "type": "object",
         "properties": {
@@ -6102,6 +6388,94 @@
           "user_password_hash"
         ]
       },
+      "RepairFinishInfo": {
+        "type": "object",
+        "properties": {
+          "aborted": {
+            "type": "boolean"
+          },
+          "repair_id": {
+            "$ref": "#/components/schemas/TypedUuidForUpstairsRepairKind"
+          },
+          "repair_type": {
+            "$ref": "#/components/schemas/UpstairsRepairType"
+          },
+          "repairs": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/DownstairsUnderRepair"
+            }
+          },
+          "session_id": {
+            "$ref": "#/components/schemas/TypedUuidForUpstairsSessionKind"
+          },
+          "time": {
+            "type": "string",
+            "format": "date-time"
+          }
+        },
+        "required": [
+          "aborted",
+          "repair_id",
+          "repair_type",
+          "repairs",
+          "session_id",
+          "time"
+        ]
+      },
+      "RepairProgress": {
+        "type": "object",
+        "properties": {
+          "current_item": {
+            "type": "integer",
+            "format": "int64"
+          },
+          "time": {
+            "type": "string",
+            "format": "date-time"
+          },
+          "total_items": {
+            "type": "integer",
+            "format": "int64"
+          }
+        },
+        "required": [
+          "current_item",
+          "time",
+          "total_items"
+        ]
+      },
+      "RepairStartInfo": {
+        "type": "object",
+        "properties": {
+          "repair_id": {
+            "$ref": "#/components/schemas/TypedUuidForUpstairsRepairKind"
+          },
+          "repair_type": {
+            "$ref": "#/components/schemas/UpstairsRepairType"
+          },
+          "repairs": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/DownstairsUnderRepair"
+            }
+          },
+          "session_id": {
+            "$ref": "#/components/schemas/TypedUuidForUpstairsSessionKind"
+          },
+          "time": {
+            "type": "string",
+            "format": "date-time"
+          }
+        },
+        "required": [
+          "repair_id",
+          "repair_type",
+          "repairs",
+          "session_id",
+          "time"
+        ]
+      },
       "RouteConfig": {
         "type": "object",
         "properties": {
@@ -6979,6 +7353,18 @@
       "SwitchPutResponse": {
         "type": "object"
       },
+      "TypedUuidForDownstairsRegionKind": {
+        "type": "string",
+        "format": "uuid"
+      },
+      "TypedUuidForUpstairsRepairKind": {
+        "type": "string",
+        "format": "uuid"
+      },
+      "TypedUuidForUpstairsSessionKind": {
+        "type": "string",
+        "format": "uuid"
+      },
       "UninitializedSled": {
         "description": "A sled that has not been added to an initialized rack yet",
         "type": "object",
@@ -7039,6 +7425,13 @@
           "items"
         ]
       },
+      "UpstairsRepairType": {
+        "type": "string",
+        "enum": [
+          "live",
+          "reconciliation"
+        ]
+      },
       "UserId": {
         "title": "A name unique within the parent collection",
         "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.",
@@ -7125,6 +7518,14 @@
         "type": "string",
         "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"
       },
+      "TypedUuidForDownstairsKind": {
+        "type": "string",
+        "format": "uuid"
+      },
+      "TypedUuidForUpstairsKind": {
+        "type": "string",
+        "format": "uuid"
+      },
       "IdSortMode": {
         "description": "Supported set of sort modes for scanning by id only.\n\nCurrently, we only support scanning in ascending order.",
         "oneOf": [
diff --git a/schema/crdb/43.0.0/up01.sql b/schema/crdb/43.0.0/up01.sql
new file mode 100644
index 0000000000..b94f43eda7
--- /dev/null
+++ b/schema/crdb/43.0.0/up01.sql
@@ -0,0 +1,5 @@
+CREATE TYPE IF NOT EXISTS omicron.public.upstairs_repair_notification_type AS ENUM (
+  'started',
+  'succeeded',
+  'failed'
+);
diff --git a/schema/crdb/43.0.0/up02.sql b/schema/crdb/43.0.0/up02.sql
new file mode 100644
index 0000000000..47c5f7f03a
--- /dev/null
+++ b/schema/crdb/43.0.0/up02.sql
@@ -0,0 +1,4 @@
+CREATE TYPE IF NOT EXISTS omicron.public.upstairs_repair_type AS ENUM (
+  'live',
+  'reconciliation'
+);
diff --git a/schema/crdb/43.0.0/up03.sql b/schema/crdb/43.0.0/up03.sql
new file mode 100644
index 0000000000..a33c83c1ab
--- /dev/null
+++ b/schema/crdb/43.0.0/up03.sql
@@ -0,0 +1,21 @@
+CREATE TABLE IF NOT EXISTS omicron.public.upstairs_repair_notification (
+    time TIMESTAMPTZ NOT NULL,
+
+    repair_id UUID NOT NULL,
+    repair_type omicron.public.upstairs_repair_type NOT NULL,
+
+    upstairs_id UUID NOT NULL,
+    session_id UUID NOT NULL,
+
+    region_id UUID NOT NULL,
+    target_ip INET NOT NULL,
+    target_port INT4 CHECK (target_port BETWEEN 0 AND 65535) NOT NULL,
+
+    notification_type omicron.public.upstairs_repair_notification_type NOT NULL,
+
+    /*
+     * A repair is uniquely identified by the four UUIDs here, and a
+     * notification is uniquely identified by its type.
+     */
+    PRIMARY KEY (repair_id, upstairs_id, session_id, region_id, notification_type)
+);
diff --git a/schema/crdb/43.0.0/up04.sql b/schema/crdb/43.0.0/up04.sql
new file mode 100644
index 0000000000..ed51c15a1e
--- /dev/null
+++ b/schema/crdb/43.0.0/up04.sql
@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS omicron.public.upstairs_repair_progress (
+    repair_id UUID NOT NULL,
+    time TIMESTAMPTZ NOT NULL,
+    current_item INT8 NOT NULL,
+    total_items INT8 NOT NULL,
+
+    PRIMARY KEY (repair_id, time, current_item, total_items)
+);
diff --git a/schema/crdb/43.0.0/up05.sql b/schema/crdb/43.0.0/up05.sql
new file mode 100644
index 0000000000..16fb91d410
--- /dev/null
+++ b/schema/crdb/43.0.0/up05.sql
@@ -0,0 +1,11 @@
+CREATE TYPE IF NOT EXISTS omicron.public.downstairs_client_stop_request_reason_type AS ENUM (
+  'replacing',
+  'disabled',
+  'failed_reconcile',
+  'io_error',
+  'bad_negotiation_order',
+  'incompatible',
+  'failed_live_repair',
+  'too_many_outstanding_jobs',
+  'deactivated'
+);
diff --git a/schema/crdb/43.0.0/up06.sql b/schema/crdb/43.0.0/up06.sql
new file mode 100644
index 0000000000..2cf45ce101
--- /dev/null
+++ b/schema/crdb/43.0.0/up06.sql
@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS omicron.public.downstairs_client_stop_request_notification (
+    time TIMESTAMPTZ NOT NULL,
+    upstairs_id UUID NOT NULL,
+    downstairs_id UUID NOT NULL,
+    reason omicron.public.downstairs_client_stop_request_reason_type NOT NULL,
+
+    PRIMARY KEY (time, upstairs_id, downstairs_id, reason)
+);
diff --git a/schema/crdb/43.0.0/up07.sql b/schema/crdb/43.0.0/up07.sql
new file mode 100644
index 0000000000..cb8903b1d3
--- /dev/null
+++ b/schema/crdb/43.0.0/up07.sql
@@ -0,0 +1,11 @@
+CREATE TYPE IF NOT EXISTS omicron.public.downstairs_client_stopped_reason_type AS ENUM (
+  'connection_timeout',
+  'connection_failed',
+  'timeout',
+  'write_failed',
+  'read_failed',
+  'requested_stop',
+  'finished',
+  'queue_closed',
+  'receive_task_cancelled'
+);
diff --git a/schema/crdb/43.0.0/up08.sql b/schema/crdb/43.0.0/up08.sql
new file mode 100644
index 0000000000..6f898f382c
--- /dev/null
+++ b/schema/crdb/43.0.0/up08.sql
@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS omicron.public.downstairs_client_stopped_notification (
+    time TIMESTAMPTZ NOT NULL,
+    upstairs_id UUID NOT NULL,
+    downstairs_id UUID NOT NULL,
+    reason omicron.public.downstairs_client_stopped_reason_type NOT NULL,
+
+    PRIMARY KEY (time, upstairs_id, downstairs_id, reason)
+);
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 9895464fb5..255cdc8135 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -3552,6 +3552,90 @@ ALTER TABLE omicron.public.external_ip ADD COLUMN IF NOT EXISTS is_probe BOOL NO
 
 ALTER TYPE omicron.public.network_interface_kind ADD VALUE IF NOT EXISTS 'probe';
 
+CREATE TYPE IF NOT EXISTS omicron.public.upstairs_repair_notification_type AS ENUM (
+  'started',
+  'succeeded',
+  'failed'
+);
+
+CREATE TYPE IF NOT EXISTS omicron.public.upstairs_repair_type AS ENUM (
+  'live',
+  'reconciliation'
+);
+
+CREATE TABLE IF NOT EXISTS omicron.public.upstairs_repair_notification (
+    time TIMESTAMPTZ NOT NULL,
+
+    repair_id UUID NOT NULL,
+    repair_type omicron.public.upstairs_repair_type NOT NULL,
+
+    upstairs_id UUID NOT NULL,
+    session_id UUID NOT NULL,
+
+    region_id UUID NOT NULL,
+    target_ip INET NOT NULL,
+    target_port INT4 CHECK (target_port BETWEEN 0 AND 65535) NOT NULL,
+
+    notification_type omicron.public.upstairs_repair_notification_type NOT NULL,
+
+    /*
+     * A repair is uniquely identified by the four UUIDs here, and a
+     * notification is uniquely identified by its type.
+     */
+    PRIMARY KEY (repair_id, upstairs_id, session_id, region_id, notification_type)
+);
+
+CREATE TABLE IF NOT EXISTS omicron.public.upstairs_repair_progress (
+    repair_id UUID NOT NULL,
+    time TIMESTAMPTZ NOT NULL,
+    current_item INT8 NOT NULL,
+    total_items INT8 NOT NULL,
+
+    PRIMARY KEY (repair_id, time, current_item, total_items)
+);
+
+CREATE TYPE IF NOT EXISTS omicron.public.downstairs_client_stop_request_reason_type AS ENUM (
+  'replacing',
+  'disabled',
+  'failed_reconcile',
+  'io_error',
+  'bad_negotiation_order',
+  'incompatible',
+  'failed_live_repair',
+  'too_many_outstanding_jobs',
+  'deactivated'
+);
+
+CREATE TABLE IF NOT EXISTS omicron.public.downstairs_client_stop_request_notification (
+    time TIMESTAMPTZ NOT NULL,
+    upstairs_id UUID NOT NULL,
+    downstairs_id UUID NOT NULL,
+    reason omicron.public.downstairs_client_stop_request_reason_type NOT NULL,
+
+    PRIMARY KEY (time, upstairs_id, downstairs_id, reason)
+);
+
+CREATE TYPE IF NOT EXISTS omicron.public.downstairs_client_stopped_reason_type AS ENUM (
+  'connection_timeout',
+  'connection_failed',
+  'timeout',
+  'write_failed',
+  'read_failed',
+  'requested_stop',
+  'finished',
+  'queue_closed',
+  'receive_task_cancelled'
+);
+
+CREATE TABLE IF NOT EXISTS omicron.public.downstairs_client_stopped_notification (
+    time TIMESTAMPTZ NOT NULL,
+    upstairs_id UUID NOT NULL,
+    downstairs_id UUID NOT NULL,
+    reason omicron.public.downstairs_client_stopped_reason_type NOT NULL,
+
+    PRIMARY KEY (time, upstairs_id, downstairs_id, reason)
+);
+
 /*
  * Metadata for the schema itself. This version number isn't great, as there's
  * nothing to ensure it gets bumped when it should be, but it's a start.
@@ -3586,7 +3670,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '42.0.0', NULL)
+    ( TRUE, NOW(), NOW(), '43.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;
diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs
index 12bc756d68..7018485b59 100644
--- a/uuid-kinds/src/lib.rs
+++ b/uuid-kinds/src/lib.rs
@@ -45,6 +45,11 @@ macro_rules! impl_typed_uuid_kind {
 // Please keep this list in alphabetical order.
 
 impl_typed_uuid_kind! {
+    DownstairsKind => "downstairs",
+    DownstairsRegionKind => "downstairs_region",
     LoopbackAddressKind => "loopback_address",
     TufRepoKind => "tuf_repo",
+    UpstairsKind => "upstairs",
+    UpstairsRepairKind => "upstairs_repair",
+    UpstairsSessionKind => "upstairs_session",
 }

From 75ad4a91cbc89392f548da32b0cb8e34b47d45ed Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Thu, 14 Mar 2024 18:17:11 -0700
Subject: [PATCH 118/157] Removes test_complex from the reconfigurator test
 suite (#5268)

This test has a lot of value, but could benefit from a little more
automation to avoid round-trips for developers.

Me putting up this PR is decidedly "an opinion" though, and I can alter
course here if we want to discuss things more.
---
 .../tests/input/complex.json                  | 8655 -----------------
 .../reconfigurator-cli/tests/test_basic.rs    |   88 -
 2 files changed, 8743 deletions(-)
 delete mode 100644 dev-tools/reconfigurator-cli/tests/input/complex.json

diff --git a/dev-tools/reconfigurator-cli/tests/input/complex.json b/dev-tools/reconfigurator-cli/tests/input/complex.json
deleted file mode 100644
index 67f1881147..0000000000
--- a/dev-tools/reconfigurator-cli/tests/input/complex.json
+++ /dev/null
@@ -1,8655 +0,0 @@
-{
-  "policy": {
-    "sleds": {
-      "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
-        "policy": {
-          "kind": "in_service",
-          "provision_policy": "provisionable"
-        },
-        "state": "active",
-        "zpools": [
-          "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03",
-          "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03",
-          "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b",
-          "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03",
-          "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
-        ],
-        "subnet": {
-          "net": "fd00:1122:3344:103::/64"
-        }
-      },
-      "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
-        "policy": {
-          "kind": "in_service",
-          "provision_policy": "provisionable"
-        },
-        "state": "active",
-        "zpools": [
-          "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03",
-          "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03",
-          "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b",
-          "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
-          "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-        ],
-        "subnet": {
-          "net": "fd00:1122:3344:104::/64"
-        }
-      },
-      "77851655-10a5-4e80-88b0-a001473cab7e": {
-        "policy": {
-          "kind": "in_service",
-          "provision_policy": "provisionable"
-        },
-        "state": "active",
-        "zpools": [
-          "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03",
-          "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03",
-          "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b",
-          "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03",
-          "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
-        ],
-        "subnet": {
-          "net": "fd00:1122:3344:102::/64"
-        }
-      },
-      "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
-        "policy": {
-          "kind": "in_service",
-          "provision_policy": "provisionable"
-        },
-        "state": "active",
-        "zpools": [
-          "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03",
-          "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03",
-          "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b",
-          "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03",
-          "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
-        ],
-        "subnet": {
-          "net": "fd00:1122:3344:101::/64"
-        }
-      }
-    },
-    "service_ip_pool_ranges": [
-      {
-        "first": "198.51.100.20",
-        "last": "198.51.100.29"
-      }
-    ],
-    "target_nexus_zone_count": 3
-  },
-  "collections": [
-    {
-      "id": "3f61b723-cfe7-40ec-b22c-e3e1f35325f9",
-      "errors": [],
-      "time_started": "2024-03-12T18:59:34.196825Z",
-      "time_done": "2024-03-12T18:59:34.397868Z",
-      "collector": "0d459323-e414-4f32-9944-76c7331da622",
-      "baseboards": [
-        {
-          "part_number": "FAKE_SIM_SIDECAR",
-          "serial_number": "SimSidecar0"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g0"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g1"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g2"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g3"
-        }
-      ],
-      "cabooses": [
-        {
-          "board": "SimGimletSp",
-          "git_commit": "ffffffff",
-          "name": "SimGimlet",
-          "version": "0.0.1"
-        },
-        {
-          "board": "SimRot",
-          "git_commit": "eeeeeeee",
-          "name": "SimGimlet",
-          "version": "0.0.1"
-        },
-        {
-          "board": "SimRot",
-          "git_commit": "eeeeeeee",
-          "name": "SimSidecar",
-          "version": "0.0.1"
-        },
-        {
-          "board": "SimSidecarSp",
-          "git_commit": "ffffffff",
-          "name": "SimSidecar",
-          "version": "0.0.1"
-        }
-      ],
-      "rot_pages": [
-        {
-          "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        }
-      ],
-      "sps": [
-        [
-          {
-            "part_number": "FAKE_SIM_SIDECAR",
-            "serial_number": "SimSidecar0"
-          },
-          {
-            "time_collected": "2024-03-12T18:59:34.203615Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "switch",
-            "sp_slot": 1,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g0"
-          },
-          {
-            "time_collected": "2024-03-12T18:59:34.208589Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 0,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g1"
-          },
-          {
-            "time_collected": "2024-03-12T18:59:34.214982Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 1,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g2"
-          },
-          {
-            "time_collected": "2024-03-12T18:59:34.220269Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 2,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g3"
-          },
-          {
-            "time_collected": "2024-03-12T18:59:34.198632Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 3,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ]
-      ],
-      "rots": [
-        [
-          {
-            "part_number": "FAKE_SIM_SIDECAR",
-            "serial_number": "SimSidecar0"
-          },
-          {
-            "time_collected": "2024-03-12T18:59:34.203615Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g0"
-          },
-          {
-            "time_collected": "2024-03-12T18:59:34.208589Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g1"
-          },
-          {
-            "time_collected": "2024-03-12T18:59:34.214982Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g2"
-          },
-          {
-            "time_collected": "2024-03-12T18:59:34.220269Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g3"
-          },
-          {
-            "time_collected": "2024-03-12T18:59:34.198632Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ]
-      ],
-      "cabooses_found": {
-        "SpSlot0": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.204198Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimSidecarSp",
-                "git_commit": "ffffffff",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.209189Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.215708Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.220965Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.199229Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ],
-        "SpSlot1": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.204765Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimSidecarSp",
-                "git_commit": "ffffffff",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.209887Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.216379Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.221612Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.199808Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ],
-        "RotSlotA": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.205525Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.210499Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.217080Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.222264Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.200511Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ],
-        "RotSlotB": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.206173Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.211127Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.217758Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.222982Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.201220Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ]
-      },
-      "rot_pages_found": {
-        "Cmpa": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.206630Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.211561Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.218199Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.223472Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.201705Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ],
-        "CfpaActive": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.207099Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.212036Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.218655Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.223978Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.202177Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ],
-        "CfpaInactive": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.207582Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.212533Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.219102Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.224471Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.202661Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ],
-        "CfpaScratch": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.208087Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.213042Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.219676Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.224951Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T18:59:34.203171Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ]
-      },
-      "sled_agents": {
-        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
-          "time_collected": "2024-03-12T18:59:34.262487Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a452a7f5-b528-40fe-80ff-4e4189e2d52b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b452a7f5-b528-40fe-80ff-4e4189e2d52b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d452a7f5-b528-40fe-80ff-4e4189e2d52b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        },
-        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
-          "time_collected": "2024-03-12T18:59:34.306155Z",
-          "source": "http://[fd00:1122:3344:104::1]:12345",
-          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:104::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        },
-        "77851655-10a5-4e80-88b0-a001473cab7e": {
-          "time_collected": "2024-03-12T18:59:34.349258Z",
-          "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a462a7f6-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b462a7f6-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d462a7f6-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        },
-        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
-          "time_collected": "2024-03-12T18:59:34.392838Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a462a4f4-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b462a4f4-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d462a4f4-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        }
-      },
-      "omicron_zones": {
-        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
-          "time_collected": "2024-03-12T18:59:34.266337Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "0db88baf-911d-47bb-af71-3aebd7c96934",
-                "underlay_address": "fd00:1122:3344:103::4",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "198.51.100.22",
-                  "external_tls": false,
-                  "internal_address": "[fd00:1122:3344:103::4]:12221",
-                  "nic": {
-                    "id": "8847a09c-cabc-429b-bf46-66d5e2e1140c",
-                    "kind": {
-                      "type": "service",
-                      "id": "0db88baf-911d-47bb-af71-3aebd7c96934"
-                    },
-                    "name": "nexus-0db88baf-911d-47bb-af71-3aebd7c96934",
-                    "ip": "172.30.2.5",
-                    "mac": "A8:40:25:FF:CC:DA",
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "213d7dd3-95a4-45a7-99ca-e4899991dab4",
-                "underlay_address": "fd00:1122:3344:103::6",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::6]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
-                "underlay_address": "fd00:1122:3344:3::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  },
-                  "dns_address": "[fd00:1122:3344:3::1]:53",
-                  "gz_address": "fd00:1122:3344:3::2",
-                  "gz_address_index": 2,
-                  "http_address": "[fd00:1122:3344:3::1]:5353"
-                }
-              },
-              {
-                "id": "40c47ead-3be8-4724-a8ef-b66233fc5c65",
-                "underlay_address": "fd00:1122:3344:103::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "425d9d2f-72ea-4f53-80b6-04e002771cdb",
-                "underlay_address": "fd00:1122:3344:103::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
-                "underlay_address": "fd00:1122:3344:103::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b"
-                  }
-                }
-              },
-              {
-                "id": "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
-                "underlay_address": "fd00:1122:3344:103::5",
-                "zone_type": {
-                  "type": "clickhouse",
-                  "address": "[fd00:1122:3344:103::5]:8123",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "7611a549-effc-41b6-b6cb-c03e0c70a31f",
-                "underlay_address": "fd00:1122:3344:103::b",
-                "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:103::b]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
-                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
-                  ]
-                }
-              },
-              {
-                "id": "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
-                "underlay_address": "fd00:1122:3344:103::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:103::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
-                "underlay_address": "fd00:1122:3344:103::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
-          "time_collected": "2024-03-12T18:59:34.311965Z",
-          "source": "http://[fd00:1122:3344:104::1]:12345",
-          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
-                "underlay_address": "fd00:1122:3344:104::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
-                "underlay_address": "fd00:1122:3344:104::4",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:104::4]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "57c712a6-6429-44b7-a9b2-9957cb9054d4",
-                "underlay_address": "fd00:1122:3344:104::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
-                "underlay_address": "fd00:1122:3344:104::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "73916098-788f-48d8-b745-e13fdcfb771b",
-                "underlay_address": "fd00:1122:3344:104::c",
-                "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:104::c]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
-                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
-                  ]
-                }
-              },
-              {
-                "id": "9bb513e3-7505-416c-aa70-b26f1152bbdc",
-                "underlay_address": "fd00:1122:3344:104::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b"
-                  }
-                }
-              },
-              {
-                "id": "9c14c879-137b-4f5b-8068-1292bffc83ed",
-                "underlay_address": "fd00:1122:3344:104::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:104::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "9cf90589-5da1-4b42-b09b-d784e4134637",
-                "underlay_address": "fd00:1122:3344:104::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "198.51.100.23",
-                  "external_tls": false,
-                  "internal_address": "[fd00:1122:3344:104::5]:12221",
-                  "nic": {
-                    "id": "fa76e7e1-dee6-4574-b71f-e07ea1cffd08",
-                    "kind": {
-                      "type": "service",
-                      "id": "9cf90589-5da1-4b42-b09b-d784e4134637"
-                    },
-                    "name": "nexus-9cf90589-5da1-4b42-b09b-d784e4134637",
-                    "ip": "172.30.2.6",
-                    "mac": "A8:40:25:FF:C4:6F",
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "d48b2dfa-c749-47dd-98b6-3f478cd63934",
-                "underlay_address": "fd00:1122:3344:104::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:104::6]:17000"
-                }
-              },
-              {
-                "id": "fd003bee-818a-4c95-aae5-e378ac601522",
-                "underlay_address": "fd00:1122:3344:104::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "77851655-10a5-4e80-88b0-a001473cab7e": {
-          "time_collected": "2024-03-12T18:59:34.356678Z",
-          "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "084cac6e-44b5-4333-a2ff-b0993b534898",
-                "underlay_address": "fd00:1122:3344:102::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "11c527c5-0598-428e-9270-be5b74d3461b",
-                "underlay_address": "fd00:1122:3344:102::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "20be9299-495d-428d-b315-694c2c4b64af",
-                "underlay_address": "fd00:1122:3344:102::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:102::6]:17000"
-                }
-              },
-              {
-                "id": "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
-                "underlay_address": "fd00:1122:3344:102::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "8d452bdd-e8b8-4108-a690-46fc59197db0",
-                "underlay_address": "fd00:1122:3344:102::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "953019c0-8e85-4417-b231-d42820470125",
-                "underlay_address": "fd00:1122:3344:102::5",
-                "zone_type": {
-                  "type": "oximeter",
-                  "address": "[fd00:1122:3344:102::5]:12223"
-                }
-              },
-              {
-                "id": "99c635b9-8e97-414a-8316-8261d2d767d0",
-                "underlay_address": "fd00:1122:3344:102::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
-                "underlay_address": "fd00:1122:3344:2::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "[fd00:1122:3344:2::1]:53",
-                  "gz_address": "fd00:1122:3344:2::2",
-                  "gz_address_index": 1,
-                  "http_address": "[fd00:1122:3344:2::1]:5353"
-                }
-              },
-              {
-                "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-                "underlay_address": "fd00:1122:3344:102::c",
-                "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:102::c]:123",
-                  "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "nic": {
-                    "id": "659f33f7-e7bd-4875-9140-bf8a8b2d964a",
-                    "kind": {
-                      "type": "service",
-                      "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded"
-                    },
-                    "name": "ntp-d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-                    "ip": "172.30.3.6",
-                    "mac": "A8:40:25:FF:B1:99",
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  },
-                  "ntp_servers": [
-                    "time.cloudflare.com"
-                  ],
-                  "snat_cfg": {
-                    "ip": "198.51.100.26",
-                    "first_port": 16384,
-                    "last_port": 32767
-                  }
-                }
-              },
-              {
-                "id": "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
-                "underlay_address": "fd00:1122:3344:102::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b"
-                  }
-                }
-              },
-              {
-                "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-                "underlay_address": "fd00:1122:3344:102::4",
-                "zone_type": {
-                  "type": "external_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "198.51.100.21:53",
-                  "http_address": "[fd00:1122:3344:102::4]:5353",
-                  "nic": {
-                    "id": "d4dd2f7a-8a3b-405e-8444-62f99a754a8a",
-                    "kind": {
-                      "type": "service",
-                      "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802"
-                    },
-                    "name": "external-dns-f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-                    "ip": "172.30.1.6",
-                    "mac": "A8:40:25:FF:F4:5C",
-                    "subnet": "172.30.1.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
-          "time_collected": "2024-03-12T18:59:34.396440Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "0d459323-e414-4f32-9944-76c7331da622",
-                "underlay_address": "fd00:1122:3344:101::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "198.51.100.24",
-                  "external_tls": false,
-                  "internal_address": "[fd00:1122:3344:101::5]:12221",
-                  "nic": {
-                    "id": "514902b8-7a1a-42a3-95a6-d00861833b5c",
-                    "kind": {
-                      "type": "service",
-                      "id": "0d459323-e414-4f32-9944-76c7331da622"
-                    },
-                    "name": "nexus-0d459323-e414-4f32-9944-76c7331da622",
-                    "ip": "172.30.2.7",
-                    "mac": "A8:40:25:FF:DE:AC",
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "2902972b-3ae4-489d-b937-6480352e214a",
-                "underlay_address": "fd00:1122:3344:101::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "2ee832de-af75-47ac-abac-f91304857ca4",
-                "underlay_address": "fd00:1122:3344:101::4",
-                "zone_type": {
-                  "type": "external_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "198.51.100.20:53",
-                  "http_address": "[fd00:1122:3344:101::4]:5353",
-                  "nic": {
-                    "id": "fadff4ab-62d7-466a-9203-04080e4078ed",
-                    "kind": {
-                      "type": "service",
-                      "id": "2ee832de-af75-47ac-abac-f91304857ca4"
-                    },
-                    "name": "external-dns-2ee832de-af75-47ac-abac-f91304857ca4",
-                    "ip": "172.30.1.5",
-                    "mac": "A8:40:25:FF:B4:93",
-                    "subnet": "172.30.1.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "30050a7e-7fc3-4965-b07b-1635212393eb",
-                "underlay_address": "fd00:1122:3344:101::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:101::6]:17000"
-                }
-              },
-              {
-                "id": "8494befe-6df6-42d1-9baa-599ba4228430",
-                "underlay_address": "fd00:1122:3344:101::c",
-                "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:101::c]:123",
-                  "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "nic": {
-                    "id": "23041748-e3a6-4846-930d-5a348c316e91",
-                    "kind": {
-                      "type": "service",
-                      "id": "8494befe-6df6-42d1-9baa-599ba4228430"
-                    },
-                    "name": "ntp-8494befe-6df6-42d1-9baa-599ba4228430",
-                    "ip": "172.30.3.5",
-                    "mac": "A8:40:25:FF:D1:C4",
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  },
-                  "ntp_servers": [
-                    "time.cloudflare.com"
-                  ],
-                  "snat_cfg": {
-                    "ip": "198.51.100.25",
-                    "first_port": 0,
-                    "last_port": 16383
-                  }
-                }
-              },
-              {
-                "id": "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
-                "underlay_address": "fd00:1122:3344:101::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
-                "underlay_address": "fd00:1122:3344:101::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "e668e3f5-5457-4689-97da-b2af537787ac",
-                "underlay_address": "fd00:1122:3344:101::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
-                "underlay_address": "fd00:1122:3344:101::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b"
-                  }
-                }
-              },
-              {
-                "id": "f8c79ba4-194a-4cfe-8090-5315e297b780",
-                "underlay_address": "fd00:1122:3344:1::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "[fd00:1122:3344:1::1]:53",
-                  "gz_address": "fd00:1122:3344:1::2",
-                  "gz_address_index": 0,
-                  "http_address": "[fd00:1122:3344:1::1]:5353"
-                }
-              },
-              {
-                "id": "fa5e0176-b4a8-4598-aa31-40fd562d246c",
-                "underlay_address": "fd00:1122:3344:101::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              }
-            ]
-          }
-        }
-      }
-    },
-    {
-      "id": "37c212f1-bfdd-4cfd-a17c-e65bd93bea5f",
-      "errors": [],
-      "time_started": "2024-03-12T19:07:28.237276Z",
-      "time_done": "2024-03-12T19:07:28.472922Z",
-      "collector": "0db88baf-911d-47bb-af71-3aebd7c96934",
-      "baseboards": [
-        {
-          "part_number": "FAKE_SIM_SIDECAR",
-          "serial_number": "SimSidecar0"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g0"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g1"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g2"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g3"
-        }
-      ],
-      "cabooses": [
-        {
-          "board": "SimGimletSp",
-          "git_commit": "ffffffff",
-          "name": "SimGimlet",
-          "version": "0.0.1"
-        },
-        {
-          "board": "SimRot",
-          "git_commit": "eeeeeeee",
-          "name": "SimGimlet",
-          "version": "0.0.1"
-        },
-        {
-          "board": "SimRot",
-          "git_commit": "eeeeeeee",
-          "name": "SimSidecar",
-          "version": "0.0.1"
-        },
-        {
-          "board": "SimSidecarSp",
-          "git_commit": "ffffffff",
-          "name": "SimSidecar",
-          "version": "0.0.1"
-        }
-      ],
-      "rot_pages": [
-        {
-          "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        }
-      ],
-      "sps": [
-        [
-          {
-            "part_number": "FAKE_SIM_SIDECAR",
-            "serial_number": "SimSidecar0"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.244597Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "switch",
-            "sp_slot": 1,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g0"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.251644Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 0,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g1"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.257993Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 1,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g2"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.263543Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 2,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g3"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.239320Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 3,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ]
-      ],
-      "rots": [
-        [
-          {
-            "part_number": "FAKE_SIM_SIDECAR",
-            "serial_number": "SimSidecar0"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.244597Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g0"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.251644Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g1"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.257993Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g2"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.263543Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g3"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.239320Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ]
-      ],
-      "cabooses_found": {
-        "SpSlot0": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.245320Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimSidecarSp",
-                "git_commit": "ffffffff",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.252625Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.258749Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.273088Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.240046Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ],
-        "SpSlot1": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.245980Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimSidecarSp",
-                "git_commit": "ffffffff",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.253364Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.259496Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.273807Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.240736Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ],
-        "RotSlotA": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.246605Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.254069Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.260195Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.274568Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.241356Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ],
-        "RotSlotB": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.247577Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.254910Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.260882Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.275221Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.241947Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ]
-      },
-      "rot_pages_found": {
-        "Cmpa": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.248572Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.255711Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.261401Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.275673Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.242430Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ],
-        "CfpaActive": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.249231Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.256314Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.261975Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.276189Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.242948Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ],
-        "CfpaInactive": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.249902Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.256959Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.262485Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.276907Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.243514Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ],
-        "CfpaScratch": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.250611Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.257483Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.262983Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.277386Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.244032Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ]
-      },
-      "sled_agents": {
-        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
-          "time_collected": "2024-03-12T19:07:28.330250Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a452a7f5-b528-40fe-80ff-4e4189e2d52b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b452a7f5-b528-40fe-80ff-4e4189e2d52b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d452a7f5-b528-40fe-80ff-4e4189e2d52b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        },
-        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
-          "time_collected": "2024-03-12T19:07:28.371508Z",
-          "source": "http://[fd00:1122:3344:104::1]:12345",
-          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:104::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        },
-        "77851655-10a5-4e80-88b0-a001473cab7e": {
-          "time_collected": "2024-03-12T19:07:28.409958Z",
-          "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a462a7f6-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b462a7f6-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d462a7f6-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        },
-        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
-          "time_collected": "2024-03-12T19:07:28.461671Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a462a4f4-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b462a4f4-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d462a4f4-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        }
-      },
-      "omicron_zones": {
-        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
-          "time_collected": "2024-03-12T19:07:28.340118Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "0db88baf-911d-47bb-af71-3aebd7c96934",
-                "underlay_address": "fd00:1122:3344:103::4",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "198.51.100.22",
-                  "external_tls": false,
-                  "internal_address": "[fd00:1122:3344:103::4]:12221",
-                  "nic": {
-                    "id": "8847a09c-cabc-429b-bf46-66d5e2e1140c",
-                    "kind": {
-                      "type": "service",
-                      "id": "0db88baf-911d-47bb-af71-3aebd7c96934"
-                    },
-                    "name": "nexus-0db88baf-911d-47bb-af71-3aebd7c96934",
-                    "ip": "172.30.2.5",
-                    "mac": "A8:40:25:FF:CC:DA",
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "213d7dd3-95a4-45a7-99ca-e4899991dab4",
-                "underlay_address": "fd00:1122:3344:103::6",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::6]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
-                "underlay_address": "fd00:1122:3344:3::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  },
-                  "dns_address": "[fd00:1122:3344:3::1]:53",
-                  "gz_address": "fd00:1122:3344:3::2",
-                  "gz_address_index": 2,
-                  "http_address": "[fd00:1122:3344:3::1]:5353"
-                }
-              },
-              {
-                "id": "40c47ead-3be8-4724-a8ef-b66233fc5c65",
-                "underlay_address": "fd00:1122:3344:103::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "425d9d2f-72ea-4f53-80b6-04e002771cdb",
-                "underlay_address": "fd00:1122:3344:103::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
-                "underlay_address": "fd00:1122:3344:103::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b"
-                  }
-                }
-              },
-              {
-                "id": "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
-                "underlay_address": "fd00:1122:3344:103::5",
-                "zone_type": {
-                  "type": "clickhouse",
-                  "address": "[fd00:1122:3344:103::5]:8123",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "7611a549-effc-41b6-b6cb-c03e0c70a31f",
-                "underlay_address": "fd00:1122:3344:103::b",
-                "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:103::b]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
-                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
-                  ]
-                }
-              },
-              {
-                "id": "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
-                "underlay_address": "fd00:1122:3344:103::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:103::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
-                "underlay_address": "fd00:1122:3344:103::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
-          "time_collected": "2024-03-12T19:07:28.381330Z",
-          "source": "http://[fd00:1122:3344:104::1]:12345",
-          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
-                "underlay_address": "fd00:1122:3344:104::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
-                "underlay_address": "fd00:1122:3344:104::4",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:104::4]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "57c712a6-6429-44b7-a9b2-9957cb9054d4",
-                "underlay_address": "fd00:1122:3344:104::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
-                "underlay_address": "fd00:1122:3344:104::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "73916098-788f-48d8-b745-e13fdcfb771b",
-                "underlay_address": "fd00:1122:3344:104::c",
-                "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:104::c]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
-                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
-                  ]
-                }
-              },
-              {
-                "id": "9bb513e3-7505-416c-aa70-b26f1152bbdc",
-                "underlay_address": "fd00:1122:3344:104::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b"
-                  }
-                }
-              },
-              {
-                "id": "9c14c879-137b-4f5b-8068-1292bffc83ed",
-                "underlay_address": "fd00:1122:3344:104::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:104::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "9cf90589-5da1-4b42-b09b-d784e4134637",
-                "underlay_address": "fd00:1122:3344:104::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "198.51.100.23",
-                  "external_tls": false,
-                  "internal_address": "[fd00:1122:3344:104::5]:12221",
-                  "nic": {
-                    "id": "fa76e7e1-dee6-4574-b71f-e07ea1cffd08",
-                    "kind": {
-                      "type": "service",
-                      "id": "9cf90589-5da1-4b42-b09b-d784e4134637"
-                    },
-                    "name": "nexus-9cf90589-5da1-4b42-b09b-d784e4134637",
-                    "ip": "172.30.2.6",
-                    "mac": "A8:40:25:FF:C4:6F",
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "d48b2dfa-c749-47dd-98b6-3f478cd63934",
-                "underlay_address": "fd00:1122:3344:104::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:104::6]:17000"
-                }
-              },
-              {
-                "id": "fd003bee-818a-4c95-aae5-e378ac601522",
-                "underlay_address": "fd00:1122:3344:104::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "77851655-10a5-4e80-88b0-a001473cab7e": {
-          "time_collected": "2024-03-12T19:07:28.426275Z",
-          "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "084cac6e-44b5-4333-a2ff-b0993b534898",
-                "underlay_address": "fd00:1122:3344:102::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "11c527c5-0598-428e-9270-be5b74d3461b",
-                "underlay_address": "fd00:1122:3344:102::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "20be9299-495d-428d-b315-694c2c4b64af",
-                "underlay_address": "fd00:1122:3344:102::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:102::6]:17000"
-                }
-              },
-              {
-                "id": "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
-                "underlay_address": "fd00:1122:3344:102::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "8d452bdd-e8b8-4108-a690-46fc59197db0",
-                "underlay_address": "fd00:1122:3344:102::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "953019c0-8e85-4417-b231-d42820470125",
-                "underlay_address": "fd00:1122:3344:102::5",
-                "zone_type": {
-                  "type": "oximeter",
-                  "address": "[fd00:1122:3344:102::5]:12223"
-                }
-              },
-              {
-                "id": "99c635b9-8e97-414a-8316-8261d2d767d0",
-                "underlay_address": "fd00:1122:3344:102::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
-                "underlay_address": "fd00:1122:3344:2::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "[fd00:1122:3344:2::1]:53",
-                  "gz_address": "fd00:1122:3344:2::2",
-                  "gz_address_index": 1,
-                  "http_address": "[fd00:1122:3344:2::1]:5353"
-                }
-              },
-              {
-                "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-                "underlay_address": "fd00:1122:3344:102::c",
-                "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:102::c]:123",
-                  "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "nic": {
-                    "id": "659f33f7-e7bd-4875-9140-bf8a8b2d964a",
-                    "kind": {
-                      "type": "service",
-                      "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded"
-                    },
-                    "name": "ntp-d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-                    "ip": "172.30.3.6",
-                    "mac": "A8:40:25:FF:B1:99",
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  },
-                  "ntp_servers": [
-                    "time.cloudflare.com"
-                  ],
-                  "snat_cfg": {
-                    "ip": "198.51.100.26",
-                    "first_port": 16384,
-                    "last_port": 32767
-                  }
-                }
-              },
-              {
-                "id": "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
-                "underlay_address": "fd00:1122:3344:102::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b"
-                  }
-                }
-              },
-              {
-                "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-                "underlay_address": "fd00:1122:3344:102::4",
-                "zone_type": {
-                  "type": "external_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "198.51.100.21:53",
-                  "http_address": "[fd00:1122:3344:102::4]:5353",
-                  "nic": {
-                    "id": "d4dd2f7a-8a3b-405e-8444-62f99a754a8a",
-                    "kind": {
-                      "type": "service",
-                      "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802"
-                    },
-                    "name": "external-dns-f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-                    "ip": "172.30.1.6",
-                    "mac": "A8:40:25:FF:F4:5C",
-                    "subnet": "172.30.1.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
-          "time_collected": "2024-03-12T19:07:28.471504Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "0d459323-e414-4f32-9944-76c7331da622",
-                "underlay_address": "fd00:1122:3344:101::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "198.51.100.24",
-                  "external_tls": false,
-                  "internal_address": "[fd00:1122:3344:101::5]:12221",
-                  "nic": {
-                    "id": "514902b8-7a1a-42a3-95a6-d00861833b5c",
-                    "kind": {
-                      "type": "service",
-                      "id": "0d459323-e414-4f32-9944-76c7331da622"
-                    },
-                    "name": "nexus-0d459323-e414-4f32-9944-76c7331da622",
-                    "ip": "172.30.2.7",
-                    "mac": "A8:40:25:FF:DE:AC",
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "2902972b-3ae4-489d-b937-6480352e214a",
-                "underlay_address": "fd00:1122:3344:101::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "2ee832de-af75-47ac-abac-f91304857ca4",
-                "underlay_address": "fd00:1122:3344:101::4",
-                "zone_type": {
-                  "type": "external_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "198.51.100.20:53",
-                  "http_address": "[fd00:1122:3344:101::4]:5353",
-                  "nic": {
-                    "id": "fadff4ab-62d7-466a-9203-04080e4078ed",
-                    "kind": {
-                      "type": "service",
-                      "id": "2ee832de-af75-47ac-abac-f91304857ca4"
-                    },
-                    "name": "external-dns-2ee832de-af75-47ac-abac-f91304857ca4",
-                    "ip": "172.30.1.5",
-                    "mac": "A8:40:25:FF:B4:93",
-                    "subnet": "172.30.1.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "30050a7e-7fc3-4965-b07b-1635212393eb",
-                "underlay_address": "fd00:1122:3344:101::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:101::6]:17000"
-                }
-              },
-              {
-                "id": "8494befe-6df6-42d1-9baa-599ba4228430",
-                "underlay_address": "fd00:1122:3344:101::c",
-                "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:101::c]:123",
-                  "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "nic": {
-                    "id": "23041748-e3a6-4846-930d-5a348c316e91",
-                    "kind": {
-                      "type": "service",
-                      "id": "8494befe-6df6-42d1-9baa-599ba4228430"
-                    },
-                    "name": "ntp-8494befe-6df6-42d1-9baa-599ba4228430",
-                    "ip": "172.30.3.5",
-                    "mac": "A8:40:25:FF:D1:C4",
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  },
-                  "ntp_servers": [
-                    "time.cloudflare.com"
-                  ],
-                  "snat_cfg": {
-                    "ip": "198.51.100.25",
-                    "first_port": 0,
-                    "last_port": 16383
-                  }
-                }
-              },
-              {
-                "id": "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
-                "underlay_address": "fd00:1122:3344:101::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
-                "underlay_address": "fd00:1122:3344:101::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "e668e3f5-5457-4689-97da-b2af537787ac",
-                "underlay_address": "fd00:1122:3344:101::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
-                "underlay_address": "fd00:1122:3344:101::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b"
-                  }
-                }
-              },
-              {
-                "id": "f8c79ba4-194a-4cfe-8090-5315e297b780",
-                "underlay_address": "fd00:1122:3344:1::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "[fd00:1122:3344:1::1]:53",
-                  "gz_address": "fd00:1122:3344:1::2",
-                  "gz_address_index": 0,
-                  "http_address": "[fd00:1122:3344:1::1]:5353"
-                }
-              },
-              {
-                "id": "fa5e0176-b4a8-4598-aa31-40fd562d246c",
-                "underlay_address": "fd00:1122:3344:101::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              }
-            ]
-          }
-        }
-      }
-    },
-    {
-      "id": "3700dcd4-64bb-4997-bfdb-dcfc7cbbc998",
-      "errors": [],
-      "time_started": "2024-03-12T19:07:28.555090Z",
-      "time_done": "2024-03-12T19:07:28.728530Z",
-      "collector": "9cf90589-5da1-4b42-b09b-d784e4134637",
-      "baseboards": [
-        {
-          "part_number": "FAKE_SIM_SIDECAR",
-          "serial_number": "SimSidecar0"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g0"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g1"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g2"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g3"
-        }
-      ],
-      "cabooses": [
-        {
-          "board": "SimGimletSp",
-          "git_commit": "ffffffff",
-          "name": "SimGimlet",
-          "version": "0.0.1"
-        },
-        {
-          "board": "SimRot",
-          "git_commit": "eeeeeeee",
-          "name": "SimGimlet",
-          "version": "0.0.1"
-        },
-        {
-          "board": "SimRot",
-          "git_commit": "eeeeeeee",
-          "name": "SimSidecar",
-          "version": "0.0.1"
-        },
-        {
-          "board": "SimSidecarSp",
-          "git_commit": "ffffffff",
-          "name": "SimSidecar",
-          "version": "0.0.1"
-        }
-      ],
-      "rot_pages": [
-        {
-          "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        }
-      ],
-      "sps": [
-        [
-          {
-            "part_number": "FAKE_SIM_SIDECAR",
-            "serial_number": "SimSidecar0"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.560161Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "switch",
-            "sp_slot": 1,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g0"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.563500Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 0,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g1"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.566461Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 1,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g2"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.576218Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 2,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g3"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.556487Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 3,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ]
-      ],
-      "rots": [
-        [
-          {
-            "part_number": "FAKE_SIM_SIDECAR",
-            "serial_number": "SimSidecar0"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.560161Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g0"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.563500Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g1"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.566461Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g2"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.576218Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g3"
-          },
-          {
-            "time_collected": "2024-03-12T19:07:28.556487Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ]
-      ],
-      "cabooses_found": {
-        "SpSlot0": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.560637Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimSidecarSp",
-                "git_commit": "ffffffff",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.563951Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.566869Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.576918Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.557119Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ],
-        "SpSlot1": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.561066Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimSidecarSp",
-                "git_commit": "ffffffff",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.564309Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.567309Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.577611Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.557613Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ],
-        "RotSlotA": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.561535Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.564659Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.567749Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.578148Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.558097Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ],
-        "RotSlotB": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.561932Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.565057Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.568166Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.578556Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.558565Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ]
-      },
-      "rot_pages_found": {
-        "Cmpa": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.562164Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.565265Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.574062Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.578773Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.558913Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ],
-        "CfpaActive": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.562441Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.565512Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.574335Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.579009Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.559298Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ],
-        "CfpaInactive": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.562788Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.565754Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.574960Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.579245Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.559529Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ],
-        "CfpaScratch": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.563108Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.566172Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.575618Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.579487Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:07:28.559780Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ]
-      },
-      "sled_agents": {
-        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
-          "time_collected": "2024-03-12T19:07:28.622525Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a452a7f5-b528-40fe-80ff-4e4189e2d52b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b452a7f5-b528-40fe-80ff-4e4189e2d52b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d452a7f5-b528-40fe-80ff-4e4189e2d52b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        },
-        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
-          "time_collected": "2024-03-12T19:07:28.654092Z",
-          "source": "http://[fd00:1122:3344:104::1]:12345",
-          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:104::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        },
-        "77851655-10a5-4e80-88b0-a001473cab7e": {
-          "time_collected": "2024-03-12T19:07:28.685728Z",
-          "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a462a7f6-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b462a7f6-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d462a7f6-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        },
-        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
-          "time_collected": "2024-03-12T19:07:28.725605Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a462a4f4-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b462a4f4-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d462a4f4-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        }
-      },
-      "omicron_zones": {
-        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
-          "time_collected": "2024-03-12T19:07:28.623604Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "0db88baf-911d-47bb-af71-3aebd7c96934",
-                "underlay_address": "fd00:1122:3344:103::4",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "198.51.100.22",
-                  "external_tls": false,
-                  "internal_address": "[fd00:1122:3344:103::4]:12221",
-                  "nic": {
-                    "id": "8847a09c-cabc-429b-bf46-66d5e2e1140c",
-                    "kind": {
-                      "type": "service",
-                      "id": "0db88baf-911d-47bb-af71-3aebd7c96934"
-                    },
-                    "name": "nexus-0db88baf-911d-47bb-af71-3aebd7c96934",
-                    "ip": "172.30.2.5",
-                    "mac": "A8:40:25:FF:CC:DA",
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "213d7dd3-95a4-45a7-99ca-e4899991dab4",
-                "underlay_address": "fd00:1122:3344:103::6",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::6]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
-                "underlay_address": "fd00:1122:3344:3::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  },
-                  "dns_address": "[fd00:1122:3344:3::1]:53",
-                  "gz_address": "fd00:1122:3344:3::2",
-                  "gz_address_index": 2,
-                  "http_address": "[fd00:1122:3344:3::1]:5353"
-                }
-              },
-              {
-                "id": "40c47ead-3be8-4724-a8ef-b66233fc5c65",
-                "underlay_address": "fd00:1122:3344:103::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "425d9d2f-72ea-4f53-80b6-04e002771cdb",
-                "underlay_address": "fd00:1122:3344:103::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
-                "underlay_address": "fd00:1122:3344:103::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b"
-                  }
-                }
-              },
-              {
-                "id": "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
-                "underlay_address": "fd00:1122:3344:103::5",
-                "zone_type": {
-                  "type": "clickhouse",
-                  "address": "[fd00:1122:3344:103::5]:8123",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "7611a549-effc-41b6-b6cb-c03e0c70a31f",
-                "underlay_address": "fd00:1122:3344:103::b",
-                "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:103::b]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
-                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
-                  ]
-                }
-              },
-              {
-                "id": "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
-                "underlay_address": "fd00:1122:3344:103::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:103::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
-                "underlay_address": "fd00:1122:3344:103::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
-          "time_collected": "2024-03-12T19:07:28.655005Z",
-          "source": "http://[fd00:1122:3344:104::1]:12345",
-          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
-                "underlay_address": "fd00:1122:3344:104::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
-                "underlay_address": "fd00:1122:3344:104::4",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:104::4]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "57c712a6-6429-44b7-a9b2-9957cb9054d4",
-                "underlay_address": "fd00:1122:3344:104::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
-                "underlay_address": "fd00:1122:3344:104::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "73916098-788f-48d8-b745-e13fdcfb771b",
-                "underlay_address": "fd00:1122:3344:104::c",
-                "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:104::c]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
-                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
-                  ]
-                }
-              },
-              {
-                "id": "9bb513e3-7505-416c-aa70-b26f1152bbdc",
-                "underlay_address": "fd00:1122:3344:104::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b"
-                  }
-                }
-              },
-              {
-                "id": "9c14c879-137b-4f5b-8068-1292bffc83ed",
-                "underlay_address": "fd00:1122:3344:104::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:104::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "9cf90589-5da1-4b42-b09b-d784e4134637",
-                "underlay_address": "fd00:1122:3344:104::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "198.51.100.23",
-                  "external_tls": false,
-                  "internal_address": "[fd00:1122:3344:104::5]:12221",
-                  "nic": {
-                    "id": "fa76e7e1-dee6-4574-b71f-e07ea1cffd08",
-                    "kind": {
-                      "type": "service",
-                      "id": "9cf90589-5da1-4b42-b09b-d784e4134637"
-                    },
-                    "name": "nexus-9cf90589-5da1-4b42-b09b-d784e4134637",
-                    "ip": "172.30.2.6",
-                    "mac": "A8:40:25:FF:C4:6F",
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "d48b2dfa-c749-47dd-98b6-3f478cd63934",
-                "underlay_address": "fd00:1122:3344:104::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:104::6]:17000"
-                }
-              },
-              {
-                "id": "fd003bee-818a-4c95-aae5-e378ac601522",
-                "underlay_address": "fd00:1122:3344:104::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "77851655-10a5-4e80-88b0-a001473cab7e": {
-          "time_collected": "2024-03-12T19:07:28.687431Z",
-          "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "084cac6e-44b5-4333-a2ff-b0993b534898",
-                "underlay_address": "fd00:1122:3344:102::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "11c527c5-0598-428e-9270-be5b74d3461b",
-                "underlay_address": "fd00:1122:3344:102::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "20be9299-495d-428d-b315-694c2c4b64af",
-                "underlay_address": "fd00:1122:3344:102::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:102::6]:17000"
-                }
-              },
-              {
-                "id": "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
-                "underlay_address": "fd00:1122:3344:102::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "8d452bdd-e8b8-4108-a690-46fc59197db0",
-                "underlay_address": "fd00:1122:3344:102::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "953019c0-8e85-4417-b231-d42820470125",
-                "underlay_address": "fd00:1122:3344:102::5",
-                "zone_type": {
-                  "type": "oximeter",
-                  "address": "[fd00:1122:3344:102::5]:12223"
-                }
-              },
-              {
-                "id": "99c635b9-8e97-414a-8316-8261d2d767d0",
-                "underlay_address": "fd00:1122:3344:102::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
-                "underlay_address": "fd00:1122:3344:2::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "[fd00:1122:3344:2::1]:53",
-                  "gz_address": "fd00:1122:3344:2::2",
-                  "gz_address_index": 1,
-                  "http_address": "[fd00:1122:3344:2::1]:5353"
-                }
-              },
-              {
-                "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-                "underlay_address": "fd00:1122:3344:102::c",
-                "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:102::c]:123",
-                  "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "nic": {
-                    "id": "659f33f7-e7bd-4875-9140-bf8a8b2d964a",
-                    "kind": {
-                      "type": "service",
-                      "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded"
-                    },
-                    "name": "ntp-d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-                    "ip": "172.30.3.6",
-                    "mac": "A8:40:25:FF:B1:99",
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  },
-                  "ntp_servers": [
-                    "time.cloudflare.com"
-                  ],
-                  "snat_cfg": {
-                    "ip": "198.51.100.26",
-                    "first_port": 16384,
-                    "last_port": 32767
-                  }
-                }
-              },
-              {
-                "id": "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
-                "underlay_address": "fd00:1122:3344:102::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b"
-                  }
-                }
-              },
-              {
-                "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-                "underlay_address": "fd00:1122:3344:102::4",
-                "zone_type": {
-                  "type": "external_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "198.51.100.21:53",
-                  "http_address": "[fd00:1122:3344:102::4]:5353",
-                  "nic": {
-                    "id": "d4dd2f7a-8a3b-405e-8444-62f99a754a8a",
-                    "kind": {
-                      "type": "service",
-                      "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802"
-                    },
-                    "name": "external-dns-f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-                    "ip": "172.30.1.6",
-                    "mac": "A8:40:25:FF:F4:5C",
-                    "subnet": "172.30.1.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
-          "time_collected": "2024-03-12T19:07:28.727281Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "0d459323-e414-4f32-9944-76c7331da622",
-                "underlay_address": "fd00:1122:3344:101::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "198.51.100.24",
-                  "external_tls": false,
-                  "internal_address": "[fd00:1122:3344:101::5]:12221",
-                  "nic": {
-                    "id": "514902b8-7a1a-42a3-95a6-d00861833b5c",
-                    "kind": {
-                      "type": "service",
-                      "id": "0d459323-e414-4f32-9944-76c7331da622"
-                    },
-                    "name": "nexus-0d459323-e414-4f32-9944-76c7331da622",
-                    "ip": "172.30.2.7",
-                    "mac": "A8:40:25:FF:DE:AC",
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "2902972b-3ae4-489d-b937-6480352e214a",
-                "underlay_address": "fd00:1122:3344:101::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "2ee832de-af75-47ac-abac-f91304857ca4",
-                "underlay_address": "fd00:1122:3344:101::4",
-                "zone_type": {
-                  "type": "external_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "198.51.100.20:53",
-                  "http_address": "[fd00:1122:3344:101::4]:5353",
-                  "nic": {
-                    "id": "fadff4ab-62d7-466a-9203-04080e4078ed",
-                    "kind": {
-                      "type": "service",
-                      "id": "2ee832de-af75-47ac-abac-f91304857ca4"
-                    },
-                    "name": "external-dns-2ee832de-af75-47ac-abac-f91304857ca4",
-                    "ip": "172.30.1.5",
-                    "mac": "A8:40:25:FF:B4:93",
-                    "subnet": "172.30.1.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "30050a7e-7fc3-4965-b07b-1635212393eb",
-                "underlay_address": "fd00:1122:3344:101::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:101::6]:17000"
-                }
-              },
-              {
-                "id": "8494befe-6df6-42d1-9baa-599ba4228430",
-                "underlay_address": "fd00:1122:3344:101::c",
-                "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:101::c]:123",
-                  "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "nic": {
-                    "id": "23041748-e3a6-4846-930d-5a348c316e91",
-                    "kind": {
-                      "type": "service",
-                      "id": "8494befe-6df6-42d1-9baa-599ba4228430"
-                    },
-                    "name": "ntp-8494befe-6df6-42d1-9baa-599ba4228430",
-                    "ip": "172.30.3.5",
-                    "mac": "A8:40:25:FF:D1:C4",
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  },
-                  "ntp_servers": [
-                    "time.cloudflare.com"
-                  ],
-                  "snat_cfg": {
-                    "ip": "198.51.100.25",
-                    "first_port": 0,
-                    "last_port": 16383
-                  }
-                }
-              },
-              {
-                "id": "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
-                "underlay_address": "fd00:1122:3344:101::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
-                "underlay_address": "fd00:1122:3344:101::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "e668e3f5-5457-4689-97da-b2af537787ac",
-                "underlay_address": "fd00:1122:3344:101::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
-                "underlay_address": "fd00:1122:3344:101::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b"
-                  }
-                }
-              },
-              {
-                "id": "f8c79ba4-194a-4cfe-8090-5315e297b780",
-                "underlay_address": "fd00:1122:3344:1::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "[fd00:1122:3344:1::1]:53",
-                  "gz_address": "fd00:1122:3344:1::2",
-                  "gz_address_index": 0,
-                  "http_address": "[fd00:1122:3344:1::1]:5353"
-                }
-              },
-              {
-                "id": "fa5e0176-b4a8-4598-aa31-40fd562d246c",
-                "underlay_address": "fd00:1122:3344:101::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              }
-            ]
-          }
-        }
-      }
-    },
-    {
-      "id": "5e1d6ae0-c5b4-4884-ac41-680cd4f5762d",
-      "errors": [],
-      "time_started": "2024-03-12T19:09:36.927333Z",
-      "time_done": "2024-03-12T19:09:37.116573Z",
-      "collector": "0d459323-e414-4f32-9944-76c7331da622",
-      "baseboards": [
-        {
-          "part_number": "FAKE_SIM_SIDECAR",
-          "serial_number": "SimSidecar0"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g0"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g1"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g2"
-        },
-        {
-          "part_number": "i86pc",
-          "serial_number": "g3"
-        }
-      ],
-      "cabooses": [
-        {
-          "board": "SimGimletSp",
-          "git_commit": "ffffffff",
-          "name": "SimGimlet",
-          "version": "0.0.1"
-        },
-        {
-          "board": "SimRot",
-          "git_commit": "eeeeeeee",
-          "name": "SimGimlet",
-          "version": "0.0.1"
-        },
-        {
-          "board": "SimRot",
-          "git_commit": "eeeeeeee",
-          "name": "SimSidecar",
-          "version": "0.0.1"
-        },
-        {
-          "board": "SimSidecarSp",
-          "git_commit": "ffffffff",
-          "name": "SimSidecar",
-          "version": "0.0.1"
-        }
-      ],
-      "rot_pages": [
-        {
-          "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        },
-        {
-          "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-        }
-      ],
-      "sps": [
-        [
-          {
-            "part_number": "FAKE_SIM_SIDECAR",
-            "serial_number": "SimSidecar0"
-          },
-          {
-            "time_collected": "2024-03-12T19:09:36.934628Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "switch",
-            "sp_slot": 1,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g0"
-          },
-          {
-            "time_collected": "2024-03-12T19:09:36.940025Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 0,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g1"
-          },
-          {
-            "time_collected": "2024-03-12T19:09:36.945090Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 1,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g2"
-          },
-          {
-            "time_collected": "2024-03-12T19:09:36.949989Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 2,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g3"
-          },
-          {
-            "time_collected": "2024-03-12T19:09:36.929081Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "sp_type": "sled",
-            "sp_slot": 3,
-            "baseboard_revision": 0,
-            "hubris_archive": "0000000000000000",
-            "power_state": "A2"
-          }
-        ]
-      ],
-      "rots": [
-        [
-          {
-            "part_number": "FAKE_SIM_SIDECAR",
-            "serial_number": "SimSidecar0"
-          },
-          {
-            "time_collected": "2024-03-12T19:09:36.934628Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g0"
-          },
-          {
-            "time_collected": "2024-03-12T19:09:36.940025Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g1"
-          },
-          {
-            "time_collected": "2024-03-12T19:09:36.945090Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g2"
-          },
-          {
-            "time_collected": "2024-03-12T19:09:36.949989Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ],
-        [
-          {
-            "part_number": "i86pc",
-            "serial_number": "g3"
-          },
-          {
-            "time_collected": "2024-03-12T19:09:36.929081Z",
-            "source": "http://[fd00:1122:3344:104::2]:12225",
-            "active_slot": {
-              "slot": "a"
-            },
-            "persistent_boot_preference": {
-              "slot": "a"
-            },
-            "pending_persistent_boot_preference": null,
-            "transient_boot_preference": null,
-            "slot_a_sha3_256_digest": null,
-            "slot_b_sha3_256_digest": null
-          }
-        ]
-      ],
-      "cabooses_found": {
-        "SpSlot0": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.935488Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimSidecarSp",
-                "git_commit": "ffffffff",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.940638Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.945685Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.950734Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.929758Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ],
-        "SpSlot1": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.936167Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimSidecarSp",
-                "git_commit": "ffffffff",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.941223Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.946254Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.951335Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.930670Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimGimletSp",
-                "git_commit": "ffffffff",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ],
-        "RotSlotA": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.936811Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.941800Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.946826Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.951934Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.931245Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ],
-        "RotSlotB": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.937534Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimSidecar",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.942409Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.947558Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.953023Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.931825Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "caboose": {
-                "board": "SimRot",
-                "git_commit": "eeeeeeee",
-                "name": "SimGimlet",
-                "version": "0.0.1"
-              }
-            }
-          ]
-        ]
-      },
-      "rot_pages_found": {
-        "Cmpa": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.938012Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.942853Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.948038Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.954159Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.932312Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ],
-        "CfpaActive": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.938490Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.943336Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.948514Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.954600Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.932820Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ],
-        "CfpaInactive": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.938996Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.944180Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.948958Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.955267Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.933319Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ],
-        "CfpaScratch": [
-          [
-            {
-              "part_number": "FAKE_SIM_SIDECAR",
-              "serial_number": "SimSidecar0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.939491Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g0"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.944650Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g1"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.949453Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g2"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.955924Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ],
-          [
-            {
-              "part_number": "i86pc",
-              "serial_number": "g3"
-            },
-            {
-              "time_collected": "2024-03-12T19:09:36.933794Z",
-              "source": "http://[fd00:1122:3344:104::2]:12225",
-              "page": {
-                "data_base64": "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-              }
-            }
-          ]
-        ]
-      },
-      "sled_agents": {
-        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
-          "time_collected": "2024-03-12T19:09:37.008918Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:103::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a452a7f5-b528-40fe-80ff-4e4189e2d52b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b452a7f5-b528-40fe-80ff-4e4189e2d52b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d452a7f5-b528-40fe-80ff-4e4189e2d52b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc85-ab45-49fb-a4b4-d351ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        },
-        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
-          "time_collected": "2024-03-12T19:09:37.040839Z",
-          "source": "http://[fd00:1122:3344:104::1]:12345",
-          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:104::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a462a7f7-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b462a7f7-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d462a7f7-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc87-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        },
-        "77851655-10a5-4e80-88b0-a001473cab7e": {
-          "time_collected": "2024-03-12T19:09:37.086600Z",
-          "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:102::1]:12345",
-          "sled_role": "gimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a462a7f6-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b462a7f6-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d462a7f6-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc86-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        },
-        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
-          "time_collected": "2024-03-12T19:09:37.114717Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
-          "baseboard_id": null,
-          "sled_agent_address": "[fd00:1122:3344:101::1]:12345",
-          "sled_role": "scrimlet",
-          "usable_hardware_threads": 8,
-          "usable_physical_ram": 8582262784,
-          "reservoir_size": 0,
-          "disks": [
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-a462a4f4-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1024
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-b462a4f4-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "m2",
-              "slot": 1025
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-d462a4f4-b628-40fe-80ff-4e4189e2d62b",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1026
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-e4b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1027
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-f4b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1028
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-14b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1029
-            },
-            {
-              "identity": {
-                "vendor": "synthetic-vendor",
-                "serial": "synthetic-serial-24b4dc84-ab46-49fb-a4b4-d361ae214c03",
-                "model": "synthetic-model"
-              },
-              "variant": "u2",
-              "slot": 1030
-            }
-          ]
-        }
-      },
-      "omicron_zones": {
-        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
-          "time_collected": "2024-03-12T19:09:37.010960Z",
-          "source": "http://[fd00:1122:3344:103::1]:12345",
-          "sled_id": "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "0db88baf-911d-47bb-af71-3aebd7c96934",
-                "underlay_address": "fd00:1122:3344:103::4",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "198.51.100.22",
-                  "external_tls": false,
-                  "internal_address": "[fd00:1122:3344:103::4]:12221",
-                  "nic": {
-                    "id": "8847a09c-cabc-429b-bf46-66d5e2e1140c",
-                    "kind": {
-                      "type": "service",
-                      "id": "0db88baf-911d-47bb-af71-3aebd7c96934"
-                    },
-                    "name": "nexus-0db88baf-911d-47bb-af71-3aebd7c96934",
-                    "ip": "172.30.2.5",
-                    "mac": "A8:40:25:FF:CC:DA",
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "213d7dd3-95a4-45a7-99ca-e4899991dab4",
-                "underlay_address": "fd00:1122:3344:103::6",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::6]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
-                "underlay_address": "fd00:1122:3344:3::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  },
-                  "dns_address": "[fd00:1122:3344:3::1]:53",
-                  "gz_address": "fd00:1122:3344:3::2",
-                  "gz_address_index": 2,
-                  "http_address": "[fd00:1122:3344:3::1]:5353"
-                }
-              },
-              {
-                "id": "40c47ead-3be8-4724-a8ef-b66233fc5c65",
-                "underlay_address": "fd00:1122:3344:103::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "425d9d2f-72ea-4f53-80b6-04e002771cdb",
-                "underlay_address": "fd00:1122:3344:103::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
-                "underlay_address": "fd00:1122:3344:103::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b"
-                  }
-                }
-              },
-              {
-                "id": "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
-                "underlay_address": "fd00:1122:3344:103::5",
-                "zone_type": {
-                  "type": "clickhouse",
-                  "address": "[fd00:1122:3344:103::5]:8123",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "7611a549-effc-41b6-b6cb-c03e0c70a31f",
-                "underlay_address": "fd00:1122:3344:103::b",
-                "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:103::b]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
-                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
-                  ]
-                }
-              },
-              {
-                "id": "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
-                "underlay_address": "fd00:1122:3344:103::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:103::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
-                "underlay_address": "fd00:1122:3344:103::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:103::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
-          "time_collected": "2024-03-12T19:09:37.049002Z",
-          "source": "http://[fd00:1122:3344:104::1]:12345",
-          "sled_id": "67a1268b-52d3-4159-b3ed-7a0777f1f340",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
-                "underlay_address": "fd00:1122:3344:104::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
-                "underlay_address": "fd00:1122:3344:104::4",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:104::4]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "57c712a6-6429-44b7-a9b2-9957cb9054d4",
-                "underlay_address": "fd00:1122:3344:104::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
-                "underlay_address": "fd00:1122:3344:104::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "73916098-788f-48d8-b745-e13fdcfb771b",
-                "underlay_address": "fd00:1122:3344:104::c",
-                "zone_type": {
-                  "type": "internal_ntp",
-                  "address": "[fd00:1122:3344:104::c]:123",
-                  "dns_servers": [
-                    "fd00:1122:3344:1::1",
-                    "fd00:1122:3344:2::1",
-                    "fd00:1122:3344:3::1"
-                  ],
-                  "ntp_servers": [
-                    "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
-                    "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
-                  ]
-                }
-              },
-              {
-                "id": "9bb513e3-7505-416c-aa70-b26f1152bbdc",
-                "underlay_address": "fd00:1122:3344:104::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b"
-                  }
-                }
-              },
-              {
-                "id": "9c14c879-137b-4f5b-8068-1292bffc83ed",
-                "underlay_address": "fd00:1122:3344:104::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:104::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "9cf90589-5da1-4b42-b09b-d784e4134637",
-                "underlay_address": "fd00:1122:3344:104::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "198.51.100.23",
-                  "external_tls": false,
-                  "internal_address": "[fd00:1122:3344:104::5]:12221",
-                  "nic": {
-                    "id": "fa76e7e1-dee6-4574-b71f-e07ea1cffd08",
-                    "kind": {
-                      "type": "service",
-                      "id": "9cf90589-5da1-4b42-b09b-d784e4134637"
-                    },
-                    "name": "nexus-9cf90589-5da1-4b42-b09b-d784e4134637",
-                    "ip": "172.30.2.6",
-                    "mac": "A8:40:25:FF:C4:6F",
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "d48b2dfa-c749-47dd-98b6-3f478cd63934",
-                "underlay_address": "fd00:1122:3344:104::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:104::6]:17000"
-                }
-              },
-              {
-                "id": "fd003bee-818a-4c95-aae5-e378ac601522",
-                "underlay_address": "fd00:1122:3344:104::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:104::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "77851655-10a5-4e80-88b0-a001473cab7e": {
-          "time_collected": "2024-03-12T19:09:37.088155Z",
-          "source": "http://[fd00:1122:3344:102::1]:12345",
-          "sled_id": "77851655-10a5-4e80-88b0-a001473cab7e",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "084cac6e-44b5-4333-a2ff-b0993b534898",
-                "underlay_address": "fd00:1122:3344:102::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "11c527c5-0598-428e-9270-be5b74d3461b",
-                "underlay_address": "fd00:1122:3344:102::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:102::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "20be9299-495d-428d-b315-694c2c4b64af",
-                "underlay_address": "fd00:1122:3344:102::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:102::6]:17000"
-                }
-              },
-              {
-                "id": "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
-                "underlay_address": "fd00:1122:3344:102::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "8d452bdd-e8b8-4108-a690-46fc59197db0",
-                "underlay_address": "fd00:1122:3344:102::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "953019c0-8e85-4417-b231-d42820470125",
-                "underlay_address": "fd00:1122:3344:102::5",
-                "zone_type": {
-                  "type": "oximeter",
-                  "address": "[fd00:1122:3344:102::5]:12223"
-                }
-              },
-              {
-                "id": "99c635b9-8e97-414a-8316-8261d2d767d0",
-                "underlay_address": "fd00:1122:3344:102::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
-                "underlay_address": "fd00:1122:3344:2::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "[fd00:1122:3344:2::1]:53",
-                  "gz_address": "fd00:1122:3344:2::2",
-                  "gz_address_index": 1,
-                  "http_address": "[fd00:1122:3344:2::1]:5353"
-                }
-              },
-              {
-                "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-                "underlay_address": "fd00:1122:3344:102::c",
-                "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:102::c]:123",
-                  "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "nic": {
-                    "id": "659f33f7-e7bd-4875-9140-bf8a8b2d964a",
-                    "kind": {
-                      "type": "service",
-                      "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded"
-                    },
-                    "name": "ntp-d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-                    "ip": "172.30.3.6",
-                    "mac": "A8:40:25:FF:B1:99",
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  },
-                  "ntp_servers": [
-                    "time.cloudflare.com"
-                  ],
-                  "snat_cfg": {
-                    "ip": "198.51.100.26",
-                    "first_port": 16384,
-                    "last_port": 32767
-                  }
-                }
-              },
-              {
-                "id": "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
-                "underlay_address": "fd00:1122:3344:102::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:102::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b"
-                  }
-                }
-              },
-              {
-                "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-                "underlay_address": "fd00:1122:3344:102::4",
-                "zone_type": {
-                  "type": "external_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "198.51.100.21:53",
-                  "http_address": "[fd00:1122:3344:102::4]:5353",
-                  "nic": {
-                    "id": "d4dd2f7a-8a3b-405e-8444-62f99a754a8a",
-                    "kind": {
-                      "type": "service",
-                      "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802"
-                    },
-                    "name": "external-dns-f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-                    "ip": "172.30.1.6",
-                    "mac": "A8:40:25:FF:F4:5C",
-                    "subnet": "172.30.1.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              }
-            ]
-          }
-        },
-        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
-          "time_collected": "2024-03-12T19:09:37.115884Z",
-          "source": "http://[fd00:1122:3344:101::1]:12345",
-          "sled_id": "f122b3a2-e383-4e42-b2a5-74a69a9d110a",
-          "zones": {
-            "generation": 5,
-            "zones": [
-              {
-                "id": "0d459323-e414-4f32-9944-76c7331da622",
-                "underlay_address": "fd00:1122:3344:101::5",
-                "zone_type": {
-                  "type": "nexus",
-                  "external_dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "external_ip": "198.51.100.24",
-                  "external_tls": false,
-                  "internal_address": "[fd00:1122:3344:101::5]:12221",
-                  "nic": {
-                    "id": "514902b8-7a1a-42a3-95a6-d00861833b5c",
-                    "kind": {
-                      "type": "service",
-                      "id": "0d459323-e414-4f32-9944-76c7331da622"
-                    },
-                    "name": "nexus-0d459323-e414-4f32-9944-76c7331da622",
-                    "ip": "172.30.2.7",
-                    "mac": "A8:40:25:FF:DE:AC",
-                    "subnet": "172.30.2.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "2902972b-3ae4-489d-b937-6480352e214a",
-                "underlay_address": "fd00:1122:3344:101::8",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::8]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "2ee832de-af75-47ac-abac-f91304857ca4",
-                "underlay_address": "fd00:1122:3344:101::4",
-                "zone_type": {
-                  "type": "external_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "198.51.100.20:53",
-                  "http_address": "[fd00:1122:3344:101::4]:5353",
-                  "nic": {
-                    "id": "fadff4ab-62d7-466a-9203-04080e4078ed",
-                    "kind": {
-                      "type": "service",
-                      "id": "2ee832de-af75-47ac-abac-f91304857ca4"
-                    },
-                    "name": "external-dns-2ee832de-af75-47ac-abac-f91304857ca4",
-                    "ip": "172.30.1.5",
-                    "mac": "A8:40:25:FF:B4:93",
-                    "subnet": "172.30.1.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  }
-                }
-              },
-              {
-                "id": "30050a7e-7fc3-4965-b07b-1635212393eb",
-                "underlay_address": "fd00:1122:3344:101::6",
-                "zone_type": {
-                  "type": "crucible_pantry",
-                  "address": "[fd00:1122:3344:101::6]:17000"
-                }
-              },
-              {
-                "id": "8494befe-6df6-42d1-9baa-599ba4228430",
-                "underlay_address": "fd00:1122:3344:101::c",
-                "zone_type": {
-                  "type": "boundary_ntp",
-                  "address": "[fd00:1122:3344:101::c]:123",
-                  "dns_servers": [
-                    "1.1.1.1",
-                    "9.9.9.9"
-                  ],
-                  "nic": {
-                    "id": "23041748-e3a6-4846-930d-5a348c316e91",
-                    "kind": {
-                      "type": "service",
-                      "id": "8494befe-6df6-42d1-9baa-599ba4228430"
-                    },
-                    "name": "ntp-8494befe-6df6-42d1-9baa-599ba4228430",
-                    "ip": "172.30.3.5",
-                    "mac": "A8:40:25:FF:D1:C4",
-                    "subnet": "172.30.3.0/24",
-                    "vni": 100,
-                    "primary": true,
-                    "slot": 0
-                  },
-                  "ntp_servers": [
-                    "time.cloudflare.com"
-                  ],
-                  "snat_cfg": {
-                    "ip": "198.51.100.25",
-                    "first_port": 0,
-                    "last_port": 16383
-                  }
-                }
-              },
-              {
-                "id": "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
-                "underlay_address": "fd00:1122:3344:101::7",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::7]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
-                "underlay_address": "fd00:1122:3344:101::3",
-                "zone_type": {
-                  "type": "cockroach_db",
-                  "address": "[fd00:1122:3344:101::3]:32221",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "e668e3f5-5457-4689-97da-b2af537787ac",
-                "underlay_address": "fd00:1122:3344:101::b",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::b]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              },
-              {
-                "id": "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
-                "underlay_address": "fd00:1122:3344:101::9",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::9]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b"
-                  }
-                }
-              },
-              {
-                "id": "f8c79ba4-194a-4cfe-8090-5315e297b780",
-                "underlay_address": "fd00:1122:3344:1::1",
-                "zone_type": {
-                  "type": "internal_dns",
-                  "dataset": {
-                    "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  },
-                  "dns_address": "[fd00:1122:3344:1::1]:53",
-                  "gz_address": "fd00:1122:3344:1::2",
-                  "gz_address_index": 0,
-                  "http_address": "[fd00:1122:3344:1::1]:5353"
-                }
-              },
-              {
-                "id": "fa5e0176-b4a8-4598-aa31-40fd562d246c",
-                "underlay_address": "fd00:1122:3344:101::a",
-                "zone_type": {
-                  "type": "crucible",
-                  "address": "[fd00:1122:3344:101::a]:32345",
-                  "dataset": {
-                    "pool_name": "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                  }
-                }
-              }
-            ]
-          }
-        }
-      }
-    }
-  ],
-  "blueprints": [
-    {
-      "id": "649e444d-d00f-4067-a915-12e15647bf37",
-      "omicron_zones": {
-        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "0db88baf-911d-47bb-af71-3aebd7c96934",
-              "underlay_address": "fd00:1122:3344:103::4",
-              "zone_type": {
-                "type": "nexus",
-                "external_dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "external_ip": "198.51.100.22",
-                "external_tls": false,
-                "internal_address": "[fd00:1122:3344:103::4]:12221",
-                "nic": {
-                  "id": "8847a09c-cabc-429b-bf46-66d5e2e1140c",
-                  "kind": {
-                    "type": "service",
-                    "id": "0db88baf-911d-47bb-af71-3aebd7c96934"
-                  },
-                  "name": "nexus-0db88baf-911d-47bb-af71-3aebd7c96934",
-                  "ip": "172.30.2.5",
-                  "mac": "A8:40:25:FF:CC:DA",
-                  "subnet": "172.30.2.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                }
-              }
-            },
-            {
-              "id": "213d7dd3-95a4-45a7-99ca-e4899991dab4",
-              "underlay_address": "fd00:1122:3344:103::6",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::6]:32345",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                }
-              }
-            },
-            {
-              "id": "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
-              "underlay_address": "fd00:1122:3344:3::1",
-              "zone_type": {
-                "type": "internal_dns",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                },
-                "dns_address": "[fd00:1122:3344:3::1]:53",
-                "gz_address": "fd00:1122:3344:3::2",
-                "gz_address_index": 2,
-                "http_address": "[fd00:1122:3344:3::1]:5353"
-              }
-            },
-            {
-              "id": "40c47ead-3be8-4724-a8ef-b66233fc5c65",
-              "underlay_address": "fd00:1122:3344:103::7",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::7]:32345",
-                "dataset": {
-                  "pool_name": "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                }
-              }
-            },
-            {
-              "id": "425d9d2f-72ea-4f53-80b6-04e002771cdb",
-              "underlay_address": "fd00:1122:3344:103::9",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                }
-              }
-            },
-            {
-              "id": "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
-              "underlay_address": "fd00:1122:3344:103::8",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b"
-                }
-              }
-            },
-            {
-              "id": "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
-              "underlay_address": "fd00:1122:3344:103::5",
-              "zone_type": {
-                "type": "clickhouse",
-                "address": "[fd00:1122:3344:103::5]:8123",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                }
-              }
-            },
-            {
-              "id": "7611a549-effc-41b6-b6cb-c03e0c70a31f",
-              "underlay_address": "fd00:1122:3344:103::b",
-              "zone_type": {
-                "type": "internal_ntp",
-                "address": "[fd00:1122:3344:103::b]:123",
-                "dns_servers": [
-                  "fd00:1122:3344:1::1",
-                  "fd00:1122:3344:2::1",
-                  "fd00:1122:3344:3::1"
-                ],
-                "ntp_servers": [
-                  "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
-                  "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
-                ]
-              }
-            },
-            {
-              "id": "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
-              "underlay_address": "fd00:1122:3344:103::3",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:103::3]:32221",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                }
-              }
-            },
-            {
-              "id": "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
-              "underlay_address": "fd00:1122:3344:103::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                }
-              }
-            }
-          ]
-        },
-        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
-              "underlay_address": "fd00:1122:3344:104::b",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:104::b]:32345",
-                "dataset": {
-                  "pool_name": "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
-              "underlay_address": "fd00:1122:3344:104::4",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:104::4]:32221",
-                "dataset": {
-                  "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "57c712a6-6429-44b7-a9b2-9957cb9054d4",
-              "underlay_address": "fd00:1122:3344:104::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:104::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
-              "underlay_address": "fd00:1122:3344:104::7",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:104::7]:32345",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "73916098-788f-48d8-b745-e13fdcfb771b",
-              "underlay_address": "fd00:1122:3344:104::c",
-              "zone_type": {
-                "type": "internal_ntp",
-                "address": "[fd00:1122:3344:104::c]:123",
-                "dns_servers": [
-                  "fd00:1122:3344:1::1",
-                  "fd00:1122:3344:2::1",
-                  "fd00:1122:3344:3::1"
-                ],
-                "ntp_servers": [
-                  "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
-                  "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
-                ]
-              }
-            },
-            {
-              "id": "9bb513e3-7505-416c-aa70-b26f1152bbdc",
-              "underlay_address": "fd00:1122:3344:104::9",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:104::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b"
-                }
-              }
-            },
-            {
-              "id": "9c14c879-137b-4f5b-8068-1292bffc83ed",
-              "underlay_address": "fd00:1122:3344:104::3",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:104::3]:32221",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "9cf90589-5da1-4b42-b09b-d784e4134637",
-              "underlay_address": "fd00:1122:3344:104::5",
-              "zone_type": {
-                "type": "nexus",
-                "external_dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "external_ip": "198.51.100.23",
-                "external_tls": false,
-                "internal_address": "[fd00:1122:3344:104::5]:12221",
-                "nic": {
-                  "id": "fa76e7e1-dee6-4574-b71f-e07ea1cffd08",
-                  "kind": {
-                    "type": "service",
-                    "id": "9cf90589-5da1-4b42-b09b-d784e4134637"
-                  },
-                  "name": "nexus-9cf90589-5da1-4b42-b09b-d784e4134637",
-                  "ip": "172.30.2.6",
-                  "mac": "A8:40:25:FF:C4:6F",
-                  "subnet": "172.30.2.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                }
-              }
-            },
-            {
-              "id": "d48b2dfa-c749-47dd-98b6-3f478cd63934",
-              "underlay_address": "fd00:1122:3344:104::6",
-              "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:104::6]:17000"
-              }
-            },
-            {
-              "id": "fd003bee-818a-4c95-aae5-e378ac601522",
-              "underlay_address": "fd00:1122:3344:104::8",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:104::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            }
-          ]
-        },
-        "77851655-10a5-4e80-88b0-a001473cab7e": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "084cac6e-44b5-4333-a2ff-b0993b534898",
-              "underlay_address": "fd00:1122:3344:102::8",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "11c527c5-0598-428e-9270-be5b74d3461b",
-              "underlay_address": "fd00:1122:3344:102::3",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:102::3]:32221",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "20be9299-495d-428d-b315-694c2c4b64af",
-              "underlay_address": "fd00:1122:3344:102::6",
-              "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:102::6]:17000"
-              }
-            },
-            {
-              "id": "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
-              "underlay_address": "fd00:1122:3344:102::b",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::b]:32345",
-                "dataset": {
-                  "pool_name": "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "8d452bdd-e8b8-4108-a690-46fc59197db0",
-              "underlay_address": "fd00:1122:3344:102::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "953019c0-8e85-4417-b231-d42820470125",
-              "underlay_address": "fd00:1122:3344:102::5",
-              "zone_type": {
-                "type": "oximeter",
-                "address": "[fd00:1122:3344:102::5]:12223"
-              }
-            },
-            {
-              "id": "99c635b9-8e97-414a-8316-8261d2d767d0",
-              "underlay_address": "fd00:1122:3344:102::7",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::7]:32345",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
-              "underlay_address": "fd00:1122:3344:2::1",
-              "zone_type": {
-                "type": "internal_dns",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                },
-                "dns_address": "[fd00:1122:3344:2::1]:53",
-                "gz_address": "fd00:1122:3344:2::2",
-                "gz_address_index": 1,
-                "http_address": "[fd00:1122:3344:2::1]:5353"
-              }
-            },
-            {
-              "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-              "underlay_address": "fd00:1122:3344:102::c",
-              "zone_type": {
-                "type": "boundary_ntp",
-                "address": "[fd00:1122:3344:102::c]:123",
-                "dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "nic": {
-                  "id": "659f33f7-e7bd-4875-9140-bf8a8b2d964a",
-                  "kind": {
-                    "type": "service",
-                    "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded"
-                  },
-                  "name": "ntp-d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-                  "ip": "172.30.3.6",
-                  "mac": "A8:40:25:FF:B1:99",
-                  "subnet": "172.30.3.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                },
-                "ntp_servers": [
-                  "time.cloudflare.com"
-                ],
-                "snat_cfg": {
-                  "ip": "198.51.100.26",
-                  "first_port": 16384,
-                  "last_port": 32767
-                }
-              }
-            },
-            {
-              "id": "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
-              "underlay_address": "fd00:1122:3344:102::9",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b"
-                }
-              }
-            },
-            {
-              "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-              "underlay_address": "fd00:1122:3344:102::4",
-              "zone_type": {
-                "type": "external_dns",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                },
-                "dns_address": "198.51.100.21:53",
-                "http_address": "[fd00:1122:3344:102::4]:5353",
-                "nic": {
-                  "id": "d4dd2f7a-8a3b-405e-8444-62f99a754a8a",
-                  "kind": {
-                    "type": "service",
-                    "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802"
-                  },
-                  "name": "external-dns-f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-                  "ip": "172.30.1.6",
-                  "mac": "A8:40:25:FF:F4:5C",
-                  "subnet": "172.30.1.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                }
-              }
-            }
-          ]
-        },
-        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "0d459323-e414-4f32-9944-76c7331da622",
-              "underlay_address": "fd00:1122:3344:101::5",
-              "zone_type": {
-                "type": "nexus",
-                "external_dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "external_ip": "198.51.100.24",
-                "external_tls": false,
-                "internal_address": "[fd00:1122:3344:101::5]:12221",
-                "nic": {
-                  "id": "514902b8-7a1a-42a3-95a6-d00861833b5c",
-                  "kind": {
-                    "type": "service",
-                    "id": "0d459323-e414-4f32-9944-76c7331da622"
-                  },
-                  "name": "nexus-0d459323-e414-4f32-9944-76c7331da622",
-                  "ip": "172.30.2.7",
-                  "mac": "A8:40:25:FF:DE:AC",
-                  "subnet": "172.30.2.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                }
-              }
-            },
-            {
-              "id": "2902972b-3ae4-489d-b937-6480352e214a",
-              "underlay_address": "fd00:1122:3344:101::8",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "2ee832de-af75-47ac-abac-f91304857ca4",
-              "underlay_address": "fd00:1122:3344:101::4",
-              "zone_type": {
-                "type": "external_dns",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                },
-                "dns_address": "198.51.100.20:53",
-                "http_address": "[fd00:1122:3344:101::4]:5353",
-                "nic": {
-                  "id": "fadff4ab-62d7-466a-9203-04080e4078ed",
-                  "kind": {
-                    "type": "service",
-                    "id": "2ee832de-af75-47ac-abac-f91304857ca4"
-                  },
-                  "name": "external-dns-2ee832de-af75-47ac-abac-f91304857ca4",
-                  "ip": "172.30.1.5",
-                  "mac": "A8:40:25:FF:B4:93",
-                  "subnet": "172.30.1.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                }
-              }
-            },
-            {
-              "id": "30050a7e-7fc3-4965-b07b-1635212393eb",
-              "underlay_address": "fd00:1122:3344:101::6",
-              "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:101::6]:17000"
-              }
-            },
-            {
-              "id": "8494befe-6df6-42d1-9baa-599ba4228430",
-              "underlay_address": "fd00:1122:3344:101::c",
-              "zone_type": {
-                "type": "boundary_ntp",
-                "address": "[fd00:1122:3344:101::c]:123",
-                "dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "nic": {
-                  "id": "23041748-e3a6-4846-930d-5a348c316e91",
-                  "kind": {
-                    "type": "service",
-                    "id": "8494befe-6df6-42d1-9baa-599ba4228430"
-                  },
-                  "name": "ntp-8494befe-6df6-42d1-9baa-599ba4228430",
-                  "ip": "172.30.3.5",
-                  "mac": "A8:40:25:FF:D1:C4",
-                  "subnet": "172.30.3.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                },
-                "ntp_servers": [
-                  "time.cloudflare.com"
-                ],
-                "snat_cfg": {
-                  "ip": "198.51.100.25",
-                  "first_port": 0,
-                  "last_port": 16383
-                }
-              }
-            },
-            {
-              "id": "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
-              "underlay_address": "fd00:1122:3344:101::7",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::7]:32345",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
-              "underlay_address": "fd00:1122:3344:101::3",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:101::3]:32221",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "e668e3f5-5457-4689-97da-b2af537787ac",
-              "underlay_address": "fd00:1122:3344:101::b",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::b]:32345",
-                "dataset": {
-                  "pool_name": "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
-              "underlay_address": "fd00:1122:3344:101::9",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b"
-                }
-              }
-            },
-            {
-              "id": "f8c79ba4-194a-4cfe-8090-5315e297b780",
-              "underlay_address": "fd00:1122:3344:1::1",
-              "zone_type": {
-                "type": "internal_dns",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                },
-                "dns_address": "[fd00:1122:3344:1::1]:53",
-                "gz_address": "fd00:1122:3344:1::2",
-                "gz_address_index": 0,
-                "http_address": "[fd00:1122:3344:1::1]:5353"
-              }
-            },
-            {
-              "id": "fa5e0176-b4a8-4598-aa31-40fd562d246c",
-              "underlay_address": "fd00:1122:3344:101::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            }
-          ]
-        }
-      },
-      "zones_in_service": [
-        "084cac6e-44b5-4333-a2ff-b0993b534898",
-        "0d459323-e414-4f32-9944-76c7331da622",
-        "0db88baf-911d-47bb-af71-3aebd7c96934",
-        "11c527c5-0598-428e-9270-be5b74d3461b",
-        "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
-        "20be9299-495d-428d-b315-694c2c4b64af",
-        "213d7dd3-95a4-45a7-99ca-e4899991dab4",
-        "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
-        "2902972b-3ae4-489d-b937-6480352e214a",
-        "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
-        "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
-        "2ee832de-af75-47ac-abac-f91304857ca4",
-        "30050a7e-7fc3-4965-b07b-1635212393eb",
-        "40c47ead-3be8-4724-a8ef-b66233fc5c65",
-        "425d9d2f-72ea-4f53-80b6-04e002771cdb",
-        "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
-        "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
-        "57c712a6-6429-44b7-a9b2-9957cb9054d4",
-        "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
-        "73916098-788f-48d8-b745-e13fdcfb771b",
-        "7611a549-effc-41b6-b6cb-c03e0c70a31f",
-        "8494befe-6df6-42d1-9baa-599ba4228430",
-        "8d452bdd-e8b8-4108-a690-46fc59197db0",
-        "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
-        "953019c0-8e85-4417-b231-d42820470125",
-        "99c635b9-8e97-414a-8316-8261d2d767d0",
-        "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
-        "9bb513e3-7505-416c-aa70-b26f1152bbdc",
-        "9c14c879-137b-4f5b-8068-1292bffc83ed",
-        "9cf90589-5da1-4b42-b09b-d784e4134637",
-        "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
-        "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
-        "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
-        "d48b2dfa-c749-47dd-98b6-3f478cd63934",
-        "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-        "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
-        "e668e3f5-5457-4689-97da-b2af537787ac",
-        "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-        "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
-        "f8c79ba4-194a-4cfe-8090-5315e297b780",
-        "fa5e0176-b4a8-4598-aa31-40fd562d246c",
-        "fd003bee-818a-4c95-aae5-e378ac601522"
-      ],
-      "parent_blueprint_id": null,
-      "internal_dns_version": 1,
-      "external_dns_version": 1,
-      "time_created": "2024-03-12T19:09:41.953854Z",
-      "creator": "0d459323-e414-4f32-9944-76c7331da622",
-      "comment": "from collection 3f61b723-cfe7-40ec-b22c-e3e1f35325f9"
-    },
-    {
-      "id": "cd34e87b-d821-4768-b32f-f41c28db835e",
-      "omicron_zones": {
-        "1f34b5cc-6dac-40cc-9ea1-95f32c33e59b": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "0db88baf-911d-47bb-af71-3aebd7c96934",
-              "underlay_address": "fd00:1122:3344:103::4",
-              "zone_type": {
-                "type": "nexus",
-                "external_dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "external_ip": "198.51.100.22",
-                "external_tls": false,
-                "internal_address": "[fd00:1122:3344:103::4]:12221",
-                "nic": {
-                  "id": "8847a09c-cabc-429b-bf46-66d5e2e1140c",
-                  "kind": {
-                    "type": "service",
-                    "id": "0db88baf-911d-47bb-af71-3aebd7c96934"
-                  },
-                  "name": "nexus-0db88baf-911d-47bb-af71-3aebd7c96934",
-                  "ip": "172.30.2.5",
-                  "mac": "A8:40:25:FF:CC:DA",
-                  "subnet": "172.30.2.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                }
-              }
-            },
-            {
-              "id": "213d7dd3-95a4-45a7-99ca-e4899991dab4",
-              "underlay_address": "fd00:1122:3344:103::6",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::6]:32345",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                }
-              }
-            },
-            {
-              "id": "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
-              "underlay_address": "fd00:1122:3344:3::1",
-              "zone_type": {
-                "type": "internal_dns",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                },
-                "dns_address": "[fd00:1122:3344:3::1]:53",
-                "gz_address": "fd00:1122:3344:3::2",
-                "gz_address_index": 2,
-                "http_address": "[fd00:1122:3344:3::1]:5353"
-              }
-            },
-            {
-              "id": "40c47ead-3be8-4724-a8ef-b66233fc5c65",
-              "underlay_address": "fd00:1122:3344:103::7",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::7]:32345",
-                "dataset": {
-                  "pool_name": "oxp_24b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                }
-              }
-            },
-            {
-              "id": "425d9d2f-72ea-4f53-80b6-04e002771cdb",
-              "underlay_address": "fd00:1122:3344:103::9",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_e4b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                }
-              }
-            },
-            {
-              "id": "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
-              "underlay_address": "fd00:1122:3344:103::8",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_d452a7f5-b528-40fe-80ff-4e4189e2d52b"
-                }
-              }
-            },
-            {
-              "id": "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
-              "underlay_address": "fd00:1122:3344:103::5",
-              "zone_type": {
-                "type": "clickhouse",
-                "address": "[fd00:1122:3344:103::5]:8123",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                }
-              }
-            },
-            {
-              "id": "7611a549-effc-41b6-b6cb-c03e0c70a31f",
-              "underlay_address": "fd00:1122:3344:103::b",
-              "zone_type": {
-                "type": "internal_ntp",
-                "address": "[fd00:1122:3344:103::b]:123",
-                "dns_servers": [
-                  "fd00:1122:3344:1::1",
-                  "fd00:1122:3344:2::1",
-                  "fd00:1122:3344:3::1"
-                ],
-                "ntp_servers": [
-                  "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
-                  "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
-                ]
-              }
-            },
-            {
-              "id": "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
-              "underlay_address": "fd00:1122:3344:103::3",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:103::3]:32221",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                }
-              }
-            },
-            {
-              "id": "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
-              "underlay_address": "fd00:1122:3344:103::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:103::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_f4b4dc85-ab45-49fb-a4b4-d351ae214c03"
-                }
-              }
-            }
-          ]
-        },
-        "67a1268b-52d3-4159-b3ed-7a0777f1f340": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
-              "underlay_address": "fd00:1122:3344:104::b",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:104::b]:32345",
-                "dataset": {
-                  "pool_name": "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
-              "underlay_address": "fd00:1122:3344:104::4",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:104::4]:32221",
-                "dataset": {
-                  "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "57c712a6-6429-44b7-a9b2-9957cb9054d4",
-              "underlay_address": "fd00:1122:3344:104::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:104::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
-              "underlay_address": "fd00:1122:3344:104::7",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:104::7]:32345",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "73916098-788f-48d8-b745-e13fdcfb771b",
-              "underlay_address": "fd00:1122:3344:104::c",
-              "zone_type": {
-                "type": "internal_ntp",
-                "address": "[fd00:1122:3344:104::c]:123",
-                "dns_servers": [
-                  "fd00:1122:3344:1::1",
-                  "fd00:1122:3344:2::1",
-                  "fd00:1122:3344:3::1"
-                ],
-                "ntp_servers": [
-                  "8494befe-6df6-42d1-9baa-599ba4228430.host.control-plane.oxide.internal",
-                  "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded.host.control-plane.oxide.internal"
-                ]
-              }
-            },
-            {
-              "id": "9bb513e3-7505-416c-aa70-b26f1152bbdc",
-              "underlay_address": "fd00:1122:3344:104::9",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:104::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b"
-                }
-              }
-            },
-            {
-              "id": "9c14c879-137b-4f5b-8068-1292bffc83ed",
-              "underlay_address": "fd00:1122:3344:104::3",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:104::3]:32221",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "9cf90589-5da1-4b42-b09b-d784e4134637",
-              "underlay_address": "fd00:1122:3344:104::5",
-              "zone_type": {
-                "type": "nexus",
-                "external_dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "external_ip": "198.51.100.23",
-                "external_tls": false,
-                "internal_address": "[fd00:1122:3344:104::5]:12221",
-                "nic": {
-                  "id": "fa76e7e1-dee6-4574-b71f-e07ea1cffd08",
-                  "kind": {
-                    "type": "service",
-                    "id": "9cf90589-5da1-4b42-b09b-d784e4134637"
-                  },
-                  "name": "nexus-9cf90589-5da1-4b42-b09b-d784e4134637",
-                  "ip": "172.30.2.6",
-                  "mac": "A8:40:25:FF:C4:6F",
-                  "subnet": "172.30.2.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                }
-              }
-            },
-            {
-              "id": "d48b2dfa-c749-47dd-98b6-3f478cd63934",
-              "underlay_address": "fd00:1122:3344:104::6",
-              "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:104::6]:17000"
-              }
-            },
-            {
-              "id": "fd003bee-818a-4c95-aae5-e378ac601522",
-              "underlay_address": "fd00:1122:3344:104::8",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:104::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            }
-          ]
-        },
-        "77851655-10a5-4e80-88b0-a001473cab7e": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "084cac6e-44b5-4333-a2ff-b0993b534898",
-              "underlay_address": "fd00:1122:3344:102::8",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_24b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "11c527c5-0598-428e-9270-be5b74d3461b",
-              "underlay_address": "fd00:1122:3344:102::3",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:102::3]:32221",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "20be9299-495d-428d-b315-694c2c4b64af",
-              "underlay_address": "fd00:1122:3344:102::6",
-              "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:102::6]:17000"
-              }
-            },
-            {
-              "id": "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
-              "underlay_address": "fd00:1122:3344:102::b",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::b]:32345",
-                "dataset": {
-                  "pool_name": "oxp_f4b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "8d452bdd-e8b8-4108-a690-46fc59197db0",
-              "underlay_address": "fd00:1122:3344:102::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_e4b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "953019c0-8e85-4417-b231-d42820470125",
-              "underlay_address": "fd00:1122:3344:102::5",
-              "zone_type": {
-                "type": "oximeter",
-                "address": "[fd00:1122:3344:102::5]:12223"
-              }
-            },
-            {
-              "id": "99c635b9-8e97-414a-8316-8261d2d767d0",
-              "underlay_address": "fd00:1122:3344:102::7",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::7]:32345",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
-              "underlay_address": "fd00:1122:3344:2::1",
-              "zone_type": {
-                "type": "internal_dns",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                },
-                "dns_address": "[fd00:1122:3344:2::1]:53",
-                "gz_address": "fd00:1122:3344:2::2",
-                "gz_address_index": 1,
-                "http_address": "[fd00:1122:3344:2::1]:5353"
-              }
-            },
-            {
-              "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-              "underlay_address": "fd00:1122:3344:102::c",
-              "zone_type": {
-                "type": "boundary_ntp",
-                "address": "[fd00:1122:3344:102::c]:123",
-                "dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "nic": {
-                  "id": "659f33f7-e7bd-4875-9140-bf8a8b2d964a",
-                  "kind": {
-                    "type": "service",
-                    "id": "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded"
-                  },
-                  "name": "ntp-d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-                  "ip": "172.30.3.6",
-                  "mac": "A8:40:25:FF:B1:99",
-                  "subnet": "172.30.3.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                },
-                "ntp_servers": [
-                  "time.cloudflare.com"
-                ],
-                "snat_cfg": {
-                  "ip": "198.51.100.26",
-                  "first_port": 16384,
-                  "last_port": 32767
-                }
-              }
-            },
-            {
-              "id": "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
-              "underlay_address": "fd00:1122:3344:102::9",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:102::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_d462a7f6-b628-40fe-80ff-4e4189e2d62b"
-                }
-              }
-            },
-            {
-              "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-              "underlay_address": "fd00:1122:3344:102::4",
-              "zone_type": {
-                "type": "external_dns",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc86-ab46-49fb-a4b4-d361ae214c03"
-                },
-                "dns_address": "198.51.100.21:53",
-                "http_address": "[fd00:1122:3344:102::4]:5353",
-                "nic": {
-                  "id": "d4dd2f7a-8a3b-405e-8444-62f99a754a8a",
-                  "kind": {
-                    "type": "service",
-                    "id": "f0ed9838-6f1b-4646-a478-9f74f2e6e802"
-                  },
-                  "name": "external-dns-f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-                  "ip": "172.30.1.6",
-                  "mac": "A8:40:25:FF:F4:5C",
-                  "subnet": "172.30.1.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                }
-              }
-            }
-          ]
-        },
-        "f122b3a2-e383-4e42-b2a5-74a69a9d110a": {
-          "generation": 5,
-          "zones": [
-            {
-              "id": "0d459323-e414-4f32-9944-76c7331da622",
-              "underlay_address": "fd00:1122:3344:101::5",
-              "zone_type": {
-                "type": "nexus",
-                "external_dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "external_ip": "198.51.100.24",
-                "external_tls": false,
-                "internal_address": "[fd00:1122:3344:101::5]:12221",
-                "nic": {
-                  "id": "514902b8-7a1a-42a3-95a6-d00861833b5c",
-                  "kind": {
-                    "type": "service",
-                    "id": "0d459323-e414-4f32-9944-76c7331da622"
-                  },
-                  "name": "nexus-0d459323-e414-4f32-9944-76c7331da622",
-                  "ip": "172.30.2.7",
-                  "mac": "A8:40:25:FF:DE:AC",
-                  "subnet": "172.30.2.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                }
-              }
-            },
-            {
-              "id": "2902972b-3ae4-489d-b937-6480352e214a",
-              "underlay_address": "fd00:1122:3344:101::8",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::8]:32345",
-                "dataset": {
-                  "pool_name": "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "2ee832de-af75-47ac-abac-f91304857ca4",
-              "underlay_address": "fd00:1122:3344:101::4",
-              "zone_type": {
-                "type": "external_dns",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                },
-                "dns_address": "198.51.100.20:53",
-                "http_address": "[fd00:1122:3344:101::4]:5353",
-                "nic": {
-                  "id": "fadff4ab-62d7-466a-9203-04080e4078ed",
-                  "kind": {
-                    "type": "service",
-                    "id": "2ee832de-af75-47ac-abac-f91304857ca4"
-                  },
-                  "name": "external-dns-2ee832de-af75-47ac-abac-f91304857ca4",
-                  "ip": "172.30.1.5",
-                  "mac": "A8:40:25:FF:B4:93",
-                  "subnet": "172.30.1.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                }
-              }
-            },
-            {
-              "id": "30050a7e-7fc3-4965-b07b-1635212393eb",
-              "underlay_address": "fd00:1122:3344:101::6",
-              "zone_type": {
-                "type": "crucible_pantry",
-                "address": "[fd00:1122:3344:101::6]:17000"
-              }
-            },
-            {
-              "id": "8494befe-6df6-42d1-9baa-599ba4228430",
-              "underlay_address": "fd00:1122:3344:101::c",
-              "zone_type": {
-                "type": "boundary_ntp",
-                "address": "[fd00:1122:3344:101::c]:123",
-                "dns_servers": [
-                  "1.1.1.1",
-                  "9.9.9.9"
-                ],
-                "nic": {
-                  "id": "23041748-e3a6-4846-930d-5a348c316e91",
-                  "kind": {
-                    "type": "service",
-                    "id": "8494befe-6df6-42d1-9baa-599ba4228430"
-                  },
-                  "name": "ntp-8494befe-6df6-42d1-9baa-599ba4228430",
-                  "ip": "172.30.3.5",
-                  "mac": "A8:40:25:FF:D1:C4",
-                  "subnet": "172.30.3.0/24",
-                  "vni": 100,
-                  "primary": true,
-                  "slot": 0
-                },
-                "ntp_servers": [
-                  "time.cloudflare.com"
-                ],
-                "snat_cfg": {
-                  "ip": "198.51.100.25",
-                  "first_port": 0,
-                  "last_port": 16383
-                }
-              }
-            },
-            {
-              "id": "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
-              "underlay_address": "fd00:1122:3344:101::7",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::7]:32345",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
-              "underlay_address": "fd00:1122:3344:101::3",
-              "zone_type": {
-                "type": "cockroach_db",
-                "address": "[fd00:1122:3344:101::3]:32221",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "e668e3f5-5457-4689-97da-b2af537787ac",
-              "underlay_address": "fd00:1122:3344:101::b",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::b]:32345",
-                "dataset": {
-                  "pool_name": "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            },
-            {
-              "id": "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
-              "underlay_address": "fd00:1122:3344:101::9",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::9]:32345",
-                "dataset": {
-                  "pool_name": "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b"
-                }
-              }
-            },
-            {
-              "id": "f8c79ba4-194a-4cfe-8090-5315e297b780",
-              "underlay_address": "fd00:1122:3344:1::1",
-              "zone_type": {
-                "type": "internal_dns",
-                "dataset": {
-                  "pool_name": "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                },
-                "dns_address": "[fd00:1122:3344:1::1]:53",
-                "gz_address": "fd00:1122:3344:1::2",
-                "gz_address_index": 0,
-                "http_address": "[fd00:1122:3344:1::1]:5353"
-              }
-            },
-            {
-              "id": "fa5e0176-b4a8-4598-aa31-40fd562d246c",
-              "underlay_address": "fd00:1122:3344:101::a",
-              "zone_type": {
-                "type": "crucible",
-                "address": "[fd00:1122:3344:101::a]:32345",
-                "dataset": {
-                  "pool_name": "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03"
-                }
-              }
-            }
-          ]
-        }
-      },
-      "zones_in_service": [
-        "084cac6e-44b5-4333-a2ff-b0993b534898",
-        "0d459323-e414-4f32-9944-76c7331da622",
-        "0db88baf-911d-47bb-af71-3aebd7c96934",
-        "11c527c5-0598-428e-9270-be5b74d3461b",
-        "1ebea0c3-1bfd-472c-9f84-ac9e23b32707",
-        "20be9299-495d-428d-b315-694c2c4b64af",
-        "213d7dd3-95a4-45a7-99ca-e4899991dab4",
-        "218f36a5-9e51-4b88-ad1a-da4d85c246c4",
-        "2902972b-3ae4-489d-b937-6480352e214a",
-        "2b29cb95-f8c6-4aff-acc0-1c0a627be392",
-        "2e9dfa8f-5371-48e5-b7f7-62a6af8283c1",
-        "2ee832de-af75-47ac-abac-f91304857ca4",
-        "30050a7e-7fc3-4965-b07b-1635212393eb",
-        "40c47ead-3be8-4724-a8ef-b66233fc5c65",
-        "425d9d2f-72ea-4f53-80b6-04e002771cdb",
-        "46a050b6-37b8-47b9-b4ae-ccb736ecd6d5",
-        "4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd",
-        "57c712a6-6429-44b7-a9b2-9957cb9054d4",
-        "59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9",
-        "73916098-788f-48d8-b745-e13fdcfb771b",
-        "7611a549-effc-41b6-b6cb-c03e0c70a31f",
-        "8494befe-6df6-42d1-9baa-599ba4228430",
-        "8d452bdd-e8b8-4108-a690-46fc59197db0",
-        "92cba955-3a5a-4d82-b0f2-30e89d4556d1",
-        "953019c0-8e85-4417-b231-d42820470125",
-        "99c635b9-8e97-414a-8316-8261d2d767d0",
-        "9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b",
-        "9bb513e3-7505-416c-aa70-b26f1152bbdc",
-        "9c14c879-137b-4f5b-8068-1292bffc83ed",
-        "9cf90589-5da1-4b42-b09b-d784e4134637",
-        "af7ccead-2bd8-4a8a-9fbf-3b788d180139",
-        "cbe1abfa-0063-44b2-9ce7-7ad76b403a94",
-        "d4428e41-ad7b-41f0-bde6-7c7c955f20c1",
-        "d48b2dfa-c749-47dd-98b6-3f478cd63934",
-        "d52e797a-8fff-4e1e-a5ff-0ab8c8293ded",
-        "dc7f703a-27d1-4802-b85d-5629ddb2d67c",
-        "e668e3f5-5457-4689-97da-b2af537787ac",
-        "f0ed9838-6f1b-4646-a478-9f74f2e6e802",
-        "f32fd972-7f40-4c58-9e00-7fc5c44273f4",
-        "f8c79ba4-194a-4cfe-8090-5315e297b780",
-        "fa5e0176-b4a8-4598-aa31-40fd562d246c",
-        "fd003bee-818a-4c95-aae5-e378ac601522"
-      ],
-      "parent_blueprint_id": null,
-      "internal_dns_version": 1,
-      "external_dns_version": 1,
-      "time_created": "2024-03-12T19:11:49.876621Z",
-      "creator": "0d459323-e414-4f32-9944-76c7331da622",
-      "comment": "from collection 5e1d6ae0-c5b4-4884-ac41-680cd4f5762d"
-    }
-  ]
-}
diff --git a/dev-tools/reconfigurator-cli/tests/test_basic.rs b/dev-tools/reconfigurator-cli/tests/test_basic.rs
index f70fcb1366..6048aece1b 100644
--- a/dev-tools/reconfigurator-cli/tests/test_basic.rs
+++ b/dev-tools/reconfigurator-cli/tests/test_basic.rs
@@ -2,10 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use anyhow::Context;
-use camino::Utf8Path;
 use expectorate::assert_contents;
-use nexus_types::deployment::UnstableReconfiguratorState;
 use omicron_test_utils::dev::test_cmds::assert_exit_code;
 use omicron_test_utils::dev::test_cmds::path_to_executable;
 use omicron_test_utils::dev::test_cmds::redact_variable;
@@ -28,88 +25,3 @@ fn test_basic() {
     assert_contents("tests/output/cmd-stdout", &stdout_text);
     assert_contents("tests/output/cmd-stderr", &stderr_text);
 }
-
-// To do more interesting testing, we need more interesting data than we can
-// currently cons up in the REPL environment.  This test uses a file saved from
-// the "madrid" rig using `omdb db reconfigurator-save`.
-//
-// This test may be broken if the format of this file changes (see the
-// UnstableReconfiguratorState struct).  To fix it, generate a new file from a
-// real system with at least one inventory collection and at least two
-// blueprints.  You will also need to set EXPECTORATE=overwrite when running
-// this for the first time with the new input file in order to generate the new
-// output file.  Check it.
-//
-// If this test becomes too burdensome, we could drop it.  But for now it's
-// useful to have a smoke test for the command.
-#[test]
-fn test_complex() {
-    let input_path = Utf8Path::new("tests/input/complex.json");
-    let input_file = std::fs::File::open(input_path)
-        .with_context(|| format!("open {:?}", input_path))
-        .unwrap();
-    let input_data: UnstableReconfiguratorState =
-        serde_json::from_reader(&input_file)
-            .with_context(|| format!("read {:?}", input_path))
-            .unwrap();
-    assert!(
-        input_data.collections.len() > 0,
-        "input file must have at least one inventory collection"
-    );
-    assert!(
-        input_data.blueprints.len() > 1,
-        "input file must have at least two blueprints"
-    );
-
-    let collection = input_data.collections.iter().next().unwrap().id;
-    let mut blueprints = input_data.blueprints.iter().rev();
-    let blueprint2 = blueprints.next().unwrap().id;
-    let blueprint1 = blueprints.next().unwrap().id;
-    let tmpdir = camino_tempfile::tempdir().expect("failed to create tmpdir");
-    let tmpfile = tmpdir.path().join("cmds.txt");
-
-    // We construct the list of commands dynamically to avoid having to re-do it
-    // when the input file has to be regenerated.
-    let input_cmds = [
-        "sled-list",
-        "inventory-list",
-        "blueprint-list",
-        &format!("file-contents {}", input_path),
-        &format!("load {} {}", input_path, collection),
-        "sled-list",
-        "inventory-list",
-        "blueprint-list",
-        &format!("blueprint-show {}", blueprint2),
-        &format!("blueprint-diff-inventory {} {}", collection, blueprint2),
-        &format!("blueprint-diff {} {}", blueprint1, blueprint2),
-        &format!("blueprint-diff {} {}", blueprint2, blueprint1),
-        "sled-add dde1c0e2-b10d-4621-b420-f179f7a7a00a",
-        &format!("blueprint-plan {} {}", blueprint2, collection),
-    ]
-    .into_iter()
-    .map(|s| format!("{}\n", s))
-    .collect::<Vec<_>>()
-    .join("\n");
-
-    println!("will execute commands:\n{}", input_cmds);
-
-    std::fs::write(&tmpfile, &input_cmds)
-        .with_context(|| format!("write {:?}", &tmpfile))
-        .unwrap();
-
-    let exec = Exec::cmd(path_to_cli()).arg(&tmpfile);
-    let (exit_status, stdout_text, stderr_text) = run_command(exec);
-    assert_exit_code(exit_status, EXIT_SUCCESS, &stderr_text);
-
-    // This is a much lighter form of redaction than `redact_variable()` does.
-    let stdout_text =
-        regex::Regex::new(r"generated blueprint .* based on parent blueprint")
-            .unwrap()
-            .replace_all(
-                &stdout_text,
-                "generated blueprint REDACTED_UUID based on parent blueprint",
-            )
-            .to_string();
-
-    assert_contents("tests/output/cmd-complex-stdout", &stdout_text);
-}

From 272398c15a2f95f97587040a93bc3b856e60e233 Mon Sep 17 00:00:00 2001
From: Nils Nieuwejaar <nils@oxidecomputer.com>
Date: Thu, 14 Mar 2024 22:26:34 -0400
Subject: [PATCH 119/157] bump lldp to pick up fix for a bench gimlet SMF crash
 (#5269)

---
 package-manifest.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/package-manifest.toml b/package-manifest.toml
index a3e1fa133d..ef4e6dfb06 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -557,8 +557,8 @@ output.intermediate_only = true
 service_name = "lldp"
 source.type = "prebuilt"
 source.repo = "lldp"
-source.commit = "e165090e393df90874329265799480e221884236"
-source.sha256 = "004a6488a948d894fa203222aa42e159e1ae12c13751607852302d111e2c58e6"
+source.commit = "daab8580d6994c29f3c45ffa5a76b1de765b30da"
+source.sha256 = "15a9068fdd4521a77f7d32c80d1e6d3a6a4ad15accab77641ed8f14175ae0db6"
 output.type = "zone"
 output.intermediate_only = true
 

From e825a085075421c4f6bc968f744f08594ce70a05 Mon Sep 17 00:00:00 2001
From: "Joshua M. Clulow" <jmc@oxide.computer>
Date: Fri, 15 Mar 2024 01:34:20 -0700
Subject: [PATCH 120/157] zone bundle tests fail on ZFS pool with 4K sectors
 (ashift=12) (#5260)

---
 sled-agent/src/zone_bundle.rs | 97 ++++++++++++++++++++++++-----------
 1 file changed, 68 insertions(+), 29 deletions(-)

diff --git a/sled-agent/src/zone_bundle.rs b/sled-agent/src/zone_bundle.rs
index 4c4cbf5490..7b0d9b8071 100644
--- a/sled-agent/src/zone_bundle.rs
+++ b/sled-agent/src/zone_bundle.rs
@@ -2176,9 +2176,12 @@ mod illumos_tests {
     use super::ZoneBundler;
     use super::ZFS;
     use anyhow::Context;
+    use chrono::DateTime;
     use chrono::TimeZone;
+    use chrono::Timelike;
     use chrono::Utc;
     use illumos_utils::zpool::ZpoolName;
+    use rand::RngCore;
     use sled_storage::disk::RawDisk;
     use sled_storage::disk::SyntheticDisk;
     use sled_storage::manager::{FakeStorageManager, StorageHandle};
@@ -2186,6 +2189,43 @@ mod illumos_tests {
     use slog::Logger;
     use tokio::process::Command;
 
+    /// An iterator that returns the date of consecutive days beginning with 1st
+    /// January 2020. The time portion of each returned date will be fixed at
+    /// midnight in UTC.
+    struct DaysOfOurBundles {
+        next: DateTime<Utc>,
+    }
+
+    impl DaysOfOurBundles {
+        fn new() -> DaysOfOurBundles {
+            DaysOfOurBundles {
+                next: Utc
+                    // Set the start date to 1st January 2020:
+                    .with_ymd_and_hms(2020, 1, 1, 0, 0, 0)
+                    .single()
+                    .unwrap(),
+            }
+        }
+    }
+
+    impl Iterator for DaysOfOurBundles {
+        type Item = DateTime<Utc>;
+
+        fn next(&mut self) -> Option<Self::Item> {
+            let out = self.next;
+
+            // We promise that all returned dates are aligned with midnight UTC:
+            assert_eq!(out.hour(), 0);
+            assert_eq!(out.minute(), 0);
+            assert_eq!(out.second(), 0);
+
+            self.next =
+                self.next.checked_add_days(chrono::Days::new(1)).unwrap();
+
+            Some(out)
+        }
+    }
+
     #[tokio::test]
     async fn test_zfs_quota() {
         let path =
@@ -2327,9 +2367,14 @@ mod illumos_tests {
 
     // Quota applied to test datasets.
     //
-    // This needs to be at least this big lest we get "out of space" errors when
-    // creating. Not sure where those come from, but could be ZFS overhead.
-    const TEST_QUOTA: u64 = 1024 * 32;
+    // ZFS will not allow a quota to be set that is smaller than the bytes
+    // presently stored for that dataset, either at dataset creation time or
+    // later by setting the "quota" property.  The exact minimum number of bytes
+    // depends on many factors, including the block size of the underlying pool;
+    // i.e., the "ashift" value.  An empty dataset is unlikely to contain more
+    // than one megabyte of overhead, so use that as a conservative test size to
+    // avoid issues.
+    const TEST_QUOTA: u64 = 1024 * 1024;
 
     async fn create_test_dataset(
         id: &Uuid,
@@ -2441,9 +2486,7 @@ mod illumos_tests {
         // back.
         let info = insert_fake_bundle(
             &paths[0],
-            2020,
-            1,
-            1,
+            DaysOfOurBundles::new().next().unwrap(),
             ZoneBundleCause::ExplicitRequest,
         )
         .await?;
@@ -2514,7 +2557,7 @@ mod illumos_tests {
             .await
             .context("failed to update cleanup context")?;
 
-        let mut day = 1;
+        let mut days = DaysOfOurBundles::new();
         let mut info = Vec::new();
         let mut utilization = ctx.bundler.utilization().await?;
         loop {
@@ -2527,13 +2570,10 @@ mod illumos_tests {
             }
             let it = insert_fake_bundle(
                 &ctx.resource_wrapper.dirs[0],
-                2020,
-                1,
-                day,
+                days.next().unwrap(),
                 ZoneBundleCause::ExplicitRequest,
             )
             .await?;
-            day += 1;
             info.push(it);
             utilization = ctx.bundler.utilization().await?;
         }
@@ -2581,20 +2621,17 @@ mod illumos_tests {
     async fn test_list_with_filter_body(
         ctx: CleanupTestContext,
     ) -> anyhow::Result<()> {
-        let mut day = 1;
+        let mut days = DaysOfOurBundles::new();
         let mut info = Vec::new();
         const N_BUNDLES: usize = 3;
         for i in 0..N_BUNDLES {
             let it = insert_fake_bundle_with_zone_name(
                 &ctx.resource_wrapper.dirs[0],
-                2020,
-                1,
-                day,
+                days.next().unwrap(),
                 ZoneBundleCause::ExplicitRequest,
                 format!("oxz_whatever_{i}").as_str(),
             )
             .await?;
-            day += 1;
             info.push(it);
         }
 
@@ -2641,16 +2678,12 @@ mod illumos_tests {
 
     async fn insert_fake_bundle(
         dir: &Utf8Path,
-        year: i32,
-        month: u32,
-        day: u32,
+        time_created: DateTime<Utc>,
         cause: ZoneBundleCause,
     ) -> anyhow::Result<ZoneBundleInfo> {
         insert_fake_bundle_with_zone_name(
             dir,
-            year,
-            month,
-            day,
+            time_created,
             cause,
             "oxz_whatever",
         )
@@ -2659,21 +2692,20 @@ mod illumos_tests {
 
     async fn insert_fake_bundle_with_zone_name(
         dir: &Utf8Path,
-        year: i32,
-        month: u32,
-        day: u32,
+        time_created: DateTime<Utc>,
         cause: ZoneBundleCause,
         zone_name: &str,
     ) -> anyhow::Result<ZoneBundleInfo> {
+        assert_eq!(time_created.hour(), 0);
+        assert_eq!(time_created.minute(), 0);
+        assert_eq!(time_created.second(), 0);
+
         let metadata = ZoneBundleMetadata {
             id: ZoneBundleId {
                 zone_name: String::from(zone_name),
                 bundle_id: uuid::Uuid::new_v4(),
             },
-            time_created: Utc
-                .with_ymd_and_hms(year, month, day, 0, 0, 0)
-                .single()
-                .context("invalid year/month/day")?,
+            time_created,
             cause,
             version: 0,
         };
@@ -2704,6 +2736,13 @@ mod illumos_tests {
             super::ZONE_BUNDLE_METADATA_FILENAME,
             contents.as_bytes(),
         )?;
+
+        // Inject some ~incompressible ballast to ensure the bundles are, though
+        // fake, not also microscopic:
+        let mut ballast = vec![0; 64 * 1024];
+        rand::thread_rng().fill_bytes(&mut ballast);
+        super::insert_data(&mut builder, "ballast.bin", &ballast)?;
+
         let _ = builder.into_inner().context("failed to finish tarball")?;
         let bytes = tokio::fs::metadata(&path).await?.len();
         Ok(ZoneBundleInfo { metadata, path, bytes })

From cc59effc9938d0c1f7f39020aa69281a2790cdfe Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Fri, 15 Mar 2024 10:23:58 -0400
Subject: [PATCH 121/157] RSS <-> Nexus handoff: Set an initial (disabled)
 target blueprint (#5244)

I would like to test that after an RSS handoff, the initial blueprint we
created in RSS and set in the Nexus handoff matches the blueprint we
would generate from the first inventory collections. I can do that by
hand on `madrid`, but I'm not sure there's a good way to do that in an
automated test. Is this something I could lean on a4x2 to check?

This does not modify the `services` table or the `services` field of the
RSS handoff. Removing those will come in a followup PR that will need an
accompanying note for deployed systems (i.e., instructions to set a
blueprint so the current users of `services` will continue to function
by checking the current target blueprint).

Closes #5222.
---
 Cargo.lock                                    |   3 +
 common/Cargo.toml                             |   1 +
 common/src/api/external/error.rs              |  11 +-
 common/src/api/external/mod.rs                |  30 ++-
 nexus/db-queries/Cargo.toml                   |   1 +
 .../db-queries/src/db/datastore/deployment.rs |  30 ++-
 nexus/db-queries/src/db/datastore/rack.rs     | 129 ++++++++++---
 nexus/reconfigurator/execution/src/dns.rs     |  63 +------
 .../planning/src/blueprint_builder.rs         |   4 +-
 nexus/reconfigurator/planning/src/planner.rs  |   2 +-
 nexus/src/app/background/blueprint_load.rs    |  29 +--
 nexus/src/app/rack.rs                         |  14 +-
 nexus/src/lib.rs                              |   3 +
 nexus/test-interface/src/lib.rs               |   2 +
 nexus/test-utils/src/lib.rs                   |  43 ++++-
 nexus/types/src/internal_api/params.rs        |   3 +
 openapi/nexus-internal.json                   |   9 +
 sled-agent/Cargo.toml                         |   1 +
 sled-agent/src/params.rs                      |   8 +-
 sled-agent/src/rack_setup/mod.rs              |   2 +
 sled-agent/src/rack_setup/service.rs          | 178 ++++++++++++++----
 sled-agent/src/services_migration.rs          |   7 +-
 sled-agent/src/sim/server.rs                  |  92 ++++++---
 23 files changed, 482 insertions(+), 183 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 84f365e2f0..2299d1df62 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4508,6 +4508,7 @@ dependencies = [
  "serde_with",
  "sled-agent-client",
  "slog",
+ "slog-error-chain",
  "static_assertions",
  "steno",
  "strum 0.26.1",
@@ -5071,6 +5072,7 @@ dependencies = [
  "serde_urlencoded",
  "serde_with",
  "slog",
+ "slog-error-chain",
  "strum 0.26.1",
  "test-strategy",
  "thiserror",
@@ -5437,6 +5439,7 @@ dependencies = [
  "mg-admin-client",
  "nexus-client",
  "nexus-config",
+ "nexus-types",
  "omicron-common",
  "omicron-test-utils",
  "omicron-workspace-hack",
diff --git a/common/Cargo.toml b/common/Cargo.toml
index b16415b828..3acd849b96 100644
--- a/common/Cargo.toml
+++ b/common/Cargo.toml
@@ -34,6 +34,7 @@ serde_human_bytes.workspace = true
 serde_json.workspace = true
 serde_with.workspace = true
 slog.workspace = true
+slog-error-chain.workspace = true
 strum.workspace = true
 test-strategy = { workspace = true, optional = true }
 thiserror.workspace = true
diff --git a/common/src/api/external/error.rs b/common/src/api/external/error.rs
index ef64aa3bbb..6b3b93187f 100644
--- a/common/src/api/external/error.rs
+++ b/common/src/api/external/error.rs
@@ -12,6 +12,7 @@ use dropshot::HttpError;
 use omicron_uuid_kinds::GenericUuid;
 use serde::Deserialize;
 use serde::Serialize;
+use slog_error_chain::SlogInlineError;
 use std::fmt::Display;
 use uuid::Uuid;
 
@@ -26,7 +27,15 @@ use uuid::Uuid;
 /// General best practices for error design apply here.  Where possible, we want
 /// to reuse existing variants rather than inventing new ones to distinguish
 /// cases that no programmatic consumer needs to distinguish.
-#[derive(Clone, Debug, Deserialize, thiserror::Error, PartialEq, Serialize)]
+#[derive(
+    Clone,
+    Debug,
+    Deserialize,
+    thiserror::Error,
+    PartialEq,
+    Serialize,
+    SlogInlineError,
+)]
 pub enum Error {
     /// An object needed as part of this operation was not found.
     #[error("Object (of type {lookup_type:?}) not found: {type_name}")]
diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs
index 58bc82c825..1a355f4929 100644
--- a/common/src/api/external/mod.rs
+++ b/common/src/api/external/mod.rs
@@ -650,16 +650,28 @@ impl From<ByteCount> for i64 {
 pub struct Generation(u64);
 
 impl Generation {
-    pub fn new() -> Generation {
+    pub const fn new() -> Generation {
         Generation(1)
     }
 
-    pub fn next(&self) -> Generation {
+    pub const fn from_u32(value: u32) -> Generation {
+        // `as` is a little distasteful because it allows lossy conversion, but
+        // (a) we know converting `u32` to `u64` will always succeed
+        // losslessly, and (b) it allows to make this function `const`, unlike
+        // if we were to use `u64::from(value)`.
+        Generation(value as u64)
+    }
+
+    pub const fn next(&self) -> Generation {
         // It should technically be an operational error if this wraps or even
         // exceeds the value allowed by an i64.  But it seems unlikely enough to
         // happen in practice that we can probably feel safe with this.
         let next_gen = self.0 + 1;
-        assert!(next_gen <= u64::try_from(i64::MAX).unwrap());
+        // `as` is a little distasteful because it allows lossy conversion, but
+        // (a) we know converting `i64::MAX` to `u64` will always succeed
+        // losslessly, and (b) it allows to make this function `const`, unlike
+        // if we were to use `u64::try_from(i64::MAX).unwrap()`.
+        assert!(next_gen <= i64::MAX as u64);
         Generation(next_gen)
     }
 }
@@ -697,11 +709,21 @@ impl TryFrom<i64> for Generation {
     fn try_from(value: i64) -> Result<Self, Self::Error> {
         Ok(Generation(
             u64::try_from(value)
-                .map_err(|_| anyhow!("generation number too large"))?,
+                .map_err(|_| anyhow!("negative generation number"))?,
         ))
     }
 }
 
+impl TryFrom<u64> for Generation {
+    type Error = anyhow::Error;
+
+    fn try_from(value: u64) -> Result<Self, Self::Error> {
+        i64::try_from(value)
+            .map_err(|_| anyhow!("generation number too large"))?;
+        Ok(Generation(value))
+    }
+}
+
 /// An RFC-1035-compliant hostname.
 #[derive(
     Clone, Debug, Deserialize, Display, Eq, PartialEq, SerializeDisplay,
diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml
index a33ca24c84..9bb7040af9 100644
--- a/nexus/db-queries/Cargo.toml
+++ b/nexus/db-queries/Cargo.toml
@@ -42,6 +42,7 @@ serde_urlencoded.workspace = true
 serde_with.workspace = true
 sled-agent-client.workspace = true
 slog.workspace = true
+slog-error-chain.workspace = true
 static_assertions.workspace = true
 steno.workspace = true
 strum.workspace = true
diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs
index 1a12b9d250..88c0897be2 100644
--- a/nexus/db-queries/src/db/datastore/deployment.rs
+++ b/nexus/db-queries/src/db/datastore/deployment.rs
@@ -81,6 +81,17 @@ impl DataStore {
         &self,
         opctx: &OpContext,
         blueprint: &Blueprint,
+    ) -> Result<(), Error> {
+        let conn = self.pool_connection_authorized(opctx).await?;
+        Self::blueprint_insert_on_connection(&conn, opctx, blueprint).await
+    }
+
+    /// Variant of [Self::blueprint_insert] which may be called from a
+    /// transaction context.
+    pub(crate) async fn blueprint_insert_on_connection(
+        conn: &async_bb8_diesel::Connection<DbConnection>,
+        opctx: &OpContext,
+        blueprint: &Blueprint,
     ) -> Result<(), Error> {
         opctx
             .authorize(authz::Action::Modify, &authz::BLUEPRINT_CONFIG)
@@ -154,8 +165,7 @@ impl DataStore {
         // batch rather than making a bunch of round-trips to the database.
         // We'd do that if we had an interface for doing that with bound
         // parameters, etc.  See oxidecomputer/omicron#973.
-        let pool = self.pool_connection_authorized(opctx).await?;
-        pool.transaction_async(|conn| async move {
+        conn.transaction_async(|conn| async move {
             // Insert the row for the blueprint.
             {
                 use db::schema::blueprint::dsl;
@@ -627,6 +637,18 @@ impl DataStore {
         &self,
         opctx: &OpContext,
         target: BlueprintTarget,
+    ) -> Result<(), Error> {
+        let conn = self.pool_connection_authorized(opctx).await?;
+        Self::blueprint_target_set_current_on_connection(&conn, opctx, target)
+            .await
+    }
+
+    /// Variant of [Self::blueprint_target_set_current] which may be called from
+    /// a transaction context.
+    pub(crate) async fn blueprint_target_set_current_on_connection(
+        conn: &async_bb8_diesel::Connection<DbConnection>,
+        opctx: &OpContext,
+        target: BlueprintTarget,
     ) -> Result<(), Error> {
         opctx
             .authorize(authz::Action::Modify, &authz::BLUEPRINT_CONFIG)
@@ -638,10 +660,8 @@ impl DataStore {
             time_made_target: target.time_made_target,
         };
 
-        let conn = self.pool_connection_authorized(opctx).await?;
-
         query
-            .execute_async(&*conn)
+            .execute_async(conn)
             .await
             .map_err(|e| Error::from(query.decode_error(e)))?;
 
diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs
index fa4d6a4210..e290343978 100644
--- a/nexus/db-queries/src/db/datastore/rack.rs
+++ b/nexus/db-queries/src/db/datastore/rack.rs
@@ -44,6 +44,7 @@ use nexus_db_model::SiloUser;
 use nexus_db_model::SiloUserPasswordHash;
 use nexus_db_model::SledUnderlaySubnetAllocation;
 use nexus_types::deployment::Blueprint;
+use nexus_types::deployment::BlueprintTarget;
 use nexus_types::deployment::OmicronZoneType;
 use nexus_types::external_api::params as external_params;
 use nexus_types::external_api::shared;
@@ -61,6 +62,7 @@ use omicron_common::api::external::LookupType;
 use omicron_common::api::external::ResourceType;
 use omicron_common::api::external::UpdateResult;
 use omicron_common::bail_unless;
+use slog_error_chain::InlineErrorChain;
 use std::net::IpAddr;
 use std::sync::{Arc, OnceLock};
 use uuid::Uuid;
@@ -70,6 +72,7 @@ use uuid::Uuid;
 pub struct RackInit {
     pub rack_id: Uuid,
     pub rack_subnet: IpNetwork,
+    pub blueprint: Blueprint,
     pub services: Vec<internal_params::ServicePutRequest>,
     pub datasets: Vec<Dataset>,
     pub service_ip_pool_ranges: Vec<IpRange>,
@@ -87,6 +90,8 @@ pub struct RackInit {
 enum RackInitError {
     AddingIp(Error),
     AddingNic(Error),
+    BlueprintInsert(Error),
+    BlueprintTargetSet(Error),
     ServiceInsert(Error),
     DatasetInsert { err: AsyncInsertError, zpool_id: Uuid },
     RackUpdate { err: DieselError, rack_id: Uuid },
@@ -128,6 +133,12 @@ impl From<RackInitError> for Error {
             RackInitError::ServiceInsert(err) => Error::internal_error(
                 &format!("failed to insert Service record: {:#}", err),
             ),
+            RackInitError::BlueprintInsert(err) => Error::internal_error(
+                &format!("failed to insert Blueprint: {:#}", err),
+            ),
+            RackInitError::BlueprintTargetSet(err) => Error::internal_error(
+                &format!("failed to insert set target Blueprint: {:#}", err),
+            ),
             RackInitError::RackUpdate { err, rack_id } => {
                 public_error_from_diesel(
                     err,
@@ -520,11 +531,12 @@ impl DataStore {
             Self::allocate_external_ip_on_connection(conn, db_ip)
                 .await
                 .map_err(|err| {
-                    warn!(
+                    error!(
                         log,
                         "Initializing Rack: Failed to allocate \
-                        IP address for {}",
-                        service.kind,
+                         IP address for {}",
+                        service.kind;
+                        "err" => %err,
                     );
                     match err.retryable() {
                         Retryable(e) => RackInitError::Retryable(e),
@@ -585,9 +597,11 @@ impl DataStore {
                 let service_pool = service_pool.clone();
                 async move {
                     let rack_id = rack_init.rack_id;
+                    let blueprint = rack_init.blueprint;
                     let services = rack_init.services;
                     let datasets = rack_init.datasets;
-                    let service_ip_pool_ranges = rack_init.service_ip_pool_ranges;
+                    let service_ip_pool_ranges =
+                        rack_init.service_ip_pool_ranges;
                     let internal_dns = rack_init.internal_dns;
                     let external_dns = rack_init.external_dns;
 
@@ -598,11 +612,16 @@ impl DataStore {
                         .get_result_async(&conn)
                         .await
                         .map_err(|e| {
-                            warn!(log, "Initializing Rack: Rack UUID not found");
+                            error!(
+                                log,
+                                "Initializing Rack: Rack UUID not found";
+                                InlineErrorChain::new(&e),
+                            );
                             err.set(RackInitError::RackUpdate {
                                 err: e,
                                 rack_id,
-                            }).unwrap();
+                            })
+                            .unwrap();
                             DieselError::RollbackTransaction
                         })?;
                     if rack.initialized {
@@ -610,7 +629,7 @@ impl DataStore {
                         return Ok::<_, DieselError>(rack);
                     }
 
-                    // Otherwise, insert services and datasets.
+                    // Otherwise, insert blueprint and datasets.
 
                     // Set up the IP pool for internal services.
                     for range in service_ip_pool_ranges {
@@ -622,15 +641,57 @@ impl DataStore {
                         )
                         .await
                         .map_err(|e| {
-                            warn!(
+                            error!(
                                 log,
-                                "Initializing Rack: Failed to add IP pool range"
+                                "Initializing Rack: Failed to add \
+                                 IP pool range";
+                                &e,
                             );
                             err.set(RackInitError::AddingIp(e)).unwrap();
                             DieselError::RollbackTransaction
                         })?;
                     }
 
+                    // Insert the RSS-generated blueprint.
+                    Self::blueprint_insert_on_connection(
+                        &conn, opctx, &blueprint,
+                    )
+                    .await
+                    .map_err(|e| {
+                        error!(
+                            log,
+                            "Initializing Rack: Failed to insert blueprint";
+                            &e,
+                        );
+                        err.set(RackInitError::BlueprintInsert(e)).unwrap();
+                        DieselError::RollbackTransaction
+                    })?;
+
+                    // Mark the RSS-generated blueprint as the current target,
+                    // DISABLED. We may change this to enabled in the future
+                    // when more of Reconfigurator is automated, but for now we
+                    // require a support operation to enable it.
+                    Self::blueprint_target_set_current_on_connection(
+                        &conn,
+                        opctx,
+                        BlueprintTarget {
+                            target_id: blueprint.id,
+                            enabled: false,
+                            time_made_target: Utc::now(),
+                        },
+                    )
+                    .await
+                    .map_err(|e| {
+                        error!(
+                            log,
+                            "Initializing Rack: Failed to set blueprint \
+                             as target";
+                            &e,
+                        );
+                        err.set(RackInitError::BlueprintTargetSet(e)).unwrap();
+                        DieselError::RollbackTransaction
+                    })?;
+
                     // Allocate records for all services.
                     for service in services {
                         self.rack_populate_service_records(
@@ -650,7 +711,7 @@ impl DataStore {
                     for dataset in datasets {
                         use db::schema::dataset::dsl;
                         let zpool_id = dataset.pool_id;
-                        <Zpool as DatastoreCollection<Dataset>>::insert_resource(
+                        Zpool::insert_resource(
                             zpool_id,
                             diesel::insert_into(dsl::dataset)
                                 .values(dataset.clone())
@@ -670,7 +731,8 @@ impl DataStore {
                             err.set(RackInitError::DatasetInsert {
                                 err: e,
                                 zpool_id,
-                            }).unwrap();
+                            })
+                            .unwrap();
                             DieselError::RollbackTransaction
                         })?;
                     }
@@ -678,20 +740,22 @@ impl DataStore {
 
                     // Insert the initial contents of the internal and external DNS
                     // zones.
-                    Self::load_dns_data(&conn, internal_dns)
-                        .await
-                        .map_err(|e| {
-                            err.set(RackInitError::DnsSerialization(e)).unwrap();
+                    Self::load_dns_data(&conn, internal_dns).await.map_err(
+                        |e| {
+                            err.set(RackInitError::DnsSerialization(e))
+                                .unwrap();
                             DieselError::RollbackTransaction
-                        })?;
+                        },
+                    )?;
                     info!(log, "Populated DNS tables for internal DNS");
 
-                    Self::load_dns_data(&conn, external_dns)
-                        .await
-                        .map_err(|e| {
-                           err.set(RackInitError::DnsSerialization(e)).unwrap();
-                           DieselError::RollbackTransaction
-                        })?;
+                    Self::load_dns_data(&conn, external_dns).await.map_err(
+                        |e| {
+                            err.set(RackInitError::DnsSerialization(e))
+                                .unwrap();
+                            DieselError::RollbackTransaction
+                        },
+                    )?;
                     info!(log, "Populated DNS tables for external DNS");
 
                     // Create the initial Recovery Silo
@@ -711,7 +775,7 @@ impl DataStore {
                         _ => {
                             err.set(e).unwrap();
                             DieselError::RollbackTransaction
-                        },
+                        }
                     })?;
 
                     let rack = diesel::update(rack_dsl::rack)
@@ -730,13 +794,13 @@ impl DataStore {
                             err.set(RackInitError::RackUpdate {
                                 err: e,
                                 rack_id,
-                            }).unwrap();
+                            })
+                            .unwrap();
                             DieselError::RollbackTransaction
                         })?;
                     Ok(rack)
                 }
-            },
-            )
+            })
             .await
             .map_err(|e| {
                 if let Some(err) = Arc::try_unwrap(err).unwrap().take() {
@@ -873,7 +937,7 @@ mod test {
     };
     use omicron_common::api::internal::shared::SourceNatConfig;
     use omicron_test_utils::dev;
-    use std::collections::HashMap;
+    use std::collections::{BTreeMap, BTreeSet, HashMap};
     use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddrV6};
     use std::num::NonZeroU32;
 
@@ -884,6 +948,17 @@ mod test {
             RackInit {
                 rack_id: Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap(),
                 rack_subnet: nexus_test_utils::RACK_SUBNET.parse().unwrap(),
+                blueprint: Blueprint {
+                    id: Uuid::new_v4(),
+                    omicron_zones: BTreeMap::new(),
+                    zones_in_service: BTreeSet::new(),
+                    parent_blueprint_id: None,
+                    internal_dns_version: *Generation::new(),
+                    external_dns_version: *Generation::new(),
+                    time_created: Utc::now(),
+                    creator: "test suite".to_string(),
+                    comment: "test suite".to_string(),
+                },
                 services: vec![],
                 datasets: vec![],
                 service_ip_pool_ranges: vec![],
diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs
index 6133da6428..c06db2c23e 100644
--- a/nexus/reconfigurator/execution/src/dns.rs
+++ b/nexus/reconfigurator/execution/src/dns.rs
@@ -490,13 +490,9 @@ mod test {
     use omicron_common::address::NEXUS_REDUNDANCY;
     use omicron_common::address::RACK_PREFIX;
     use omicron_common::address::SLED_PREFIX;
-    use omicron_common::api::external::Error;
     use omicron_common::api::external::Generation;
     use omicron_common::api::external::IdentityMetadataCreateParams;
-    use omicron_test_utils::dev::poll::wait_for_condition;
-    use omicron_test_utils::dev::poll::CondCheckError;
     use omicron_test_utils::dev::test_setup_log;
-    use slog::{debug, info};
     use std::collections::BTreeMap;
     use std::collections::BTreeSet;
     use std::collections::HashMap;
@@ -506,7 +502,6 @@ mod test {
     use std::net::SocketAddrV6;
     use std::str::FromStr;
     use std::sync::Arc;
-    use std::time::Duration;
     use uuid::Uuid;
 
     type ControlPlaneTestContext =
@@ -1171,38 +1166,6 @@ mod test {
             datastore.clone(),
         );
 
-        // First, wait until Nexus has successfully completed an inventory
-        // collection.
-        let collection = wait_for_condition(
-            || async {
-                let result =
-                    datastore.inventory_get_latest_collection(&opctx).await;
-                let log_result = match &result {
-                    Ok(Some(_)) => Ok("found"),
-                    Ok(None) => Ok("not found"),
-                    Err(error) => Err(error),
-                };
-                debug!(
-                    log,
-                    "attempt to fetch latest inventory collection";
-                    "result" => ?log_result,
-                );
-
-                match result {
-                    Ok(None) => Err(CondCheckError::NotYet),
-                    Ok(Some(c)) => Ok(c),
-                    Err(Error::ServiceUnavailable { .. }) => {
-                        Err(CondCheckError::NotYet)
-                    }
-                    Err(error) => Err(CondCheckError::Failed(error)),
-                }
-            },
-            &Duration::from_millis(50),
-            &Duration::from_secs(30),
-        )
-        .await
-        .expect("expected to find inventory collection");
-
         // Fetch the initial contents of internal and external DNS.
         let dns_initial_internal = datastore
             .dns_config_read(&opctx, DnsGroup::Internal)
@@ -1213,29 +1176,15 @@ mod test {
             .await
             .expect("fetching initial external DNS");
 
-        // Now, use the collection to construct an initial blueprint.
-        // This stores it into the database, too.
-        info!(log, "using collection"; "collection_id" => %collection.id);
-        let blueprint = nexus
-            .blueprint_generate_from_collection(&opctx, collection.id)
+        // Fetch the initial blueprint installed during rack initialization.
+        let (_blueprint_target, blueprint) = datastore
+            .blueprint_target_get_current_full(&opctx)
             .await
-            .expect("failed to generate initial blueprint");
+            .expect("failed to read current target blueprint")
+            .expect("no target blueprint set");
         eprintln!("blueprint: {:?}", blueprint);
 
-        // Set it as the current target.  We'll need this later.
-        datastore
-            .blueprint_target_set_current(
-                &opctx,
-                BlueprintTarget {
-                    target_id: blueprint.id,
-                    enabled: false,
-                    time_made_target: chrono::Utc::now(),
-                },
-            )
-            .await
-            .expect("failed to set blueprint as target");
-
-        // Now, execute the blueprint.
+        // Now, execute the initial blueprint.
         let overrides = Overridables::for_test(cptestctx);
         crate::realize_blueprint_with_overrides(
             &opctx,
diff --git a/nexus/reconfigurator/planning/src/blueprint_builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder.rs
index 8b0d440d26..c9cd79ea14 100644
--- a/nexus/reconfigurator/planning/src/blueprint_builder.rs
+++ b/nexus/reconfigurator/planning/src/blueprint_builder.rs
@@ -823,8 +823,8 @@ impl<'a> BlueprintZones<'a> {
             } else {
                 // The first generation is reserved to mean the one
                 // containing no zones.  See
-                // OMICRON_ZONES_CONFIG_INITIAL_GENERATION.  So we start
-                // with the next one.
+                // OmicronZonesConfig::INITIAL_GENERATION.  So we start with the
+                // next one.
                 OmicronZonesConfig {
                     generation: Generation::new().next(),
                     zones: vec![],
diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs
index 1f79fda222..819e56b5d0 100644
--- a/nexus/reconfigurator/planning/src/planner.rs
+++ b/nexus/reconfigurator/planning/src/planner.rs
@@ -434,7 +434,7 @@ mod test {
         let (sled_id, sled_zones) = sleds[0];
         // We have defined elsewhere that the first generation contains no
         // zones.  So the first one with zones must be newer.  See
-        // OMICRON_ZONES_CONFIG_INITIAL_GENERATION.
+        // OmicronZonesConfig::INITIAL_GENERATION.
         assert!(sled_zones.generation > Generation::new());
         assert_eq!(sled_id, new_sled_id);
         assert_eq!(sled_zones.zones.len(), 1);
diff --git a/nexus/src/app/background/blueprint_load.rs b/nexus/src/app/background/blueprint_load.rs
index b593354099..614c59ff9b 100644
--- a/nexus/src/app/background/blueprint_load.rs
+++ b/nexus/src/app/background/blueprint_load.rs
@@ -217,7 +217,7 @@ mod test {
         nexus_test_utils::ControlPlaneTestContext<crate::Server>;
 
     fn create_blueprint(
-        parent_blueprint_id: Option<Uuid>,
+        parent_blueprint_id: Uuid,
     ) -> (BlueprintTarget, Blueprint) {
         let id = Uuid::new_v4();
         (
@@ -230,7 +230,7 @@ mod test {
                 id,
                 omicron_zones: BTreeMap::new(),
                 zones_in_service: BTreeSet::new(),
-                parent_blueprint_id,
+                parent_blueprint_id: Some(parent_blueprint_id),
                 internal_dns_version: Generation::new(),
                 external_dns_version: Generation::new(),
                 time_created: now_db_precision(),
@@ -261,26 +261,31 @@ mod test {
         let mut task = TargetBlueprintLoader::new(datastore.clone());
         let mut rx = task.watcher();
 
-        // We expect an appropriate status with no blueprint in the datastore
+        // We expect to see the initial blueprint set up by nexus-test-utils
+        // (emulating RSS).
         let value = task.activate(&opctx).await;
-        assert_eq!(json!({"status": "no target blueprint"}), value);
-        assert!(rx.borrow().is_none());
+        let initial_blueprint =
+            rx.borrow_and_update().clone().expect("no initial blueprint");
+        let update = serde_json::from_value::<TargetUpdate>(value).unwrap();
+        assert_eq!(update.target_id, initial_blueprint.1.id);
+        assert_eq!(update.status, "first target blueprint");
 
-        let (target, blueprint) = create_blueprint(None);
+        let (target, blueprint) = create_blueprint(update.target_id);
 
-        // Inserting a blueprint, but not making it the target returns the same
-        // status
+        // Inserting a blueprint, but not making it the target return status
+        // indicating that the target hasn't changed
         datastore.blueprint_insert(&opctx, &blueprint).await.unwrap();
         let value = task.activate(&opctx).await;
-        assert_eq!(json!({"status": "no target blueprint"}), value);
-        assert!(rx.borrow().is_none());
+        let update = serde_json::from_value::<TargetUpdate>(value).unwrap();
+        assert_eq!(update.target_id, initial_blueprint.1.id);
+        assert_eq!(update.status, "target blueprint unchanged");
 
         // Setting a target blueprint makes the loader see it and broadcast it
         datastore.blueprint_target_set_current(&opctx, target).await.unwrap();
         let value = task.activate(&opctx).await;
         let update = serde_json::from_value::<TargetUpdate>(value).unwrap();
         assert_eq!(update.target_id, blueprint.id);
-        assert_eq!(update.status, "first target blueprint");
+        assert_eq!(update.status, "target blueprint updated");
         let rx_update = rx.borrow_and_update().clone().unwrap();
         assert_eq!(rx_update.0, target);
         assert_eq!(rx_update.1, blueprint);
@@ -293,7 +298,7 @@ mod test {
         assert_eq!(false, rx.has_changed().unwrap());
 
         // Adding a new blueprint and updating the target triggers a change
-        let (new_target, new_blueprint) = create_blueprint(Some(blueprint.id));
+        let (new_target, new_blueprint) = create_blueprint(blueprint.id);
         datastore.blueprint_insert(&opctx, &new_blueprint).await.unwrap();
         datastore
             .blueprint_target_set_current(&opctx, new_target)
diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs
index 4ba31bb0fe..9905df04ff 100644
--- a/nexus/src/app/rack.rs
+++ b/nexus/src/app/rack.rs
@@ -190,6 +190,12 @@ impl super::Nexus {
             format!("{silo_dns_name}.{}", request.external_dns_zone_name);
         dns_update.add_name(silo_dns_name, dns_records)?;
 
+        // We're providing an update to the initial `external_dns` group we
+        // defined above; also bump RSS's blueprint's `external_dns_version` to
+        // match this update.
+        let mut blueprint = request.blueprint;
+        blueprint.external_dns_version = blueprint.external_dns_version.next();
+
         // Administrators of the Recovery Silo are automatically made
         // administrators of the Fleet.
         let mapped_fleet_roles = BTreeMap::from([(
@@ -202,9 +208,10 @@ impl super::Nexus {
                 name: request.recovery_silo.silo_name,
                 description: "built-in recovery Silo".to_string(),
             },
-            // The recovery silo is initialized with no allocated capacity given it's
-            // not intended to be used to deploy workloads. Operators can add capacity
-            // after the fact if they want to use it for that purpose.
+            // The recovery silo is initialized with no allocated capacity given
+            // it's not intended to be used to deploy workloads. Operators can
+            // add capacity after the fact if they want to use it for that
+            // purpose.
             quotas: params::SiloQuotasCreate::empty(),
             discoverable: false,
             identity_mode: SiloIdentityMode::LocalOnly,
@@ -221,6 +228,7 @@ impl super::Nexus {
                 RackInit {
                     rack_subnet: rack_network_config.rack_subnet.into(),
                     rack_id,
+                    blueprint,
                     services: request.services,
                     datasets,
                     service_ip_pool_ranges,
diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs
index c0fba31afb..03470a4f3a 100644
--- a/nexus/src/lib.rs
+++ b/nexus/src/lib.rs
@@ -27,6 +27,7 @@ use dropshot::ConfigDropshot;
 use external_api::http_entrypoints::external_api;
 use internal_api::http_entrypoints::internal_api;
 use nexus_config::NexusConfig;
+use nexus_types::deployment::Blueprint;
 use nexus_types::external_api::views::SledProvisionPolicy;
 use nexus_types::internal_api::params::ServiceKind;
 use omicron_common::address::IpRange;
@@ -232,6 +233,7 @@ impl nexus_test_interface::NexusServer for Server {
     async fn start(
         internal_server: InternalServer,
         config: &NexusConfig,
+        blueprint: Blueprint,
         services: Vec<nexus_types::internal_api::params::ServicePutRequest>,
         datasets: Vec<nexus_types::internal_api::params::DatasetCreateRequest>,
         internal_dns_zone_config: nexus_types::internal_api::params::DnsConfigParams,
@@ -276,6 +278,7 @@ impl nexus_test_interface::NexusServer for Server {
                 &opctx,
                 config.deployment.rack_id,
                 internal_api::params::RackInitializationRequest {
+                    blueprint,
                     services,
                     datasets,
                     internal_services_ip_pool_ranges,
diff --git a/nexus/test-interface/src/lib.rs b/nexus/test-interface/src/lib.rs
index 10bc9e63f0..d9bb69276a 100644
--- a/nexus/test-interface/src/lib.rs
+++ b/nexus/test-interface/src/lib.rs
@@ -33,6 +33,7 @@
 
 use async_trait::async_trait;
 use nexus_config::NexusConfig;
+use nexus_types::deployment::Blueprint;
 use slog::Logger;
 use std::net::{SocketAddr, SocketAddrV6};
 use uuid::Uuid;
@@ -50,6 +51,7 @@ pub trait NexusServer: Send + Sync + 'static {
     async fn start(
         internal_server: Self::InternalServer,
         config: &NexusConfig,
+        blueprint: Blueprint,
         services: Vec<nexus_types::internal_api::params::ServicePutRequest>,
         datasets: Vec<nexus_types::internal_api::params::DatasetCreateRequest>,
         internal_dns_config: nexus_types::internal_api::params::DnsConfigParams,
diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs
index 9681d9ff97..0358cf523c 100644
--- a/nexus/test-utils/src/lib.rs
+++ b/nexus/test-utils/src/lib.rs
@@ -7,6 +7,7 @@
 use anyhow::Context;
 use anyhow::Result;
 use camino::Utf8Path;
+use chrono::Utc;
 use dns_service_client::types::DnsConfigParams;
 use dropshot::test_util::ClientTestContext;
 use dropshot::test_util::LogContext;
@@ -24,6 +25,7 @@ use nexus_config::MgdConfig;
 use nexus_config::NexusConfig;
 use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use nexus_test_interface::NexusServer;
+use nexus_types::deployment::Blueprint;
 use nexus_types::external_api::params::UserId;
 use nexus_types::internal_api::params::Certificate;
 use nexus_types::internal_api::params::DatasetCreateRequest;
@@ -54,6 +56,8 @@ use oximeter_collector::Oximeter;
 use oximeter_producer::LogConfig;
 use oximeter_producer::Server as ProducerServer;
 use slog::{debug, error, o, Logger};
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6};
@@ -785,12 +789,49 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             user_password_hash,
         };
 
+        let blueprint = {
+            let mut omicron_zones = BTreeMap::new();
+            let mut zones_in_service = BTreeSet::new();
+            for (maybe_sled_agent, zones) in [
+                (self.sled_agent.as_ref(), &self.omicron_zones),
+                (self.sled_agent2.as_ref(), &self.omicron_zones2),
+            ] {
+                if let Some(sa) = maybe_sled_agent {
+                    omicron_zones.insert(
+                        sa.sled_agent.id,
+                        OmicronZonesConfig {
+                            generation: Generation::new().next(),
+                            zones: zones.clone(),
+                        },
+                    );
+                    for z in zones {
+                        zones_in_service.insert(z.id);
+                    }
+                }
+            }
+            Blueprint {
+                id: Uuid::new_v4(),
+                omicron_zones,
+                zones_in_service,
+                parent_blueprint_id: None,
+                internal_dns_version: dns_config
+                    .generation
+                    .try_into()
+                    .expect("bad internal DNS generation"),
+                external_dns_version: Generation::new(),
+                time_created: Utc::now(),
+                creator: "nexus-test-utils".to_string(),
+                comment: "initial test blueprint".to_string(),
+            }
+        };
+
         // Handoff all known service information to Nexus
         let server = N::start(
             self.nexus_internal
                 .take()
                 .expect("Must launch internal nexus first"),
-            &self.config,
+            self.config,
+            blueprint,
             self.rack_init_builder.services.clone(),
             // NOTE: We should probably hand off
             // "self.rack_init_builder.datasets" here, but Nexus won't be happy
diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs
index e8c4703008..9f80d313fd 100644
--- a/nexus/types/src/internal_api/params.rs
+++ b/nexus/types/src/internal_api/params.rs
@@ -4,6 +4,7 @@
 
 //! Params define the request bodies of API endpoints for creating or updating resources.
 
+use crate::deployment::Blueprint;
 use crate::external_api::params::PhysicalDiskKind;
 use crate::external_api::params::UserId;
 use crate::external_api::shared::Baseboard;
@@ -248,6 +249,8 @@ impl std::fmt::Debug for Certificate {
 
 #[derive(Debug, Clone, Deserialize, JsonSchema)]
 pub struct RackInitializationRequest {
+    /// Blueprint describing services initialized by RSS.
+    pub blueprint: Blueprint,
     /// Services on the rack which have been created by RSS.
     pub services: Vec<ServicePutRequest>,
     /// Datasets on the rack which have been provisioned by RSS.
diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json
index 679597c453..3938d2f081 100644
--- a/openapi/nexus-internal.json
+++ b/openapi/nexus-internal.json
@@ -6252,6 +6252,14 @@
       "RackInitializationRequest": {
         "type": "object",
         "properties": {
+          "blueprint": {
+            "description": "Blueprint describing services initialized by RSS.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Blueprint"
+              }
+            ]
+          },
           "certs": {
             "description": "x.509 Certificates used to encrypt communication with the external API.",
             "type": "array",
@@ -6318,6 +6326,7 @@
           }
         },
         "required": [
+          "blueprint",
           "certs",
           "datasets",
           "external_dns_zone_name",
diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml
index 239887e789..52c6e61efa 100644
--- a/sled-agent/Cargo.toml
+++ b/sled-agent/Cargo.toml
@@ -46,6 +46,7 @@ macaddr.workspace = true
 mg-admin-client.workspace = true
 nexus-client.workspace = true
 nexus-config.workspace = true
+nexus-types.workspace = true
 omicron-common.workspace = true
 once_cell.workspace = true
 oximeter.workspace = true
diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs
index 51951465fd..82294326b6 100644
--- a/sled-agent/src/params.rs
+++ b/sled-agent/src/params.rs
@@ -288,9 +288,6 @@ impl std::fmt::Display for ZoneType {
     }
 }
 
-/// Generation 1 of `OmicronZonesConfig` is always the set of no zones.
-pub const OMICRON_ZONES_CONFIG_INITIAL_GENERATION: u32 = 1;
-
 /// Describes the set of Omicron-managed zones running on a sled
 #[derive(
     Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
@@ -310,6 +307,11 @@ pub struct OmicronZonesConfig {
     pub zones: Vec<OmicronZoneConfig>,
 }
 
+impl OmicronZonesConfig {
+    /// Generation 1 of `OmicronZonesConfig` is always the set of no zones.
+    pub const INITIAL_GENERATION: Generation = Generation::from_u32(1);
+}
+
 impl From<OmicronZonesConfig> for sled_agent_client::types::OmicronZonesConfig {
     fn from(local: OmicronZonesConfig) -> Self {
         Self {
diff --git a/sled-agent/src/rack_setup/mod.rs b/sled-agent/src/rack_setup/mod.rs
index cfabca6209..0ad8e0ce71 100644
--- a/sled-agent/src/rack_setup/mod.rs
+++ b/sled-agent/src/rack_setup/mod.rs
@@ -9,3 +9,5 @@ pub mod config;
 mod plan;
 /// The main implementation of the RSS service.
 pub mod service;
+
+pub use plan::service::SledConfig;
diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs
index 453495a870..6b189f423c 100644
--- a/sled-agent/src/rack_setup/service.rs
+++ b/sled-agent/src/rack_setup/service.rs
@@ -65,6 +65,7 @@
 //! thereafter.
 
 use super::config::SetupServiceConfig as Config;
+use super::plan::service::SledConfig;
 use crate::bootstrap::config::BOOTSTRAP_AGENT_HTTP_PORT;
 use crate::bootstrap::early_networking::{
     EarlyNetworkConfig, EarlyNetworkConfigBody, EarlyNetworkSetup,
@@ -74,24 +75,24 @@ use crate::bootstrap::params::BootstrapAddressDiscovery;
 use crate::bootstrap::params::StartSledAgentRequest;
 use crate::bootstrap::rss_handle::BootstrapAgentHandle;
 use crate::nexus::{d2n_params, ConvertInto};
-use crate::params::{
-    OmicronZoneType, OmicronZonesConfig, TimeSync,
-    OMICRON_ZONES_CONFIG_INITIAL_GENERATION,
-};
+use crate::params::{OmicronZoneType, OmicronZonesConfig, TimeSync};
 use crate::rack_setup::plan::service::{
     Plan as ServicePlan, PlanError as ServicePlanError,
 };
 use crate::rack_setup::plan::sled::{
     Plan as SledPlan, PlanError as SledPlanError,
 };
+use anyhow::{bail, Context};
 use bootstore::schemes::v0 as bootstore;
 use camino::Utf8PathBuf;
+use chrono::Utc;
 use ddm_admin_client::{Client as DdmAdminClient, DdmError};
 use internal_dns::resolver::{DnsError, Resolver as DnsResolver};
 use internal_dns::ServiceName;
 use nexus_client::{
     types as NexusTypes, Client as NexusClient, Error as NexusError,
 };
+use nexus_types::deployment::Blueprint;
 use omicron_common::address::get_sled_address;
 use omicron_common::api::external::Generation;
 use omicron_common::api::internal::shared::ExternalPortDiscovery;
@@ -107,11 +108,12 @@ use sled_hardware_types::underlay::BootstrapInterface;
 use sled_storage::dataset::CONFIG_DATASET;
 use sled_storage::manager::StorageHandle;
 use slog::Logger;
-use std::collections::BTreeSet;
+use std::collections::{btree_map, BTreeMap, BTreeSet};
 use std::collections::{HashMap, HashSet};
 use std::iter;
 use std::net::{Ipv6Addr, SocketAddrV6};
 use thiserror::Error;
+use uuid::Uuid;
 
 /// Describes errors which may occur while operating the setup service.
 #[derive(Error, Debug)]
@@ -168,6 +170,9 @@ pub enum SetupServiceError {
     #[error("Bootstore error: {0}")]
     Bootstore(#[from] bootstore::NodeRequestError),
 
+    #[error("Failed to convert setup plan to blueprint: {0:#}")]
+    ConvertPlanToBlueprint(anyhow::Error),
+
     // We used transparent, because `EarlyNetworkSetupError` contains a subset
     // of error variants already in this type
     #[error(transparent)]
@@ -548,6 +553,12 @@ impl ServiceInner {
     ) -> Result<(), SetupServiceError> {
         info!(self.log, "Handing off control to Nexus");
 
+        // Build a Blueprint describing our service plan. This should never
+        // fail, unless we've set up an invalid plan.
+        let blueprint =
+            build_initial_blueprint_from_plan(sled_plan, service_plan)
+                .map_err(SetupServiceError::ConvertPlanToBlueprint)?;
+
         info!(self.log, "Nexus address: {}", nexus_address.to_string());
 
         let nexus_client = NexusClient::new(
@@ -655,6 +666,7 @@ impl ServiceInner {
         info!(self.log, "rack_network_config: {:#?}", rack_network_config);
 
         let request = NexusTypes::RackInitializationRequest {
+            blueprint,
             services,
             datasets,
             internal_services_ip_pool_ranges,
@@ -970,44 +982,14 @@ impl ServiceInner {
             .await?
         };
 
-        // The service plan describes all the zones that we will eventually
-        // deploy on each sled.  But we cannot currently just deploy them all
-        // concurrently.  We'll do it in a few stages, each corresponding to a
-        // version of each sled's configuration.
-        //
-        // - version 1: no services running
-        //              (We don't have to do anything for this.  But we do
-        //              reserve this version number for "no services running" so
-        //              that sled agents can begin with an initial, valid
-        //              OmicronZonesConfig before they've got anything running.)
-        // - version 2: internal DNS only
-        // - version 3: internal DNS + NTP servers
-        // - version 4: internal DNS + NTP servers + CockroachDB
-        // - version 5: everything
-        //
-        // At each stage, we're specifying a complete configuration of what
-        // should be running on the sled -- including this version number.
-        // And Sled Agents will reject requests for versions older than the
-        // one they're currently running.  Thus, the version number is a piece
-        // of global, distributed state.
-        //
-        // For now, we hardcode the requests we make to use specific version
-        // numbers.
-        let version1_nothing =
-            Generation::from(OMICRON_ZONES_CONFIG_INITIAL_GENERATION);
-        let version2_dns_only = version1_nothing.next();
-        let version3_dns_and_ntp = version2_dns_only.next();
-        let version4_cockroachdb = version3_dns_and_ntp.next();
-        let version5_everything = version4_cockroachdb.next();
-
         // Set up internal DNS services first and write the initial
         // DNS configuration to the internal DNS servers.
         let v1generator = OmicronZonesConfigGenerator::initial_version(
             &service_plan,
-            version1_nothing,
+            DeployStepVersion::V1_NOTHING,
         );
         let v2generator = v1generator.new_version_with(
-            version2_dns_only,
+            DeployStepVersion::V2_DNS_ONLY,
             &|zone_type: &OmicronZoneType| {
                 matches!(zone_type, OmicronZoneType::InternalDns { .. })
             },
@@ -1022,7 +1004,7 @@ impl ServiceInner {
 
         // Next start up the NTP services.
         let v3generator = v2generator.new_version_with(
-            version3_dns_and_ntp,
+            DeployStepVersion::V3_DNS_AND_NTP,
             &|zone_type: &OmicronZoneType| {
                 matches!(
                     zone_type,
@@ -1040,7 +1022,7 @@ impl ServiceInner {
 
         // Wait until Cockroach has been initialized before running Nexus.
         let v4generator = v3generator.new_version_with(
-            version4_cockroachdb,
+            DeployStepVersion::V4_COCKROACHDB,
             &|zone_type: &OmicronZoneType| {
                 matches!(zone_type, OmicronZoneType::CockroachDb { .. })
             },
@@ -1052,8 +1034,8 @@ impl ServiceInner {
         self.initialize_cockroach(&service_plan).await?;
 
         // Issue the rest of the zone initialization requests.
-        let v5generator =
-            v4generator.new_version_with(version5_everything, &|_| true);
+        let v5generator = v4generator
+            .new_version_with(DeployStepVersion::V5_EVERYTHING, &|_| true);
         self.ensure_zone_config_at_least(v5generator.sled_configs()).await?;
 
         info!(self.log, "Finished setting up services");
@@ -1084,6 +1066,120 @@ impl ServiceInner {
     }
 }
 
+/// The service plan describes all the zones that we will eventually
+/// deploy on each sled.  But we cannot currently just deploy them all
+/// concurrently.  We'll do it in a few stages, each corresponding to a
+/// version of each sled's configuration.
+///
+/// - version 1: no services running
+///              (We don't have to do anything for this.  But we do
+///              reserve this version number for "no services running" so
+///              that sled agents can begin with an initial, valid
+///              OmicronZonesConfig before they've got anything running.)
+/// - version 2: internal DNS only
+/// - version 3: internal DNS + NTP servers
+/// - version 4: internal DNS + NTP servers + CockroachDB
+/// - version 5: everything
+///
+/// At each stage, we're specifying a complete configuration of what
+/// should be running on the sled -- including this version number.
+/// And Sled Agents will reject requests for versions older than the
+/// one they're currently running.  Thus, the version number is a piece
+/// of global, distributed state.
+///
+/// For now, we hardcode the requests we make to use specific version
+/// numbers.
+struct DeployStepVersion;
+
+impl DeployStepVersion {
+    const V1_NOTHING: Generation = OmicronZonesConfig::INITIAL_GENERATION;
+    const V2_DNS_ONLY: Generation = Self::V1_NOTHING.next();
+    const V3_DNS_AND_NTP: Generation = Self::V2_DNS_ONLY.next();
+    const V4_COCKROACHDB: Generation = Self::V3_DNS_AND_NTP.next();
+    const V5_EVERYTHING: Generation = Self::V4_COCKROACHDB.next();
+}
+
+fn build_initial_blueprint_from_plan(
+    sled_plan: &SledPlan,
+    service_plan: &ServicePlan,
+) -> anyhow::Result<Blueprint> {
+    let internal_dns_version =
+        Generation::try_from(service_plan.dns_config.generation)
+            .context("invalid internal dns version")?;
+
+    let mut sled_configs = BTreeMap::new();
+    for sled_request in sled_plan.sleds.values() {
+        let sled_addr = get_sled_address(sled_request.body.subnet);
+        let sled_id = sled_request.body.id;
+        let entry = match sled_configs.entry(sled_id) {
+            btree_map::Entry::Vacant(entry) => entry,
+            btree_map::Entry::Occupied(_) => {
+                bail!(
+                    "duplicate sled address found while deriving blueprint: \
+                     {sled_addr}"
+                );
+            }
+        };
+        let sled_config =
+            service_plan.services.get(&sled_addr).with_context(|| {
+                format!(
+                    "missing services in plan for sled {sled_id} ({sled_addr})"
+                )
+            })?;
+        entry.insert(sled_config.clone());
+    }
+
+    Ok(build_initial_blueprint_from_sled_configs(
+        sled_configs,
+        internal_dns_version,
+    ))
+}
+
+pub(crate) fn build_initial_blueprint_from_sled_configs(
+    sled_configs: BTreeMap<Uuid, SledConfig>,
+    internal_dns_version: Generation,
+) -> Blueprint {
+    let mut omicron_zones = BTreeMap::new();
+    let mut zones_in_service = BTreeSet::new();
+    for (sled_id, sled_config) in sled_configs {
+        for zone in &sled_config.zones {
+            zones_in_service.insert(zone.id);
+        }
+        let zones_config = sled_agent_client::types::OmicronZonesConfig::from(
+            OmicronZonesConfig {
+                // This is a bit of a hack. We only construct a blueprint after
+                // completing RSS, so we need to know the final generation value
+                // sent to all sleds. Arguably, we should record this in the
+                // serialized RSS plan; however, we have already deployed
+                // systems that did not. We know that every such system used
+                // `V5_EVERYTHING` as the final generation count, so we can just
+                // use that value here. If we ever change this, in particular in
+                // a way where newly-deployed systems will have a different
+                // value, we will need to revisit storing this in the serialized
+                // RSS plan.
+                generation: DeployStepVersion::V5_EVERYTHING,
+                zones: sled_config.zones,
+            },
+        );
+        omicron_zones.insert(sled_id, zones_config);
+    }
+
+    Blueprint {
+        id: Uuid::new_v4(),
+        omicron_zones,
+        zones_in_service,
+        parent_blueprint_id: None,
+        internal_dns_version,
+        // We don't configure external DNS during RSS, so set it to an initial
+        // generation of 1. Nexus will bump this up when it updates external DNS
+        // (including creating the recovery silo).
+        external_dns_version: Generation::new(),
+        time_created: Utc::now(),
+        creator: "RSS".to_string(),
+        comment: "initial blueprint from rack setup".to_string(),
+    }
+}
+
 /// Facilitates creating a sequence of OmicronZonesConfig objects for each sled
 /// in a service plan to enable phased rollout of services
 ///
diff --git a/sled-agent/src/services_migration.rs b/sled-agent/src/services_migration.rs
index bedd4759c8..511368e2f6 100644
--- a/sled-agent/src/services_migration.rs
+++ b/sled-agent/src/services_migration.rs
@@ -24,8 +24,8 @@
 //! past this change.
 
 use crate::params::{
-    OmicronZoneConfig, OmicronZoneDataset, OmicronZoneType, ZoneType,
-    OMICRON_ZONES_CONFIG_INITIAL_GENERATION,
+    OmicronZoneConfig, OmicronZoneDataset, OmicronZoneType, OmicronZonesConfig,
+    ZoneType,
 };
 use crate::services::{OmicronZoneConfigLocal, OmicronZonesConfigLocal};
 use anyhow::{anyhow, ensure, Context};
@@ -103,8 +103,7 @@ impl TryFrom<AllZoneRequests> for OmicronZonesConfigLocal {
         //   some buffer.)
         //
         // In summary, 2 seems fine.
-        let omicron_generation =
-            Generation::from(OMICRON_ZONES_CONFIG_INITIAL_GENERATION).next();
+        let omicron_generation = OmicronZonesConfig::INITIAL_GENERATION.next();
 
         // The ledger generation doesn't really matter.  In case it's useful, we
         // pick the generation from the ledger that we loaded.
diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs
index dd815775ff..b7577e6d02 100644
--- a/sled-agent/src/sim/server.rs
+++ b/sled-agent/src/sim/server.rs
@@ -10,21 +10,30 @@ use super::sled_agent::SledAgent;
 use super::storage::PantryServer;
 use crate::nexus::d2n_params;
 use crate::nexus::NexusClient;
+use crate::params::OmicronZoneConfig;
+use crate::params::OmicronZoneDataset;
+use crate::params::OmicronZoneType;
+use crate::rack_setup::service::build_initial_blueprint_from_sled_configs;
+use crate::rack_setup::SledConfig;
 use anyhow::anyhow;
 use crucible_agent_client::types::State as RegionState;
+use illumos_utils::zpool::ZpoolName;
 use internal_dns::ServiceName;
 use nexus_client::types as NexusTypes;
 use nexus_client::types::{IpRange, Ipv4Range, Ipv6Range};
 use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
+use nexus_types::inventory::NetworkInterfaceKind;
 use omicron_common::address::DNS_OPTE_IPV4_SUBNET;
 use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET;
 use omicron_common::api::external::Generation;
 use omicron_common::api::external::MacAddr;
+use omicron_common::api::external::Vni;
 use omicron_common::backoff::{
     retry_notify, retry_policy_internal_service_aggressive, BackoffError,
 };
 use omicron_common::FileKv;
 use slog::{info, Drain, Logger};
+use std::collections::BTreeMap;
 use std::collections::HashMap;
 use std::net::IpAddr;
 use std::net::Ipv4Addr;
@@ -334,6 +343,8 @@ pub async fn run_standalone_server(
     // Initialize the internal DNS entries
     let dns_config = dns_config_builder.build();
     dns.initialize_with_config(&log, &dns_config).await?;
+    let internal_dns_version = Generation::try_from(dns_config.generation)
+        .expect("invalid internal dns version");
 
     // Record the internal DNS server as though RSS had provisioned it so
     // that Nexus knows about it.
@@ -341,37 +352,58 @@ pub async fn run_standalone_server(
         SocketAddr::V4(_) => panic!("did not expect v4 address"),
         SocketAddr::V6(a) => a,
     };
-    let mut services = vec![NexusTypes::ServicePutRequest {
-        address: http_bound.to_string(),
-        kind: NexusTypes::ServiceKind::InternalDns,
-        service_id: Uuid::new_v4(),
-        sled_id: config.id,
-        zone_id: Some(Uuid::new_v4()),
+    let mut zones = vec![OmicronZoneConfig {
+        id: Uuid::new_v4(),
+        underlay_address: *http_bound.ip(),
+        zone_type: OmicronZoneType::InternalDns {
+            dataset: OmicronZoneDataset {
+                pool_name: ZpoolName::new_external(Uuid::new_v4()),
+            },
+            http_address: http_bound,
+            dns_address: match dns.dns_server.local_address() {
+                SocketAddr::V4(_) => panic!("did not expect v4 address"),
+                SocketAddr::V6(a) => a,
+            },
+            gz_address: Ipv6Addr::LOCALHOST,
+            gz_address_index: 0,
+        },
     }];
 
     let mut internal_services_ip_pool_ranges = vec![];
     let mut macs = MacAddr::iter_system();
     if let Some(nexus_external_addr) = rss_args.nexus_external_addr {
         let ip = nexus_external_addr.ip();
+        let id = Uuid::new_v4();
 
-        services.push(NexusTypes::ServicePutRequest {
-            address: config.nexus_address.to_string(),
-            kind: NexusTypes::ServiceKind::Nexus {
-                external_address: ip,
-                nic: NexusTypes::ServiceNic {
+        zones.push(OmicronZoneConfig {
+            id,
+            underlay_address: match ip {
+                IpAddr::V4(_) => panic!("did not expect v4 address"),
+                IpAddr::V6(a) => a,
+            },
+            zone_type: OmicronZoneType::Nexus {
+                internal_address: match config.nexus_address {
+                    SocketAddr::V4(_) => panic!("did not expect v4 address"),
+                    SocketAddr::V6(a) => a,
+                },
+                external_ip: ip,
+                nic: nexus_types::inventory::NetworkInterface {
                     id: Uuid::new_v4(),
+                    kind: NetworkInterfaceKind::Service { id },
                     name: "nexus".parse().unwrap(),
                     ip: NEXUS_OPTE_IPV4_SUBNET
                         .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES as u32 + 1)
                         .unwrap()
                         .into(),
                     mac: macs.next().unwrap(),
+                    subnet: (*NEXUS_OPTE_IPV4_SUBNET).into(),
+                    vni: Vni::SERVICES_VNI,
+                    primary: true,
                     slot: 0,
                 },
+                external_tls: false,
+                external_dns_servers: vec![],
             },
-            service_id: Uuid::new_v4(),
-            sled_id: config.id,
-            zone_id: Some(Uuid::new_v4()),
         });
 
         internal_services_ip_pool_ranges.push(match ip {
@@ -388,24 +420,31 @@ pub async fn run_standalone_server(
         rss_args.external_dns_internal_addr
     {
         let ip = *external_dns_internal_addr.ip();
-        services.push(NexusTypes::ServicePutRequest {
-            address: external_dns_internal_addr.to_string(),
-            kind: NexusTypes::ServiceKind::ExternalDns {
-                external_address: ip.into(),
-                nic: NexusTypes::ServiceNic {
+        let id = Uuid::new_v4();
+        zones.push(OmicronZoneConfig {
+            id,
+            underlay_address: ip,
+            zone_type: OmicronZoneType::ExternalDns {
+                dataset: OmicronZoneDataset {
+                    pool_name: ZpoolName::new_external(Uuid::new_v4()),
+                },
+                http_address: external_dns_internal_addr,
+                dns_address: SocketAddr::V6(external_dns_internal_addr),
+                nic: nexus_types::inventory::NetworkInterface {
                     id: Uuid::new_v4(),
+                    kind: NetworkInterfaceKind::Service { id },
                     name: "external-dns".parse().unwrap(),
                     ip: DNS_OPTE_IPV4_SUBNET
                         .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES as u32 + 1)
                         .unwrap()
                         .into(),
                     mac: macs.next().unwrap(),
+                    subnet: (*DNS_OPTE_IPV4_SUBNET).into(),
+                    vni: Vni::SERVICES_VNI,
+                    primary: true,
                     slot: 0,
                 },
             },
-            service_id: Uuid::new_v4(),
-            sled_id: config.id,
-            zone_id: Some(Uuid::new_v4()),
         });
 
         internal_services_ip_pool_ranges
@@ -450,7 +489,16 @@ pub async fn run_standalone_server(
         None => vec![],
     };
 
+    let services =
+        zones.iter().map(|z| z.to_nexus_service_req(config.id)).collect();
+    let mut sled_configs = BTreeMap::new();
+    sled_configs.insert(config.id, SledConfig { zones });
+
     let rack_init_request = NexusTypes::RackInitializationRequest {
+        blueprint: build_initial_blueprint_from_sled_configs(
+            sled_configs,
+            internal_dns_version,
+        ),
         services,
         datasets,
         internal_services_ip_pool_ranges,

From f63707a179f156b9fc588608490edddd37e3febb Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 15 Mar 2024 14:37:35 -0700
Subject: [PATCH 122/157] [nexus] Add zpool to inventory (#5249)

This PR adds zpools to the inventory, and updates the existing
`omicron.public.zpool` table to only include aspects of the zpool which
are fully controlled by Nexus.

This PR is intended to solve a problem introduced by
https://github.com/oxidecomputer/omicron/pull/5172 . In that PR,
physical disk and zpool provisioning will be decided by Nexus, and sent
to the Sled Agent as a request. As a result, the `total_size` value of
zpools is not known when a pool is provisioned. Here, we add zpools to
the inventory, and JOIN with that table to access the "total_size" of
the zpool if it is known.

Additionally, this PR adds a test to validate that the database
(appropriately) throws "Insufficient capacity" errors if the Zpool
exists, but has not appeared in the inventory.
---
 nexus/db-model/src/inventory.rs               |  40 ++++-
 nexus/db-model/src/schema.rs                  |  14 +-
 nexus/db-model/src/zpool.rs                   |  14 +-
 nexus/db-queries/src/db/datastore/dataset.rs  |   7 +-
 .../db-queries/src/db/datastore/inventory.rs  |  64 ++++++++
 nexus/db-queries/src/db/datastore/mod.rs      | 149 +++++++++++++++++-
 nexus/db-queries/src/db/datastore/region.rs   |  19 ++-
 nexus/db-queries/src/db/datastore/zpool.rs    |   1 -
 .../src/db/queries/region_allocation.rs       |   8 +-
 nexus/inventory/src/builder.rs                |   9 +-
 nexus/inventory/src/examples.rs               |   7 +
 .../reconfigurator/execution/src/datasets.rs  |   4 +-
 nexus/reconfigurator/planning/src/system.rs   |   2 +
 nexus/src/app/mod.rs                          |   6 +
 nexus/src/app/rack.rs                         |   3 +-
 nexus/src/app/sled.rs                         |   7 +-
 nexus/src/lib.rs                              |  13 ++
 nexus/test-interface/src/lib.rs               |   6 +
 nexus/test-utils/src/resource_helpers.rs      |  55 +++++++
 nexus/types/src/inventory.rs                  |  18 +++
 openapi/sled-agent.json                       |  26 ++-
 schema/crdb/44.0.0/up01.sql                   |   1 +
 schema/crdb/44.0.0/up02.sql                   |  10 ++
 schema/crdb/44.0.0/up03.sql                   |   1 +
 schema/crdb/dbinit.sql                        |  38 ++++-
 sled-agent/src/params.rs                      |   8 +
 sled-agent/src/sim/sled_agent.rs              |  17 +-
 sled-agent/src/sim/storage.rs                 |  17 +-
 sled-agent/src/sled_agent.rs                  |  23 +--
 29 files changed, 509 insertions(+), 78 deletions(-)
 create mode 100644 schema/crdb/44.0.0/up01.sql
 create mode 100644 schema/crdb/44.0.0/up02.sql
 create mode 100644 schema/crdb/44.0.0/up03.sql

diff --git a/nexus/db-model/src/inventory.rs b/nexus/db-model/src/inventory.rs
index 1c268c5914..cde067f3e8 100644
--- a/nexus/db-model/src/inventory.rs
+++ b/nexus/db-model/src/inventory.rs
@@ -9,7 +9,8 @@ use crate::schema::{
     hw_baseboard_id, inv_caboose, inv_collection, inv_collection_error,
     inv_omicron_zone, inv_omicron_zone_nic, inv_physical_disk,
     inv_root_of_trust, inv_root_of_trust_page, inv_service_processor,
-    inv_sled_agent, inv_sled_omicron_zones, sw_caboose, sw_root_of_trust_page,
+    inv_sled_agent, inv_sled_omicron_zones, inv_zpool, sw_caboose,
+    sw_root_of_trust_page,
 };
 use crate::PhysicalDiskKind;
 use crate::{
@@ -699,6 +700,43 @@ impl From<InvPhysicalDisk> for nexus_types::inventory::PhysicalDisk {
     }
 }
 
+/// See [`nexus_types::inventory::Zpool`].
+#[derive(Queryable, Clone, Debug, Selectable, Insertable)]
+#[diesel(table_name = inv_zpool)]
+pub struct InvZpool {
+    pub inv_collection_id: Uuid,
+    pub time_collected: DateTime<Utc>,
+    pub id: Uuid,
+    pub sled_id: Uuid,
+    pub total_size: ByteCount,
+}
+
+impl InvZpool {
+    pub fn new(
+        inv_collection_id: Uuid,
+        sled_id: Uuid,
+        zpool: &nexus_types::inventory::Zpool,
+    ) -> Self {
+        Self {
+            inv_collection_id,
+            time_collected: zpool.time_collected,
+            id: zpool.id,
+            sled_id,
+            total_size: zpool.total_size.into(),
+        }
+    }
+}
+
+impl From<InvZpool> for nexus_types::inventory::Zpool {
+    fn from(pool: InvZpool) -> Self {
+        Self {
+            time_collected: pool.time_collected,
+            id: pool.id,
+            total_size: *pool.total_size,
+        }
+    }
+}
+
 /// See [`nexus_types::inventory::OmicronZonesFound`].
 #[derive(Queryable, Clone, Debug, Selectable, Insertable)]
 #[diesel(table_name = inv_sled_omicron_zones)]
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index c31ce16775..2c53dc26a7 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(43, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(44, 0, 0);
 
 table! {
     disk (id) {
@@ -960,8 +960,6 @@ table! {
 
         sled_id -> Uuid,
         physical_disk_id -> Uuid,
-
-        total_size -> Int8,
     }
 }
 
@@ -1375,6 +1373,16 @@ table! {
     }
 }
 
+table! {
+    inv_zpool (inv_collection_id, sled_id, id) {
+        inv_collection_id -> Uuid,
+        time_collected -> Timestamptz,
+        id -> Uuid,
+        sled_id -> Uuid,
+        total_size -> Int8,
+    }
+}
+
 table! {
     inv_sled_omicron_zones (inv_collection_id, sled_id) {
         inv_collection_id -> Uuid,
diff --git a/nexus/db-model/src/zpool.rs b/nexus/db-model/src/zpool.rs
index 21f59e0347..1d129d4c57 100644
--- a/nexus/db-model/src/zpool.rs
+++ b/nexus/db-model/src/zpool.rs
@@ -2,7 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use super::{ByteCount, Dataset, Generation};
+use super::{Dataset, Generation};
 use crate::collection::DatastoreCollectionConfig;
 use crate::schema::{dataset, zpool};
 use chrono::{DateTime, Utc};
@@ -26,26 +26,16 @@ pub struct Zpool {
 
     // The physical disk to which this Zpool is attached.
     pub physical_disk_id: Uuid,
-
-    // TODO: In the future, we may expand this structure to include
-    // size, allocation, and health information.
-    pub total_size: ByteCount,
 }
 
 impl Zpool {
-    pub fn new(
-        id: Uuid,
-        sled_id: Uuid,
-        physical_disk_id: Uuid,
-        total_size: ByteCount,
-    ) -> Self {
+    pub fn new(id: Uuid, sled_id: Uuid, physical_disk_id: Uuid) -> Self {
         Self {
             identity: ZpoolIdentity::new(id),
             time_deleted: None,
             rcgen: Generation::new(),
             sled_id,
             physical_disk_id,
-            total_size,
         }
     }
 }
diff --git a/nexus/db-queries/src/db/datastore/dataset.rs b/nexus/db-queries/src/db/datastore/dataset.rs
index 792f8f81a4..292f13354f 100644
--- a/nexus/db-queries/src/db/datastore/dataset.rs
+++ b/nexus/db-queries/src/db/datastore/dataset.rs
@@ -229,12 +229,7 @@ mod test {
 
         // Create a fake zpool that backs our fake datasets.
         let zpool_id = Uuid::new_v4();
-        let zpool = Zpool::new(
-            zpool_id,
-            sled_id,
-            Uuid::new_v4(),
-            (1 << 30).try_into().unwrap(),
-        );
+        let zpool = Zpool::new(zpool_id, sled_id, Uuid::new_v4());
         datastore.zpool_upsert(zpool).await.expect("failed to upsert zpool");
 
         // Inserting a new dataset should succeed.
diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs
index 860725cc08..67d5684bbf 100644
--- a/nexus/db-queries/src/db/datastore/inventory.rs
+++ b/nexus/db-queries/src/db/datastore/inventory.rs
@@ -45,6 +45,7 @@ use nexus_db_model::InvRotPage;
 use nexus_db_model::InvServiceProcessor;
 use nexus_db_model::InvSledAgent;
 use nexus_db_model::InvSledOmicronZones;
+use nexus_db_model::InvZpool;
 use nexus_db_model::RotPageWhichEnum;
 use nexus_db_model::SledRole;
 use nexus_db_model::SledRoleEnum;
@@ -137,6 +138,18 @@ impl DataStore {
             })
             .collect();
 
+        // Pull zpools out of all sled agents
+        let zpools: Vec<_> = collection
+            .sled_agents
+            .iter()
+            .flat_map(|(sled_id, sled_agent)| {
+                sled_agent
+                    .zpools
+                    .iter()
+                    .map(|pool| InvZpool::new(collection_id, *sled_id, pool))
+            })
+            .collect();
+
         // Partition the sled agents into those with an associated baseboard id
         // and those without one.  We handle these pretty differently.
         let (sled_agents_baseboards, sled_agents_no_baseboards): (
@@ -670,6 +683,25 @@ impl DataStore {
                 }
             }
 
+            // Insert rows for all the zpools we found.
+            {
+                use db::schema::inv_zpool::dsl;
+
+                let batch_size = SQL_BATCH_SIZE.get().try_into().unwrap();
+                let mut zpools = zpools.into_iter();
+                loop {
+                    let some_zpools =
+                        zpools.by_ref().take(batch_size).collect::<Vec<_>>();
+                    if some_zpools.is_empty() {
+                        break;
+                    }
+                    let _ = diesel::insert_into(dsl::inv_zpool)
+                        .values(some_zpools)
+                        .execute_async(&conn)
+                        .await?;
+                }
+            }
+
             // Insert rows for the sled agents that we found.  In practice, we'd
             // expect these to all have baseboards (if using Oxide hardware) or
             // none have baseboards (if not).
@@ -1470,6 +1502,34 @@ impl DataStore {
             disks
         };
 
+        // Mapping of "Sled ID" -> "All zpools reported by that sled"
+        let zpools: BTreeMap<Uuid, Vec<nexus_types::inventory::Zpool>> = {
+            use db::schema::inv_zpool::dsl;
+
+            let mut zpools =
+                BTreeMap::<Uuid, Vec<nexus_types::inventory::Zpool>>::new();
+            let mut paginator = Paginator::new(batch_size);
+            while let Some(p) = paginator.next() {
+                let batch = paginated_multicolumn(
+                    dsl::inv_zpool,
+                    (dsl::sled_id, dsl::id),
+                    &p.current_pagparams(),
+                )
+                .filter(dsl::inv_collection_id.eq(id))
+                .select(InvZpool::as_select())
+                .load_async(&*conn)
+                .await
+                .map_err(|e| {
+                    public_error_from_diesel(e, ErrorHandler::Server)
+                })?;
+                paginator = p.found_batch(&batch, &|row| (row.sled_id, row.id));
+                for zpool in batch {
+                    zpools.entry(zpool.sled_id).or_default().push(zpool.into());
+                }
+            }
+            zpools
+        };
+
         // Collect the unique baseboard ids referenced by SPs, RoTs, and Sled
         // Agents.
         let baseboard_id_ids: BTreeSet<_> = sps
@@ -1577,6 +1637,10 @@ impl DataStore {
                             .get(&sled_id)
                             .map(|disks| disks.to_vec())
                             .unwrap_or_default(),
+                        zpools: zpools
+                            .get(&sled_id)
+                            .map(|zpools| zpools.to_vec())
+                            .unwrap_or_default(),
                     };
                     Ok((sled_id, sled_agent))
                 })
diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs
index b2b564cc09..ab00d876b8 100644
--- a/nexus/db-queries/src/db/datastore/mod.rs
+++ b/nexus/db-queries/src/db/datastore/mod.rs
@@ -661,17 +661,59 @@ mod test {
         sled_id: Uuid,
         physical_disk_id: Uuid,
     ) -> Uuid {
-        let zpool_id = Uuid::new_v4();
-        let zpool = Zpool::new(
-            zpool_id,
+        let zpool_id = create_test_zpool_not_in_inventory(
+            datastore,
             sled_id,
             physical_disk_id,
-            test_zpool_size().into(),
-        );
+        )
+        .await;
+
+        add_test_zpool_to_inventory(datastore, zpool_id, sled_id).await;
+
+        zpool_id
+    }
+
+    // Creates a test zpool, returns its UUID.
+    //
+    // However, this helper doesn't add the zpool to the inventory just yet.
+    async fn create_test_zpool_not_in_inventory(
+        datastore: &DataStore,
+        sled_id: Uuid,
+        physical_disk_id: Uuid,
+    ) -> Uuid {
+        let zpool_id = Uuid::new_v4();
+        let zpool = Zpool::new(zpool_id, sled_id, physical_disk_id);
         datastore.zpool_upsert(zpool).await.unwrap();
         zpool_id
     }
 
+    // Adds a test zpool into the inventory, with a randomly generated
+    // collection UUID.
+    async fn add_test_zpool_to_inventory(
+        datastore: &DataStore,
+        zpool_id: Uuid,
+        sled_id: Uuid,
+    ) {
+        use db::schema::inv_zpool::dsl;
+
+        let inv_collection_id = Uuid::new_v4();
+        let time_collected = Utc::now();
+        let inv_pool = nexus_db_model::InvZpool {
+            inv_collection_id,
+            time_collected,
+            id: zpool_id,
+            sled_id,
+            total_size: test_zpool_size().into(),
+        };
+        diesel::insert_into(dsl::inv_zpool)
+            .values(inv_pool)
+            .execute_async(
+                &*datastore.pool_connection_for_tests().await.unwrap(),
+            )
+            .await
+            .unwrap();
+    }
+
     fn create_test_disk_create_params(
         name: &str,
         size: ByteCount,
@@ -1156,6 +1198,103 @@ mod test {
         logctx.cleanup_successful();
     }
 
+    #[tokio::test]
+    async fn test_region_allocation_only_operates_on_zpools_in_inventory() {
+        let logctx = dev::test_setup_log(
+            "test_region_allocation_only_operates_on_zpools_in_inventory",
+        );
+        let mut db = test_setup_database(&logctx.log).await;
+        let (opctx, datastore) = datastore_test(&logctx, &db).await;
+
+        // Create a sled...
+        let sled_id = create_test_sled(&datastore).await;
+
+        // ... and a disk on that sled...
+        let physical_disk_id = create_test_physical_disk(
+            &datastore,
+            &opctx,
+            sled_id,
+            PhysicalDiskKind::U2,
+        )
+        .await;
+
+        // Create enough zpools for region allocation to succeed
+        let zpool_ids: Vec<Uuid> = stream::iter(0..REGION_REDUNDANCY_THRESHOLD)
+            .then(|_| {
+                create_test_zpool_not_in_inventory(
+                    &datastore,
+                    sled_id,
+                    physical_disk_id,
+                )
+            })
+            .collect()
+            .await;
+
+        let bogus_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 8080, 0, 0);
+
+        // 1 dataset per zpool
+        stream::iter(zpool_ids.clone())
+            .then(|zpool_id| {
+                let id = Uuid::new_v4();
+                let dataset = Dataset::new(
+                    id,
+                    zpool_id,
+                    bogus_addr,
+                    DatasetKind::Crucible,
+                );
+                let datastore = datastore.clone();
+                async move {
+                    datastore.dataset_upsert(dataset).await.unwrap();
+                    id
+                }
+            })
+            .collect::<Vec<_>>()
+            .await;
+
+        // Allocate regions from the datasets for this volume.
+        let params = create_test_disk_create_params(
+            "disk1",
+            ByteCount::from_mebibytes_u32(500),
+        );
+        let volume1_id = Uuid::new_v4();
+        let err = datastore
+            .region_allocate(
+                &opctx,
+                volume1_id,
+                &params.disk_source,
+                params.size,
+                &RegionAllocationStrategy::Random { seed: Some(0) },
+            )
+            .await
+            .unwrap_err();
+
+        let expected = "Not enough zpool space to allocate disks";
+        assert!(
+            err.to_string().contains(expected),
+            "Saw error: \'{err}\', but expected \'{expected}\'"
+        );
+        assert!(matches!(err, Error::InsufficientCapacity { .. }));
+
+        // If we add the zpools to the inventory and try again, the allocation
+        // will succeed.
+        for zpool_id in zpool_ids {
+            add_test_zpool_to_inventory(&datastore, zpool_id, sled_id).await;
+        }
+        datastore
+            .region_allocate(
+                &opctx,
+                volume1_id,
+                &params.disk_source,
+                params.size,
+                &RegionAllocationStrategy::Random { seed: Some(0) },
+            )
+            .await
+            .expect("Allocation should have worked after adding zpools to inventory");
+
+        let _ = db.cleanup().await;
+        logctx.cleanup_successful();
+    }
+
     #[tokio::test]
     async fn test_region_allocation_not_enough_zpools() {
         let logctx =
diff --git a/nexus/db-queries/src/db/datastore/region.rs b/nexus/db-queries/src/db/datastore/region.rs
index 912565cb2c..52e0ce4d88 100644
--- a/nexus/db-queries/src/db/datastore/region.rs
+++ b/nexus/db-queries/src/db/datastore/region.rs
@@ -128,17 +128,16 @@ impl DataStore {
         let (blocks_per_extent, extent_count) =
             Self::get_crucible_allocation(&block_size, size);
 
+        let query = crate::db::queries::region_allocation::RegionAllocate::new(
+            volume_id,
+            block_size.to_bytes() as u64,
+            blocks_per_extent,
+            extent_count,
+            allocation_strategy,
+        );
+        let conn = self.pool_connection_authorized(&opctx).await?;
         let dataset_and_regions: Vec<(Dataset, Region)> =
-            crate::db::queries::region_allocation::RegionAllocate::new(
-                volume_id,
-                block_size.to_bytes() as u64,
-                blocks_per_extent,
-                extent_count,
-                allocation_strategy,
-            )
-            .get_results_async(&*self.pool_connection_authorized(&opctx).await?)
-            .await
-            .map_err(|e| {
+            query.get_results_async(&*conn).await.map_err(|e| {
                 crate::db::queries::region_allocation::from_diesel(e)
             })?;
 
diff --git a/nexus/db-queries/src/db/datastore/zpool.rs b/nexus/db-queries/src/db/datastore/zpool.rs
index 5424c126eb..b894d5c509 100644
--- a/nexus/db-queries/src/db/datastore/zpool.rs
+++ b/nexus/db-queries/src/db/datastore/zpool.rs
@@ -46,7 +46,6 @@ impl DataStore {
                 .set((
                     dsl::time_modified.eq(Utc::now()),
                     dsl::sled_id.eq(excluded(dsl::sled_id)),
-                    dsl::total_size.eq(excluded(dsl::total_size)),
                 )),
         )
         .insert_and_get_result_async(
diff --git a/nexus/db-queries/src/db/queries/region_allocation.rs b/nexus/db-queries/src/db/queries/region_allocation.rs
index 96fa2058af..a657d21c97 100644
--- a/nexus/db-queries/src/db/queries/region_allocation.rs
+++ b/nexus/db-queries/src/db/queries/region_allocation.rs
@@ -17,7 +17,7 @@ use diesel::query_builder::{AstPass, Query, QueryFragment, QueryId};
 use diesel::result::Error as DieselError;
 use diesel::PgBinaryExpressionMethods;
 use diesel::{
-    sql_types, BoolExpressionMethods, Column, CombineDsl, ExpressionMethods,
+    sql_types, BoolExpressionMethods, CombineDsl, ExpressionMethods,
     Insertable, IntoSql, JoinOnDsl, NullableExpressionMethods, QueryDsl,
     RunQueryDsl,
 };
@@ -316,7 +316,11 @@ impl CandidateZpools {
         // into a BigInt, not a Numeric. I welcome a better solution.
         let it_will_fit = (old_zpool_usage::dsl::size_used
             + diesel::dsl::sql(&zpool_size_delta.to_string()))
-        .le(diesel::dsl::sql(zpool_dsl::total_size::NAME));
+        .le(diesel::dsl::sql(
+            "(SELECT total_size FROM omicron.public.inv_zpool WHERE
+              inv_zpool.id = old_zpool_usage.pool_id
+              ORDER BY inv_zpool.time_collected DESC LIMIT 1)",
+        ));
 
         // We need to join on the sled table to access provision_state.
         let with_sled = sled_dsl::sled.on(zpool_dsl::sled_id.eq(sled_dsl::id));
diff --git a/nexus/inventory/src/builder.rs b/nexus/inventory/src/builder.rs
index 80db03f18b..2e482fcebf 100644
--- a/nexus/inventory/src/builder.rs
+++ b/nexus/inventory/src/builder.rs
@@ -26,6 +26,7 @@ use nexus_types::inventory::RotPageWhich;
 use nexus_types::inventory::RotState;
 use nexus_types::inventory::ServiceProcessor;
 use nexus_types::inventory::SledAgent;
+use nexus_types::inventory::Zpool;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::sync::Arc;
@@ -451,6 +452,7 @@ impl CollectionBuilder {
                 return Ok(());
             }
         };
+        let time_collected = now_db_precision();
         let sled = SledAgent {
             source: source.to_string(),
             sled_agent_address,
@@ -459,9 +461,14 @@ impl CollectionBuilder {
             usable_hardware_threads: inventory.usable_hardware_threads,
             usable_physical_ram: inventory.usable_physical_ram,
             reservoir_size: inventory.reservoir_size,
-            time_collected: now_db_precision(),
+            time_collected,
             sled_id,
             disks: inventory.disks.into_iter().map(|d| d.into()).collect(),
+            zpools: inventory
+                .zpools
+                .into_iter()
+                .map(|z| Zpool::new(time_collected, z))
+                .collect(),
         };
 
         if let Some(previous) = self.sleds.get(&sled_id) {
diff --git a/nexus/inventory/src/examples.rs b/nexus/inventory/src/examples.rs
index 286ded7276..5cc6b687d4 100644
--- a/nexus/inventory/src/examples.rs
+++ b/nexus/inventory/src/examples.rs
@@ -313,6 +313,7 @@ pub fn representative() -> Representative {
             slot: 3,
         },
     ];
+    let zpools = vec![];
 
     builder
         .found_sled_inventory(
@@ -326,6 +327,7 @@ pub fn representative() -> Representative {
                 },
                 sled_agent_client::types::SledRole::Gimlet,
                 disks,
+                zpools,
             ),
         )
         .unwrap();
@@ -351,6 +353,7 @@ pub fn representative() -> Representative {
                 },
                 sled_agent_client::types::SledRole::Scrimlet,
                 vec![],
+                vec![],
             ),
         )
         .unwrap();
@@ -371,6 +374,7 @@ pub fn representative() -> Representative {
                 },
                 sled_agent_client::types::SledRole::Gimlet,
                 vec![],
+                vec![],
             ),
         )
         .unwrap();
@@ -389,6 +393,7 @@ pub fn representative() -> Representative {
                 sled_agent_client::types::Baseboard::Unknown,
                 sled_agent_client::types::SledRole::Gimlet,
                 vec![],
+                vec![],
             ),
         )
         .unwrap();
@@ -486,6 +491,7 @@ pub fn sled_agent(
     baseboard: sled_agent_client::types::Baseboard,
     sled_role: sled_agent_client::types::SledRole,
     disks: Vec<sled_agent_client::types::InventoryDisk>,
+    zpools: Vec<sled_agent_client::types::InventoryZpool>,
 ) -> sled_agent_client::types::Inventory {
     sled_agent_client::types::Inventory {
         baseboard,
@@ -496,5 +502,6 @@ pub fn sled_agent(
         usable_hardware_threads: 10,
         usable_physical_ram: ByteCount::from(1024 * 1024),
         disks,
+        zpools,
     }
 }
diff --git a/nexus/reconfigurator/execution/src/datasets.rs b/nexus/reconfigurator/execution/src/datasets.rs
index 5e677b98be..6da24e4279 100644
--- a/nexus/reconfigurator/execution/src/datasets.rs
+++ b/nexus/reconfigurator/execution/src/datasets.rs
@@ -200,7 +200,6 @@ mod tests {
                     zpool_name.id(),
                     sled_id,
                     Uuid::new_v4(), // physical_disk_id
-                    (1 << 30).try_into().unwrap(), // total_size
                 );
                 datastore
                     .zpool_upsert(zpool)
@@ -269,8 +268,7 @@ mod tests {
             let zpool = Zpool::new(
                 new_zpool_id,
                 sled_id,
-                Uuid::new_v4(),                // physical_disk_id
-                (1 << 30).try_into().unwrap(), // total_size
+                Uuid::new_v4(), // physical_disk_id
             );
             datastore
                 .zpool_upsert(zpool)
diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs
index 8e8283f37e..e224e3c6df 100644
--- a/nexus/reconfigurator/planning/src/system.rs
+++ b/nexus/reconfigurator/planning/src/system.rs
@@ -476,6 +476,7 @@ impl Sled {
                 usable_hardware_threads: 10,
                 usable_physical_ram: ByteCount::from(1024 * 1024),
                 disks: vec![],
+                zpools: vec![],
             }
         };
 
@@ -558,6 +559,7 @@ impl Sled {
             usable_hardware_threads: inv_sled_agent.usable_hardware_threads,
             usable_physical_ram: inv_sled_agent.usable_physical_ram,
             disks: vec![],
+            zpools: vec![],
         };
 
         Sled {
diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs
index 68f57d31f3..d387998f6a 100644
--- a/nexus/src/app/mod.rs
+++ b/nexus/src/app/mod.rs
@@ -562,6 +562,12 @@ impl Nexus {
         Some(rustls_cfg)
     }
 
+    // Called to trigger inventory collection.
+    pub(crate) fn activate_inventory_collection(&self) {
+        self.background_tasks
+            .activate(&self.background_tasks.task_inventory_collection);
+    }
+
     // Called to hand off management of external servers to Nexus.
     pub(crate) async fn set_servers(
         &self,
diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs
index 9905df04ff..35f3337625 100644
--- a/nexus/src/app/rack.rs
+++ b/nexus/src/app/rack.rs
@@ -923,8 +923,7 @@ impl super::Nexus {
 
         // Trigger an inventory collection so that the newly added sled is known
         // about.
-        self.background_tasks
-            .activate(&self.background_tasks.task_inventory_collection);
+        self.activate_inventory_collection();
 
         Ok(())
     }
diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs
index 4ad8dc36ce..06e50f2ecd 100644
--- a/nexus/src/app/sled.rs
+++ b/nexus/src/app/sled.rs
@@ -264,12 +264,7 @@ impl super::Nexus {
                 )
                 .fetch()
                 .await?;
-        let zpool = db::model::Zpool::new(
-            id,
-            sled_id,
-            db_disk.uuid(),
-            info.size.into(),
-        );
+        let zpool = db::model::Zpool::new(id, sled_id, db_disk.uuid());
         self.db_datastore.zpool_upsert(zpool).await?;
         Ok(())
     }
diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs
index 03470a4f3a..d143858bf4 100644
--- a/nexus/src/lib.rs
+++ b/nexus/src/lib.rs
@@ -30,7 +30,9 @@ use nexus_config::NexusConfig;
 use nexus_types::deployment::Blueprint;
 use nexus_types::external_api::views::SledProvisionPolicy;
 use nexus_types::internal_api::params::ServiceKind;
+use nexus_types::inventory::Collection;
 use omicron_common::address::IpRange;
+use omicron_common::api::external::Error;
 use omicron_common::api::internal::shared::{
     ExternalPortDiscovery, RackNetworkConfig, SwitchLocation,
 };
@@ -354,6 +356,17 @@ impl nexus_test_interface::NexusServer for Server {
             .unwrap();
     }
 
+    async fn inventory_collect_and_get_latest_collection(
+        &self,
+    ) -> Result<Option<Collection>, Error> {
+        let nexus = &self.apictx.nexus;
+
+        nexus.activate_inventory_collection();
+
+        let opctx = nexus.opctx_for_internal_api();
+        nexus.datastore().inventory_get_latest_collection(&opctx).await
+    }
+
     async fn close(mut self) {
         self.apictx
             .nexus
diff --git a/nexus/test-interface/src/lib.rs b/nexus/test-interface/src/lib.rs
index d9bb69276a..2e3428a1dd 100644
--- a/nexus/test-interface/src/lib.rs
+++ b/nexus/test-interface/src/lib.rs
@@ -34,6 +34,8 @@
 use async_trait::async_trait;
 use nexus_config::NexusConfig;
 use nexus_types::deployment::Blueprint;
+use nexus_types::inventory::Collection;
+use omicron_common::api::external::Error;
 use slog::Logger;
 use std::net::{SocketAddr, SocketAddrV6};
 use uuid::Uuid;
@@ -91,5 +93,9 @@ pub trait NexusServer: Send + Sync + 'static {
         address: SocketAddrV6,
     );
 
+    async fn inventory_collect_and_get_latest_collection(
+        &self,
+    ) -> Result<Option<Collection>, Error>;
+
     async fn close(self);
 }
diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs
index 7331843000..f4b5d9fa46 100644
--- a/nexus/test-utils/src/resource_helpers.rs
+++ b/nexus/test-utils/src/resource_helpers.rs
@@ -32,13 +32,18 @@ use nexus_types::identity::Resource;
 use nexus_types::internal_api::params as internal_params;
 use omicron_common::api::external::ByteCount;
 use omicron_common::api::external::Disk;
+use omicron_common::api::external::Error;
 use omicron_common::api::external::IdentityMetadataCreateParams;
 use omicron_common::api::external::Instance;
 use omicron_common::api::external::InstanceCpuCount;
 use omicron_common::api::external::NameOrId;
 use omicron_sled_agent::sim::SledAgent;
+use omicron_test_utils::dev::poll::wait_for_condition;
+use omicron_test_utils::dev::poll::CondCheckError;
+use slog::debug;
 use std::net::IpAddr;
 use std::sync::Arc;
+use std::time::Duration;
 use uuid::Uuid;
 
 pub async fn objects_list_page_authz<ItemType>(
@@ -793,6 +798,56 @@ impl DiskTest {
                 .await;
         }
 
+        let log = &cptestctx.logctx.log;
+
+        // Wait until Nexus has successfully completed an inventory collection
+        // which includes this zpool
+        wait_for_condition(
+            || async {
+                let result = cptestctx
+                    .server
+                    .inventory_collect_and_get_latest_collection()
+                    .await;
+                let log_result = match &result {
+                    Ok(Some(_)) => Ok("found"),
+                    Ok(None) => Ok("not found"),
+                    Err(error) => Err(error),
+                };
+                debug!(
+                    log,
+                    "attempt to fetch latest inventory collection";
+                    "result" => ?log_result,
+                );
+
+                match result {
+                    Ok(None) => Err(CondCheckError::NotYet),
+                    Ok(Some(c)) => {
+                        let all_zpools = c
+                            .sled_agents
+                            .values()
+                            .flat_map(|sled_agent| {
+                                sled_agent.zpools.iter().map(|z| z.id)
+                            })
+                            .collect::<std::collections::HashSet<Uuid>>();
+
+                        if all_zpools.contains(&zpool.id) {
+                            Ok(())
+                        } else {
+                            Err(CondCheckError::NotYet)
+                        }
+                    }
+                    Err(Error::ServiceUnavailable { .. }) => {
+                        Err(CondCheckError::NotYet)
+                    }
+                    Err(error) => Err(CondCheckError::Failed(error)),
+                }
+            },
+            &Duration::from_millis(50),
+            &Duration::from_secs(30),
+        )
+        .await
+        .expect("expected to find inventory collection");
+
         self.zpools.push(zpool);
     }
 
diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs
index b0afe51a76..40da26047b 100644
--- a/nexus/types/src/inventory.rs
+++ b/nexus/types/src/inventory.rs
@@ -367,6 +367,23 @@ impl From<sled_agent_client::types::InventoryDisk> for PhysicalDisk {
     }
 }
 
+/// A zpool reported by a sled agent.
+#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
+pub struct Zpool {
+    pub time_collected: DateTime<Utc>,
+    pub id: Uuid,
+    pub total_size: ByteCount,
+}
+
+impl Zpool {
+    pub fn new(
+        time_collected: DateTime<Utc>,
+        pool: sled_agent_client::types::InventoryZpool,
+    ) -> Zpool {
+        Zpool { time_collected, id: pool.id, total_size: pool.total_size }
+    }
+}
+
 /// Inventory reported by sled agent
 ///
 /// This is a software notion of a sled, distinct from an underlying baseboard.
@@ -385,6 +402,7 @@ pub struct SledAgent {
     pub usable_physical_ram: ByteCount,
     pub reservoir_size: ByteCount,
     pub disks: Vec<PhysicalDisk>,
+    pub zpools: Vec<Zpool>,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json
index 8de8be430f..5c43c40e56 100644
--- a/openapi/sled-agent.json
+++ b/openapi/sled-agent.json
@@ -5267,6 +5267,12 @@
           },
           "usable_physical_ram": {
             "$ref": "#/components/schemas/ByteCount"
+          },
+          "zpools": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/InventoryZpool"
+            }
           }
         },
         "required": [
@@ -5277,7 +5283,8 @@
           "sled_id",
           "sled_role",
           "usable_hardware_threads",
-          "usable_physical_ram"
+          "usable_physical_ram",
+          "zpools"
         ]
       },
       "InventoryDisk": {
@@ -5301,6 +5308,23 @@
           "variant"
         ]
       },
+      "InventoryZpool": {
+        "description": "Identifies information about zpools managed by the control plane",
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string",
+            "format": "uuid"
+          },
+          "total_size": {
+            "$ref": "#/components/schemas/ByteCount"
+          }
+        },
+        "required": [
+          "id",
+          "total_size"
+        ]
+      },
       "IpNet": {
         "oneOf": [
           {
diff --git a/schema/crdb/44.0.0/up01.sql b/schema/crdb/44.0.0/up01.sql
new file mode 100644
index 0000000000..a13133ee67
--- /dev/null
+++ b/schema/crdb/44.0.0/up01.sql
@@ -0,0 +1 @@
+ALTER TABLE omicron.public.zpool DROP COLUMN IF EXISTS total_size;
diff --git a/schema/crdb/44.0.0/up02.sql b/schema/crdb/44.0.0/up02.sql
new file mode 100644
index 0000000000..601888949b
--- /dev/null
+++ b/schema/crdb/44.0.0/up02.sql
@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS omicron.public.inv_zpool (
+    inv_collection_id UUID NOT NULL,
+    time_collected TIMESTAMPTZ NOT NULL,
+
+    id UUID NOT NULL,
+    sled_id UUID NOT NULL,
+    total_size INT NOT NULL,
+
+    PRIMARY KEY (inv_collection_id, sled_id, id)
+);
diff --git a/schema/crdb/44.0.0/up03.sql b/schema/crdb/44.0.0/up03.sql
new file mode 100644
index 0000000000..10195be482
--- /dev/null
+++ b/schema/crdb/44.0.0/up03.sql
@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS inv_zpool_by_id_and_time ON omicron.public.inv_zpool (id, time_collected DESC);
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 255cdc8135..c7a4db09f1 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -462,10 +462,12 @@ CREATE TABLE IF NOT EXISTS omicron.public.virtual_provisioning_resource (
     ram_provisioned INT8 NOT NULL
 );
 
-/*
- * ZPools of Storage, attached to Sleds.
- * These are backed by a single physical disk.
- */
+-- ZPools of Storage, attached to Sleds.
+-- These are backed by a single physical disk.
+--
+-- For information about the provisioned zpool, reference the
+-- "omicron.public.inv_zpool" table, which returns information
+-- that has actually been returned from the underlying sled.
 CREATE TABLE IF NOT EXISTS omicron.public.zpool (
     /* Identity metadata (asset) */
     id UUID PRIMARY KEY,
@@ -478,9 +480,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.zpool (
     sled_id UUID NOT NULL,
 
     /* FK into the Physical Disk table */
-    physical_disk_id UUID NOT NULL,
-
-    total_size INT NOT NULL
+    physical_disk_id UUID NOT NULL
 );
 
 /* Create an index on the physical disk id */
@@ -2986,6 +2986,28 @@ CREATE TABLE IF NOT EXISTS omicron.public.inv_physical_disk (
     PRIMARY KEY (inv_collection_id, sled_id, slot)
 );
 
+CREATE TABLE IF NOT EXISTS omicron.public.inv_zpool (
+    -- where this observation came from
+    -- (foreign key into `inv_collection` table)
+    inv_collection_id UUID NOT NULL,
+    -- when this observation was made
+    time_collected TIMESTAMPTZ NOT NULL,
+
+    -- The control plane ID of the zpool
+    id UUID NOT NULL,
+    sled_id UUID NOT NULL,
+    total_size INT NOT NULL,
+
+    -- PK consisting of:
+    -- - Which collection this was
+    -- - The sled reporting the disk
+    -- - The slot in which this disk was found
+    PRIMARY KEY (inv_collection_id, sled_id, id)
+);
+
+-- Allow looking up the most recent Zpool by ID
+CREATE INDEX IF NOT EXISTS inv_zpool_by_id_and_time ON omicron.public.inv_zpool (id, time_collected DESC);
+
 CREATE TABLE IF NOT EXISTS omicron.public.inv_sled_omicron_zones (
     -- where this observation came from
     -- (foreign key into `inv_collection` table)
@@ -3670,7 +3692,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '43.0.0', NULL)
+    ( TRUE, NOW(), NOW(), '44.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;
diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs
index 82294326b6..f74438a678 100644
--- a/sled-agent/src/params.rs
+++ b/sled-agent/src/params.rs
@@ -882,6 +882,13 @@ pub struct InventoryDisk {
     pub slot: i64,
 }
 
+/// Identifies information about zpools managed by the control plane
+#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]
+pub struct InventoryZpool {
+    pub id: Uuid,
+    pub total_size: ByteCount,
+}
+
 /// Identity and basic status information about this sled agent
 #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]
 pub struct Inventory {
@@ -893,6 +900,7 @@ pub struct Inventory {
     pub usable_physical_ram: ByteCount,
     pub reservoir_size: ByteCount,
     pub disks: Vec<InventoryDisk>,
+    pub zpools: Vec<InventoryZpool>,
 }
 
 #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]
diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs
index 425f4e31a4..b772a60347 100644
--- a/sled-agent/src/sim/sled_agent.rs
+++ b/sled-agent/src/sim/sled_agent.rs
@@ -739,6 +739,8 @@ impl SledAgent {
             }
             SocketAddr::V6(v6) => v6,
         };
+
+        let storage = self.storage.lock().await;
         Ok(Inventory {
             sled_id: self.id,
             sled_agent_address,
@@ -753,10 +755,7 @@ impl SledAgent {
                 self.config.hardware.reservoir_ram,
             )
             .context("reservoir_size")?,
-            disks: self
-                .storage
-                .lock()
-                .await
+            disks: storage
                 .physical_disks()
                 .iter()
                 .map(|(identity, info)| crate::params::InventoryDisk {
@@ -765,6 +764,16 @@ impl SledAgent {
                     slot: info.slot,
                 })
                 .collect(),
+            zpools: storage
+                .zpools()
+                .iter()
+                .map(|(id, zpool)| {
+                    Ok(crate::params::InventoryZpool {
+                        id: *id,
+                        total_size: ByteCount::try_from(zpool.total_size())?,
+                    })
+                })
+                .collect::<Result<Vec<_>, anyhow::Error>>()?,
         })
     }
 
diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs
index d50cd296ae..8fb362c5b7 100644
--- a/sled-agent/src/sim/storage.rs
+++ b/sled-agent/src/sim/storage.rs
@@ -478,13 +478,14 @@ pub(crate) struct PhysicalDisk {
     pub(crate) slot: i64,
 }
 
-struct Zpool {
+pub(crate) struct Zpool {
     datasets: HashMap<Uuid, CrucibleServer>,
+    total_size: u64,
 }
 
 impl Zpool {
-    fn new() -> Self {
-        Zpool { datasets: HashMap::new() }
+    fn new(total_size: u64) -> Self {
+        Zpool { datasets: HashMap::new(), total_size }
     }
 
     fn insert_dataset(
@@ -501,6 +502,10 @@ impl Zpool {
             .expect("Failed to get the dataset we just inserted")
     }
 
+    pub fn total_size(&self) -> u64 {
+        self.total_size
+    }
+
     pub async fn get_dataset_for_region(
         &self,
         region_id: Uuid,
@@ -614,7 +619,7 @@ impl Storage {
         size: u64,
     ) {
         // Update our local data
-        self.zpools.insert(zpool_id, Zpool::new());
+        self.zpools.insert(zpool_id, Zpool::new(size));
 
         // Notify Nexus
         let request = ZpoolPutRequest {
@@ -629,6 +634,10 @@ impl Storage {
             .expect("Failed to notify Nexus about new Zpool");
     }
 
+    /// Returns an immutable reference to all zpools
+    pub fn zpools(&self) -> &HashMap<Uuid, Zpool> {
+        &self.zpools
+    }
     /// Adds a Dataset to the sled's simulated storage.
     pub async fn insert_dataset(
         &mut self,
diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs
index 91ddfde153..713b10428f 100644
--- a/sled-agent/src/sled_agent.rs
+++ b/sled-agent/src/sled_agent.rs
@@ -1107,18 +1107,22 @@ impl SledAgent {
         } else {
             crate::params::SledRole::Gimlet
         };
-        let disks = self
-            .storage()
-            .get_latest_resources()
-            .await
-            .disks()
-            .iter()
-            .map(|(identity, (disk, _pool))| crate::params::InventoryDisk {
+
+        let mut disks = vec![];
+        let mut zpools = vec![];
+        for (identity, (disk, pool)) in
+            self.storage().get_latest_resources().await.disks().iter()
+        {
+            disks.push(crate::params::InventoryDisk {
                 identity: identity.clone(),
                 variant: disk.variant(),
                 slot: disk.slot(),
-            })
-            .collect();
+            });
+            zpools.push(crate::params::InventoryZpool {
+                id: pool.name.id(),
+                total_size: ByteCount::try_from(pool.info.size())?,
+            });
+        }
 
         Ok(Inventory {
             sled_id,
@@ -1129,6 +1133,7 @@ impl SledAgent {
             usable_physical_ram: ByteCount::try_from(usable_physical_ram)?,
             reservoir_size,
             disks,
+            zpools,
         })
     }
 }

From 255b14acdfd882d498e2013cf95fc405ca959080 Mon Sep 17 00:00:00 2001
From: "Andrew J. Stone" <andrew@oxidecomputer.com>
Date: Fri, 15 Mar 2024 22:09:17 -0400
Subject: [PATCH 123/157] [sa] Move retry interval outside loop (#5276)

Fixes #5275
---
 sled-agent/src/nexus.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/sled-agent/src/nexus.rs b/sled-agent/src/nexus.rs
index 59ae797ab4..12fcc05ce3 100644
--- a/sled-agent/src/nexus.rs
+++ b/sled-agent/src/nexus.rs
@@ -446,10 +446,11 @@ impl NexusNotifierTask {
             (Some(tx), rx)
         };
 
+        const RETRY_TIMEOUT: Duration = Duration::from_secs(2);
+        let mut interval = interval(RETRY_TIMEOUT);
+        interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
+
         loop {
-            const RETRY_TIMEOUT: Duration = Duration::from_secs(2);
-            let mut interval = interval(RETRY_TIMEOUT);
-            interval.set_missed_tick_behavior(MissedTickBehavior::Delay);
             tokio::select! {
                 req = self.rx.recv() => {
                     let Some(req) = req else {

From 05ed7903e584a1ef641c68a641b9d0cafc168f08 Mon Sep 17 00:00:00 2001
From: Levon Tarver <11586085+internet-diglett@users.noreply.github.com>
Date: Sat, 16 Mar 2024 16:29:39 -0500
Subject: [PATCH 124/157] Rpws for all networking (#4822)

Overview
---
This PR ensures the rest of our `dpd` configuration is covered by a RPW
to help recover state in the event of `dendrite` crashing, the switch
zone restarting / being replaced, or the sled restarting. This is
accomplished via a background task in Nexus that periodically ensures
`dpd` is up to date with the tables in Nexus. The tradeoffs of this
design is that we don't track versioning and reconcile the entire state
every time, but since the actual number of ports will never be that high
(relative to something like NAT entries) the tradeoff of less efficiency
for much greater simplicity seems to make sense today, and it requires
much less rework in Nexus and Dendrite should we choose to replace this
strategy down the road.

Tasks
---
- [x] Ensure that Service Zones configured during rss, cold boot, and
nexus have their NAT entries added to the NAT RPW table (extracted into
#4857)
- [x] Create background task that periodically reconciles switch port
configuration for dendrite instances
- [x] Move switch zone uplink SMF property updates to RPW
- [x] Move routing updates (via mg) to RPW
    - [x] Static Routing
    - [x] BGP
- [x] Move bootstore updates to RPW
- [x] Move loopback address management to RPW
- [x] Move Nexus-side switch zone service on-demand lookups as outlined
in #5092

Verifications Performed
---
- [x] Basic instance deployment
- [x] Loopback Address Creation
- [x] BGP configuration (a4x2)
- [ ] BGP configuration modification (a4x2)
- [x] Static routing
- [x] Static routing configuration modification

Related
---
Closes #4715
Closes #4650

Depends on https://github.com/oxidecomputer/dendrite/pull/838

---------

Co-authored-by: Levon Tarver <levon@oxidecomputer.com>
---
 .github/buildomat/jobs/deploy.sh              |   18 +-
 clients/mg-admin-client/src/lib.rs            |   17 +
 dev-tools/omdb/tests/env.out                  |   12 +
 dev-tools/omdb/tests/successes.out            |   15 +-
 nexus-config/src/nexus_config.rs              |   17 +
 nexus/db-model/src/address_lot.rs             |    6 +-
 nexus/db-model/src/bootstore.rs               |   18 +-
 nexus/db-model/src/schema.rs                  |   12 +-
 nexus/db-model/src/unsigned.rs                |    1 +
 .../src/db/datastore/address_lot.rs           |  113 +-
 nexus/db-queries/src/db/datastore/bgp.rs      |  130 +-
 .../db-queries/src/db/datastore/bootstore.rs  |   50 +-
 nexus/db-queries/src/db/datastore/rack.rs     |   15 +
 nexus/examples/config.toml                    |    5 +-
 nexus/src/app/background/bfd.rs               |   45 +-
 nexus/src/app/background/init.rs              |   33 +-
 nexus/src/app/background/mod.rs               |    2 +
 nexus/src/app/background/nat_cleanup.rs       |   36 +-
 nexus/src/app/background/networking.rs        |  113 ++
 .../app/background/sync_service_zone_nat.rs   |   40 +-
 .../background/sync_switch_configuration.rs   | 1418 +++++++++++++++++
 nexus/src/app/bfd.rs                          |   15 +-
 nexus/src/app/bgp.rs                          |   16 +-
 nexus/src/app/instance_network.rs             |   18 +-
 nexus/src/app/mod.rs                          |   86 +-
 nexus/src/app/probe.rs                        |    9 +-
 nexus/src/app/rack.rs                         |  268 +---
 .../src/app/sagas/loopback_address_create.rs  |  178 ---
 .../src/app/sagas/loopback_address_delete.rs  |  190 ---
 nexus/src/app/sagas/mod.rs                    |   21 -
 .../app/sagas/switch_port_settings_apply.rs   |  586 -------
 .../app/sagas/switch_port_settings_clear.rs   |  546 -------
 .../app/sagas/switch_port_settings_common.rs  |  603 -------
 nexus/src/app/switch_interface.rs             |   56 +-
 nexus/src/app/switch_port.rs                  |  112 +-
 nexus/tests/config.test.toml                  |    5 +-
 nexus/tests/integration_tests/address_lots.rs |   22 -
 .../integration_tests/loopback_address.rs     |  389 -----
 nexus/tests/integration_tests/mod.rs          |    1 -
 .../tests/output/self-stat-schema.json        |    4 +-
 package-manifest.toml                         |  138 +-
 schema/crdb/45.0.0/up01.sql                   |    1 +
 schema/crdb/45.0.0/up02.sql                   |    8 +
 schema/crdb/45.0.0/up03.sql                   |    1 +
 schema/crdb/dbinit.sql                        |   33 +-
 sled-agent/src/bootstrap/early_networking.rs  |    6 +-
 smf/nexus/multi-sled/config-partial.toml      |    1 +
 smf/nexus/single-sled/config-partial.toml     |    1 +
 tools/dendrite_openapi_version                |    4 +-
 tools/dendrite_stub_checksums                 |    6 +-
 tools/virtual_hardware.sh                     |    2 +-
 wicketd/src/preflight_check/uplink.rs         |   20 +-
 52 files changed, 2434 insertions(+), 3028 deletions(-)
 create mode 100644 nexus/src/app/background/networking.rs
 create mode 100644 nexus/src/app/background/sync_switch_configuration.rs
 delete mode 100644 nexus/src/app/sagas/loopback_address_create.rs
 delete mode 100644 nexus/src/app/sagas/loopback_address_delete.rs
 delete mode 100644 nexus/src/app/sagas/switch_port_settings_apply.rs
 delete mode 100644 nexus/src/app/sagas/switch_port_settings_clear.rs
 delete mode 100644 nexus/src/app/sagas/switch_port_settings_common.rs
 delete mode 100644 nexus/tests/integration_tests/loopback_address.rs
 create mode 100644 schema/crdb/45.0.0/up01.sql
 create mode 100644 schema/crdb/45.0.0/up02.sql
 create mode 100644 schema/crdb/45.0.0/up03.sql

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index a2957d35b4..e7990dafe5 100755
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -231,12 +231,10 @@ first = \"$SERVICE_IP_POOL_START\"
 		/^last/c\\
 last = \"$SERVICE_IP_POOL_END\"
 	}
-	/^\\[rack_network_config/,/^$/ {
-		/^infra_ip_first/c\\
+	/^infra_ip_first/c\\
 infra_ip_first = \"$UPLINK_IP\"
-		/^infra_ip_last/c\\
+	/^infra_ip_last/c\\
 infra_ip_last = \"$UPLINK_IP\"
-	}
 	/^\\[\\[rack_network_config.ports/,/^\$/ {
 		/^routes/c\\
 routes = \\[{nexthop = \"$GATEWAY_IP\", destination = \"0.0.0.0/0\"}\\]
@@ -335,6 +333,18 @@ while [[ $(pfexec svcs -z $(zoneadm list -n | grep oxz_ntp) \
 done
 echo "Waited for chrony: ${retry}s"
 
+# Wait for at least one nexus zone to become available
+retry=0
+until zoneadm list | grep nexus; do
+	if [[ $retry -gt 300 ]]; then
+		echo "Failed to start at least one nexus zone after 300 seconds"
+		exit 1
+	fi
+	sleep 1
+	retry=$((retry + 1))
+done
+echo "Waited for nexus: ${retry}s"
+
 export RUST_BACKTRACE=1
 export E2E_TLS_CERT IPPOOL_START IPPOOL_END
 eval "$(./tests/bootstrap)"
diff --git a/clients/mg-admin-client/src/lib.rs b/clients/mg-admin-client/src/lib.rs
index bb1d925c73..b246cddc26 100644
--- a/clients/mg-admin-client/src/lib.rs
+++ b/clients/mg-admin-client/src/lib.rs
@@ -17,11 +17,13 @@ mod inner {
 }
 
 pub use inner::types;
+use inner::types::Prefix4;
 pub use inner::Error;
 
 use inner::Client as InnerClient;
 use omicron_common::api::external::BgpPeerState;
 use slog::Logger;
+use std::hash::Hash;
 use std::net::Ipv6Addr;
 use std::net::SocketAddr;
 use thiserror::Error;
@@ -81,3 +83,18 @@ impl Client {
         Ok(Self { inner, log })
     }
 }
+
+impl Eq for Prefix4 {}
+
+impl PartialEq for Prefix4 {
+    fn eq(&self, other: &Self) -> bool {
+        self.value == other.value && self.length == other.length
+    }
+}
+
+impl Hash for Prefix4 {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.value.hash(state);
+        self.length.hash(state);
+    }
+}
diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out
index ef8cf1631e..0f0aff1789 100644
--- a/dev-tools/omdb/tests/env.out
+++ b/dev-tools/omdb/tests/env.out
@@ -92,6 +92,10 @@ task: "service_zone_nat_tracker"
     ensures service zone nat records are recorded in NAT RPW table
 
 
+task: "switch_port_config_manager"
+    manages switch port settings for rack switches
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT
@@ -182,6 +186,10 @@ task: "service_zone_nat_tracker"
     ensures service zone nat records are recorded in NAT RPW table
 
 
+task: "switch_port_config_manager"
+    manages switch port settings for rack switches
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.
@@ -259,6 +267,10 @@ task: "service_zone_nat_tracker"
     ensures service zone nat records are recorded in NAT RPW table
 
 
+task: "switch_port_config_manager"
+    manages switch port settings for rack switches
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.
diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out
index 2da6e4dceb..dcdd3b3e26 100644
--- a/dev-tools/omdb/tests/successes.out
+++ b/dev-tools/omdb/tests/successes.out
@@ -299,6 +299,10 @@ task: "service_zone_nat_tracker"
     ensures service zone nat records are recorded in NAT RPW table
 
 
+task: "switch_port_config_manager"
+    manages switch port settings for rack switches
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/
@@ -368,7 +372,7 @@ task: "nat_v4_garbage_collector"
   currently executing: no
   last completed activation: iter 2, triggered by an explicit signal
     started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
-warning: unknown background task: "nat_v4_garbage_collector" (don't know how to interpret details: Null)
+    last completion reported error: failed to resolve addresses for Dendrite services: no record found for Query { name: Name("_dendrite._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }
 
 task: "blueprint_loader"
   configured period: every 1m 40s
@@ -389,7 +393,7 @@ task: "bfd_manager"
   currently executing: no
   last completed activation: iter 2, triggered by an explicit signal
     started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
-warning: unknown background task: "bfd_manager" (don't know how to interpret details: Object {})
+    last completion reported error: failed to resolve addresses for Dendrite services: no record found for Query { name: Name("_dendrite._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN }
 
 task: "external_endpoints"
   configured period: every 1m
@@ -440,6 +444,13 @@ task: "service_zone_nat_tracker"
     started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
     last completion reported error: inventory collection is None
 
+task: "switch_port_config_manager"
+  configured period: every 30s
+  currently executing: no
+  last completed activation: iter 2, triggered by an explicit signal
+    started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+warning: unknown background task: "switch_port_config_manager" (don't know how to interpret details: Object {})
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/
diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs
index 0c40326001..8b94d0154b 100644
--- a/nexus-config/src/nexus_config.rs
+++ b/nexus-config/src/nexus_config.rs
@@ -367,6 +367,8 @@ pub struct BackgroundTaskConfig {
     pub sync_service_zone_nat: SyncServiceZoneNatConfig,
     /// configuration for the bfd manager task
     pub bfd_manager: BfdManagerConfig,
+    /// configuration for the switch port settings manager task
+    pub switch_port_settings_manager: SwitchPortSettingsManagerConfig,
     /// configuration for region replacement task
     pub region_replacement: RegionReplacementConfig,
 }
@@ -427,6 +429,15 @@ pub struct SyncServiceZoneNatConfig {
     pub period_secs: Duration,
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct SwitchPortSettingsManagerConfig {
+    /// Interval (in seconds) for periodic activations of this background task.
+    /// This task is also activated on-demand when any of the switch port settings
+    /// api endpoints are called.
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+}
 #[serde_as]
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 pub struct InventoryConfig {
@@ -713,6 +724,7 @@ mod test {
             blueprints.period_secs_load = 10
             blueprints.period_secs_execute = 60
             sync_service_zone_nat.period_secs = 30
+            switch_port_settings_manager.period_secs = 30
             region_replacement.period_secs = 30
             [default_region_allocation_strategy]
             type = "random"
@@ -828,6 +840,10 @@ mod test {
                         sync_service_zone_nat: SyncServiceZoneNatConfig {
                             period_secs: Duration::from_secs(30)
                         },
+                        switch_port_settings_manager:
+                            SwitchPortSettingsManagerConfig {
+                                period_secs: Duration::from_secs(30),
+                            },
                         region_replacement: RegionReplacementConfig {
                             period_secs: Duration::from_secs(30),
                         },
@@ -893,6 +909,7 @@ mod test {
             blueprints.period_secs_load = 10
             blueprints.period_secs_execute = 60
             sync_service_zone_nat.period_secs = 30
+            switch_port_settings_manager.period_secs = 30
             region_replacement.period_secs = 30
             [default_region_allocation_strategy]
             type = "random"
diff --git a/nexus/db-model/src/address_lot.rs b/nexus/db-model/src/address_lot.rs
index 4fef2466e6..9c6065ddc8 100644
--- a/nexus/db-model/src/address_lot.rs
+++ b/nexus/db-model/src/address_lot.rs
@@ -11,8 +11,10 @@ use omicron_common::api::external;
 use serde::{Deserialize, Serialize};
 use uuid::Uuid;
 
+pub const INFRA_LOT: &str = "initial-infra";
+
 impl_enum_type!(
-    #[derive(SqlType, Debug, Clone, Copy)]
+    #[derive(SqlType, Debug, Clone, Copy, QueryId)]
     #[diesel(postgres_type(name = "address_lot_kind", schema = "public"))]
     pub struct AddressLotKindEnum;
 
@@ -24,7 +26,7 @@ impl_enum_type!(
         FromSqlRow,
         PartialEq,
         Serialize,
-        Deserialize
+        Deserialize,
     )]
     #[diesel(sql_type = AddressLotKindEnum)]
     pub enum AddressLotKind;
diff --git a/nexus/db-model/src/bootstore.rs b/nexus/db-model/src/bootstore.rs
index 38afd37f54..1d7ac82068 100644
--- a/nexus/db-model/src/bootstore.rs
+++ b/nexus/db-model/src/bootstore.rs
@@ -1,4 +1,5 @@
-use crate::schema::bootstore_keys;
+use crate::schema::{bootstore_config, bootstore_keys};
+use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 
 pub const NETWORK_KEY: &str = "network_key";
@@ -11,3 +12,18 @@ pub struct BootstoreKeys {
     pub key: String,
     pub generation: i64,
 }
+
+/// BootstoreConfig is a key-value store for bootstrapping data.
+/// We serialize the data as json because it is inherently polymorphic and it
+/// is not intended to be queried directly.
+#[derive(
+    Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize,
+)]
+#[diesel(table_name = bootstore_config)]
+pub struct BootstoreConfig {
+    pub key: String,
+    pub generation: i64,
+    pub data: serde_json::Value,
+    pub time_created: DateTime<Utc>,
+    pub time_deleted: Option<DateTime<Utc>>,
+}
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index 2c53dc26a7..771d6836dc 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(44, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(45, 0, 0);
 
 table! {
     disk (id) {
@@ -1529,6 +1529,16 @@ table! {
     }
 }
 
+table! {
+    bootstore_config (key, generation) {
+        key -> Text,
+        generation -> Int8,
+        data -> Jsonb,
+        time_created -> Timestamptz,
+        time_deleted -> Nullable<Timestamptz>,
+    }
+}
+
 table! {
     bfd_session (remote, switch) {
         id -> Uuid,
diff --git a/nexus/db-model/src/unsigned.rs b/nexus/db-model/src/unsigned.rs
index 920cad1cff..5c76168ceb 100644
--- a/nexus/db-model/src/unsigned.rs
+++ b/nexus/db-model/src/unsigned.rs
@@ -130,6 +130,7 @@ where
     FromSqlRow,
     Serialize,
     Deserialize,
+    QueryId,
 )]
 #[diesel(sql_type = sql_types::BigInt)]
 #[repr(transparent)]
diff --git a/nexus/db-queries/src/db/datastore/address_lot.rs b/nexus/db-queries/src/db/datastore/address_lot.rs
index 5c2ffbf1d0..9c75c6fd1b 100644
--- a/nexus/db-queries/src/db/datastore/address_lot.rs
+++ b/nexus/db-queries/src/db/datastore/address_lot.rs
@@ -53,14 +53,35 @@ impl DataStore {
             .transaction(&conn, |conn| async move {
                 let lot = AddressLot::new(&params.identity, params.kind.into());
 
-                let db_lot: AddressLot =
-                    diesel::insert_into(lot_dsl::address_lot)
-                        .values(lot)
-                        .returning(AddressLot::as_returning())
-                        .get_result_async(&conn)
-                        .await?;
+                // @internet-diglett says:
+                // I hate this. I know how to replace this transaction with
+                // CTEs but for the life of me I can't get it to work in
+                // diesel. I gave up and just extended the logic inside
+                // of the transaction instead chasing diesel trait bound errors.
+                let found_lot: Option<AddressLot> = lot_dsl::address_lot
+                    .filter(
+                        lot_dsl::name
+                            .eq(Name::from(params.identity.name.clone())),
+                    )
+                    .filter(lot_dsl::time_deleted.is_null())
+                    .select(AddressLot::as_select())
+                    .limit(1)
+                    .first_async(&conn)
+                    .await
+                    .ok();
+
+                let db_lot = match found_lot {
+                    Some(v) => v,
+                    None => {
+                        diesel::insert_into(lot_dsl::address_lot)
+                            .values(lot)
+                            .returning(AddressLot::as_returning())
+                            .get_result_async(&conn)
+                            .await?
+                    }
+                };
 
-                let blocks: Vec<AddressLotBlock> = params
+                let desired_blocks: Vec<AddressLotBlock> = params
                     .blocks
                     .iter()
                     .map(|b| {
@@ -72,14 +93,51 @@ impl DataStore {
                     })
                     .collect();
 
-                let db_blocks =
-                    diesel::insert_into(block_dsl::address_lot_block)
-                        .values(blocks)
-                        .returning(AddressLotBlock::as_returning())
+                let found_blocks: Vec<AddressLotBlock> =
+                    block_dsl::address_lot_block
+                        .filter(block_dsl::address_lot_id.eq(db_lot.id()))
+                        .filter(
+                            block_dsl::first_address.eq_any(
+                                desired_blocks
+                                    .iter()
+                                    .map(|b| b.first_address)
+                                    .collect::<Vec<_>>(),
+                            ),
+                        )
+                        .filter(
+                            block_dsl::last_address.eq_any(
+                                desired_blocks
+                                    .iter()
+                                    .map(|b| b.last_address)
+                                    .collect::<Vec<_>>(),
+                            ),
+                        )
                         .get_results_async(&conn)
                         .await?;
 
-                Ok(AddressLotCreateResult { lot: db_lot, blocks: db_blocks })
+                let mut blocks = vec![];
+
+                // If the block is found in the database, use the found block.
+                // If the block is not found in the database, insert it.
+                for desired_block in desired_blocks {
+                    let block = match found_blocks.iter().find(|db_b| {
+                        db_b.first_address == desired_block.first_address
+                            && db_b.last_address == desired_block.last_address
+                    }) {
+                        Some(block) => block.clone(),
+                        None => {
+                            diesel::insert_into(block_dsl::address_lot_block)
+                                .values(desired_block)
+                                .returning(AddressLotBlock::as_returning())
+                                .get_results_async(&conn)
+                                .await?[0]
+                                .clone()
+                        }
+                    };
+                    blocks.push(block);
+                }
+
+                Ok(AddressLotCreateResult { lot: db_lot, blocks })
             })
             .await
             .map_err(|e| {
@@ -87,7 +145,7 @@ impl DataStore {
                     e,
                     ErrorHandler::Conflict(
                         ResourceType::AddressLot,
-                        &params.identity.name.as_str(),
+                        params.identity.name.as_str(),
                     ),
                 )
             })
@@ -225,6 +283,35 @@ impl DataStore {
 
         Ok(address_lot_id)
     }
+
+    // Take the name of an address lot and look up its blocks
+    pub async fn address_lot_blocks_by_name(
+        &self,
+        opctx: &OpContext,
+        name: String,
+    ) -> LookupResult<Vec<AddressLotBlock>> {
+        let conn = self.pool_connection_authorized(opctx).await?;
+
+        use db::schema::address_lot::dsl as lot_dsl;
+        use db::schema::address_lot_block::dsl as block_dsl;
+
+        let address_lot_id = lot_dsl::address_lot
+            .filter(lot_dsl::name.eq(name))
+            .select(lot_dsl::id)
+            .limit(1)
+            .first_async::<Uuid>(&*conn)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        let blocks = block_dsl::address_lot_block
+            .filter(block_dsl::address_lot_id.eq(address_lot_id))
+            .select(AddressLotBlock::as_select())
+            .load_async::<AddressLotBlock>(&*conn)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        Ok(blocks)
+    }
 }
 
 #[derive(Debug)]
diff --git a/nexus/db-queries/src/db/datastore/bgp.rs b/nexus/db-queries/src/db/datastore/bgp.rs
index e5ac35d19a..eb8285e4dc 100644
--- a/nexus/db-queries/src/db/datastore/bgp.rs
+++ b/nexus/db-queries/src/db/datastore/bgp.rs
@@ -14,6 +14,7 @@ use crate::transaction_retry::OptionalError;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use chrono::Utc;
 use diesel::{ExpressionMethods, QueryDsl, SelectableHelper};
+use ipnetwork::IpNetwork;
 use nexus_types::external_api::params;
 use nexus_types::identity::Resource;
 use omicron_common::api::external::http_pagination::PaginatedBy;
@@ -34,10 +35,13 @@ impl DataStore {
         use db::schema::{
             bgp_announce_set, bgp_announce_set::dsl as announce_set_dsl,
         };
+        use diesel::sql_types;
+        use diesel::IntoSql;
+
         let conn = self.pool_connection_authorized(opctx).await?;
         self.transaction_retry_wrapper("bgp_config_set")
             .transaction(&conn, |conn| async move {
-                let id: Uuid = match &config.bgp_announce_set_id {
+                let announce_set_id: Uuid = match &config.bgp_announce_set_id {
                     NameOrId::Name(name) => {
                         announce_set_dsl::bgp_announce_set
                             .filter(bgp_announce_set::time_deleted.is_null())
@@ -50,14 +54,58 @@ impl DataStore {
                     NameOrId::Id(id) => *id,
                 };
 
-                let config = BgpConfig::from_config_create(config, id);
-
-                let result = diesel::insert_into(dsl::bgp_config)
-                    .values(config.clone())
-                    .returning(BgpConfig::as_returning())
-                    .get_result_async(&conn)
+                let config =
+                    BgpConfig::from_config_create(config, announce_set_id);
+
+                let matching_entry_subquery = dsl::bgp_config
+                    .filter(dsl::name.eq(Name::from(config.name().clone())))
+                    .filter(dsl::time_deleted.is_null())
+                    .select(dsl::name);
+
+                // SELECT exactly the values we're trying to INSERT, but only
+                // if it does not already exist.
+                let new_entry_subquery = diesel::dsl::select((
+                    config.id().into_sql::<sql_types::Uuid>(),
+                    config.name().to_string().into_sql::<sql_types::Text>(),
+                    config
+                        .description()
+                        .to_string()
+                        .into_sql::<sql_types::Text>(),
+                    config.asn.into_sql::<sql_types::BigInt>(),
+                    config.bgp_announce_set_id.into_sql::<sql_types::Uuid>(),
+                    config
+                        .vrf
+                        .clone()
+                        .into_sql::<sql_types::Nullable<sql_types::Text>>(),
+                    Utc::now().into_sql::<sql_types::Timestamptz>(),
+                    Utc::now().into_sql::<sql_types::Timestamptz>(),
+                ))
+                .filter(diesel::dsl::not(diesel::dsl::exists(
+                    matching_entry_subquery,
+                )));
+
+                diesel::insert_into(dsl::bgp_config)
+                    .values(new_entry_subquery)
+                    .into_columns((
+                        dsl::id,
+                        dsl::name,
+                        dsl::description,
+                        dsl::asn,
+                        dsl::bgp_announce_set_id,
+                        dsl::vrf,
+                        dsl::time_created,
+                        dsl::time_modified,
+                    ))
+                    .execute_async(&conn)
                     .await?;
-                Ok(result)
+
+                dsl::bgp_config
+                    .filter(dsl::name.eq(Name::from(config.name().clone())))
+                    .filter(dsl::time_deleted.is_null())
+                    .select(BgpConfig::as_select())
+                    .limit(1)
+                    .first_async(&conn)
+                    .await
             })
             .await
             .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
@@ -277,12 +325,29 @@ impl DataStore {
             .transaction(&conn, |conn| async move {
                 let bas: BgpAnnounceSet = announce.clone().into();
 
-                let db_as: BgpAnnounceSet =
-                    diesel::insert_into(announce_set_dsl::bgp_announce_set)
-                        .values(bas.clone())
-                        .returning(BgpAnnounceSet::as_returning())
-                        .get_result_async::<BgpAnnounceSet>(&conn)
-                        .await?;
+                let found_as: Option<BgpAnnounceSet> =
+                    announce_set_dsl::bgp_announce_set
+                        .filter(
+                            announce_set_dsl::name
+                                .eq(Name::from(bas.name().clone())),
+                        )
+                        .filter(announce_set_dsl::time_deleted.is_null())
+                        .select(BgpAnnounceSet::as_select())
+                        .limit(1)
+                        .first_async(&conn)
+                        .await
+                        .ok();
+
+                let db_as = match found_as {
+                    Some(v) => v,
+                    None => {
+                        diesel::insert_into(announce_set_dsl::bgp_announce_set)
+                            .values(bas.clone())
+                            .returning(BgpAnnounceSet::as_returning())
+                            .get_result_async::<BgpAnnounceSet>(&conn)
+                            .await?
+                    }
+                };
 
                 let mut db_annoucements = Vec::new();
                 for a in &announce.announcement {
@@ -291,13 +356,36 @@ impl DataStore {
                         address_lot_block_id: bas.identity.id,
                         network: a.network.into(),
                     };
-                    let an = diesel::insert_into(
-                        bgp_announcement_dsl::bgp_announcement,
-                    )
-                    .values(an.clone())
-                    .returning(BgpAnnouncement::as_returning())
-                    .get_result_async::<BgpAnnouncement>(&conn)
-                    .await?;
+
+                    let found_an: Option<BgpAnnouncement> =
+                        bgp_announcement_dsl::bgp_announcement
+                            .filter(
+                                bgp_announcement_dsl::announce_set_id
+                                    .eq(db_as.id()),
+                            )
+                            .filter(
+                                bgp_announcement_dsl::network
+                                    .eq(IpNetwork::from(a.network)),
+                            )
+                            .select(BgpAnnouncement::as_select())
+                            .limit(1)
+                            .first_async(&conn)
+                            .await
+                            .ok();
+
+                    let an = match found_an {
+                        Some(v) => v,
+                        None => {
+                            diesel::insert_into(
+                                bgp_announcement_dsl::bgp_announcement,
+                            )
+                            .values(an.clone())
+                            .returning(BgpAnnouncement::as_returning())
+                            .get_result_async::<BgpAnnouncement>(&conn)
+                            .await?
+                        }
+                    };
+
                     db_annoucements.push(an);
                 }
 
diff --git a/nexus/db-queries/src/db/datastore/bootstore.rs b/nexus/db-queries/src/db/datastore/bootstore.rs
index 44f7a2036e..9384a1e13f 100644
--- a/nexus/db-queries/src/db/datastore/bootstore.rs
+++ b/nexus/db-queries/src/db/datastore/bootstore.rs
@@ -3,10 +3,11 @@ use crate::context::OpContext;
 use crate::db;
 use crate::db::error::{public_error_from_diesel, ErrorHandler};
 use async_bb8_diesel::AsyncRunQueryDsl;
+use diesel::prelude::*;
 use diesel::ExpressionMethods;
 use diesel::SelectableHelper;
-use nexus_db_model::BootstoreKeys;
-use omicron_common::api::external::LookupResult;
+use nexus_db_model::{BootstoreConfig, BootstoreKeys};
+use omicron_common::api::external::{CreateResult, LookupResult};
 
 impl DataStore {
     pub async fn bump_bootstore_generation(
@@ -34,4 +35,49 @@ impl DataStore {
 
         Ok(bks.generation)
     }
+
+    pub async fn ensure_bootstore_config(
+        &self,
+        opctx: &OpContext,
+        config: BootstoreConfig,
+    ) -> CreateResult<()> {
+        use db::schema::bootstore_config::dsl;
+
+        let conn = self.pool_connection_authorized(opctx).await?;
+
+        diesel::insert_into(dsl::bootstore_config)
+            .values(config)
+            .on_conflict((dsl::key, dsl::generation))
+            .do_nothing()
+            .execute_async(&*conn)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        Ok(())
+    }
+
+    pub async fn get_latest_bootstore_config(
+        &self,
+        opctx: &OpContext,
+        key: String,
+    ) -> LookupResult<Option<BootstoreConfig>> {
+        use db::schema::bootstore_config::dsl;
+
+        let conn = self.pool_connection_authorized(opctx).await?;
+
+        let result = dsl::bootstore_config
+            .filter(dsl::key.eq(key))
+            .select(BootstoreConfig::as_select())
+            .order(dsl::generation.desc())
+            .limit(1)
+            .load_async(&*conn)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        if let Some(nat_entry) = result.first() {
+            Ok(Some(nat_entry.clone()))
+        } else {
+            Ok(None)
+        }
+    }
 }
diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs
index e290343978..c86c002c33 100644
--- a/nexus/db-queries/src/db/datastore/rack.rs
+++ b/nexus/db-queries/src/db/datastore/rack.rs
@@ -185,6 +185,21 @@ impl DataStore {
             .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
     }
 
+    pub async fn rack_list_initialized(
+        &self,
+        opctx: &OpContext,
+        pagparams: &DataPageParams<'_, Uuid>,
+    ) -> ListResultVec<Rack> {
+        opctx.authorize(authz::Action::Read, &authz::FLEET).await?;
+        use db::schema::rack::dsl;
+        paginated(dsl::rack, dsl::id, pagparams)
+            .select(Rack::as_select())
+            .filter(dsl::initialized.eq(true))
+            .load_async(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))
+    }
+
     /// Stores a new rack in the database.
     ///
     /// This function is a no-op if the rack already exists.
diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml
index ac9d894050..2e946a9c38 100644
--- a/nexus/examples/config.toml
+++ b/nexus/examples/config.toml
@@ -5,7 +5,7 @@
 [console]
 # Directory for static assets. Absolute path or relative to CWD.
 static_dir = "out/console-assets"
-session_idle_timeout_minutes = 480 # 6 hours
+session_idle_timeout_minutes = 480      # 6 hours
 session_absolute_timeout_minutes = 1440 # 24 hours
 
 # List of authentication schemes to support.
@@ -35,7 +35,7 @@ rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"
 
 # Nexus may need to resolve external hosts (e.g. to grab IdP metadata).
 # These are the DNS servers it should use.
-external_dns_servers = [ "1.1.1.1", "9.9.9.9" ]
+external_dns_servers = ["1.1.1.1", "9.9.9.9"]
 
 [deployment.dropshot_external]
 # IP Address and TCP port on which to listen for the external API
@@ -109,6 +109,7 @@ phantom_disks.period_secs = 30
 blueprints.period_secs_load = 10
 blueprints.period_secs_execute = 60
 sync_service_zone_nat.period_secs = 30
+switch_port_settings_manager.period_secs = 30
 region_replacement.period_secs = 30
 
 [default_region_allocation_strategy]
diff --git a/nexus/src/app/background/bfd.rs b/nexus/src/app/background/bfd.rs
index a586db54e4..ca23fb7f65 100644
--- a/nexus/src/app/background/bfd.rs
+++ b/nexus/src/app/background/bfd.rs
@@ -5,16 +5,21 @@
 //! Background task for managing switch bidirectional forwarding detection
 //! (BFD) sessions.
 
+use crate::app::{
+    background::networking::build_mgd_clients, map_switch_zone_addrs,
+};
+
 use super::common::BackgroundTask;
 use futures::future::BoxFuture;
 use futures::FutureExt;
+use internal_dns::{resolver::Resolver, ServiceName};
 use mg_admin_client::types::{BfdPeerConfig, SessionMode};
 use nexus_db_model::{BfdMode, BfdSession};
 use nexus_db_queries::{context::OpContext, db::DataStore};
 use omicron_common::api::external::{DataPageParams, SwitchLocation};
 use serde_json::json;
 use std::{
-    collections::{HashMap, HashSet},
+    collections::HashSet,
     hash::Hash,
     net::{IpAddr, Ipv4Addr},
     sync::Arc,
@@ -22,15 +27,12 @@ use std::{
 
 pub struct BfdManager {
     datastore: Arc<DataStore>,
-    mgd_clients: HashMap<SwitchLocation, Arc<mg_admin_client::Client>>,
+    resolver: Resolver,
 }
 
 impl BfdManager {
-    pub fn new(
-        datastore: Arc<DataStore>,
-        mgd_clients: HashMap<SwitchLocation, Arc<mg_admin_client::Client>>,
-    ) -> Self {
-        Self { datastore, mgd_clients }
+    pub fn new(datastore: Arc<DataStore>, resolver: Resolver) -> Self {
+        Self { datastore, resolver }
     }
 }
 
@@ -114,7 +116,30 @@ impl BackgroundTask for BfdManager {
 
             let mut current: HashSet<BfdSessionKey> = HashSet::new();
 
-            for (location, c) in &self.mgd_clients {
+            let switch_zone_addresses = match self
+                .resolver
+                .lookup_all_ipv6(ServiceName::Dendrite)
+                .await
+            {
+                Ok(addrs) => addrs,
+                Err(e) => {
+                    error!(log, "failed to resolve addresses for Dendrite services"; "error" => %e);
+                    return json!({
+                        "error":
+                            format!(
+                                "failed to resolve addresses for Dendrite services: {:#}",
+                                e
+                            )
+                        });
+                },
+            };
+
+            let mappings =
+                map_switch_zone_addrs(log, switch_zone_addresses).await;
+
+            let mgd_clients = build_mgd_clients(mappings, log);
+
+            for (location, c) in &mgd_clients {
                 let client_current = match c.inner.get_bfd_peers().await {
                     Ok(x) => x.into_inner(),
                     Err(e) => {
@@ -159,7 +184,7 @@ impl BackgroundTask for BfdManager {
             }
 
             for x in &to_add {
-                let mg = match self.mgd_clients.get(&x.switch) {
+                let mg = match mgd_clients.get(&x.switch) {
                     Some(mg) => mg,
                     None => {
                         error!(&log, "failed to get mg client";
@@ -190,7 +215,7 @@ impl BackgroundTask for BfdManager {
             }
 
             for x in &to_del {
-                let mg = match self.mgd_clients.get(&x.switch) {
+                let mg = match mgd_clients.get(&x.switch) {
                     Some(mg) => mg,
                     None => {
                         error!(&log, "failed to get mg client";
diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs
index a213f7da72..e3f2154046 100644
--- a/nexus/src/app/background/init.rs
+++ b/nexus/src/app/background/init.rs
@@ -17,15 +17,14 @@ use super::nat_cleanup;
 use super::phantom_disks;
 use super::region_replacement;
 use super::sync_service_zone_nat::ServiceZoneNatTracker;
+use super::sync_switch_configuration::SwitchPortSettingsManager;
 use crate::app::sagas::SagaRequest;
 use nexus_config::BackgroundTaskConfig;
 use nexus_config::DnsTasksConfig;
 use nexus_db_model::DnsGroup;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db::DataStore;
-use omicron_common::api::internal::shared::SwitchLocation;
 use std::collections::BTreeMap;
-use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::mpsc::Sender;
 use uuid::Uuid;
@@ -76,6 +75,9 @@ pub struct BackgroundTasks {
     /// task handle for the service zone nat tracker
     pub task_service_zone_nat_tracker: common::TaskHandle,
 
+    /// task handle for the switch port settings manager
+    pub task_switch_port_settings_manager: common::TaskHandle,
+
     /// task handle for the task that detects if regions need replacement and
     /// begins the process
     pub task_region_replacement: common::TaskHandle,
@@ -88,8 +90,6 @@ impl BackgroundTasks {
         opctx: &OpContext,
         datastore: Arc<DataStore>,
         config: &BackgroundTaskConfig,
-        dpd_clients: &HashMap<SwitchLocation, Arc<dpd_client::Client>>,
-        mgd_clients: &HashMap<SwitchLocation, Arc<mg_admin_client::Client>>,
         nexus_id: Uuid,
         resolver: internal_dns::resolver::Resolver,
         saga_request: Sender<SagaRequest>,
@@ -134,8 +134,6 @@ impl BackgroundTasks {
             (task, watcher_channel)
         };
 
-        let dpd_clients: Vec<_> = dpd_clients.values().cloned().collect();
-
         let nat_cleanup = {
             driver.register(
                 "nat_v4_garbage_collector".to_string(),
@@ -146,7 +144,7 @@ impl BackgroundTasks {
                 config.nat_cleanup.period_secs,
                 Box::new(nat_cleanup::Ipv4NatGarbageCollector::new(
                     datastore.clone(),
-                    dpd_clients.clone(),
+                    resolver.clone()
                 )),
                 opctx.child(BTreeMap::new()),
                 vec![],
@@ -163,7 +161,7 @@ impl BackgroundTasks {
                 config.bfd_manager.period_secs,
                 Box::new(bfd::BfdManager::new(
                     datastore.clone(),
-                    mgd_clients.clone(),
+                    resolver.clone(),
                 )),
                 opctx.child(BTreeMap::new()),
                 vec![],
@@ -227,7 +225,7 @@ impl BackgroundTasks {
         let task_inventory_collection = {
             let collector = inventory_collection::InventoryCollector::new(
                 datastore.clone(),
-                resolver,
+                resolver.clone(),
                 &nexus_id.to_string(),
                 config.inventory.nkeep,
                 config.inventory.disable,
@@ -256,7 +254,21 @@ impl BackgroundTasks {
                 config.sync_service_zone_nat.period_secs,
                 Box::new(ServiceZoneNatTracker::new(
                     datastore.clone(),
-                    dpd_clients.clone(),
+                    resolver.clone(),
+                )),
+                opctx.child(BTreeMap::new()),
+                vec![],
+            )
+        };
+
+        let task_switch_port_settings_manager = {
+            driver.register(
+                "switch_port_config_manager".to_string(),
+                String::from("manages switch port settings for rack switches"),
+                config.switch_port_settings_manager.period_secs,
+                Box::new(SwitchPortSettingsManager::new(
+                    datastore.clone(),
+                    resolver.clone(),
                 )),
                 opctx.child(BTreeMap::new()),
                 vec![],
@@ -298,6 +310,7 @@ impl BackgroundTasks {
             task_blueprint_loader,
             task_blueprint_executor,
             task_service_zone_nat_tracker,
+            task_switch_port_settings_manager,
             task_region_replacement,
         }
     }
diff --git a/nexus/src/app/background/mod.rs b/nexus/src/app/background/mod.rs
index 0065a41a9e..9867f1dc6d 100644
--- a/nexus/src/app/background/mod.rs
+++ b/nexus/src/app/background/mod.rs
@@ -15,9 +15,11 @@ mod external_endpoints;
 mod init;
 mod inventory_collection;
 mod nat_cleanup;
+mod networking;
 mod phantom_disks;
 mod region_replacement;
 mod status;
 mod sync_service_zone_nat;
+mod sync_switch_configuration;
 
 pub use init::BackgroundTasks;
diff --git a/nexus/src/app/background/nat_cleanup.rs b/nexus/src/app/background/nat_cleanup.rs
index 5014dc0553..16b1b7e357 100644
--- a/nexus/src/app/background/nat_cleanup.rs
+++ b/nexus/src/app/background/nat_cleanup.rs
@@ -6,10 +6,15 @@
 //! Responsible for cleaning up soft deleted entries once they
 //! have been propagated to running dpd instances.
 
+use crate::app::map_switch_zone_addrs;
+
 use super::common::BackgroundTask;
+use super::networking::build_dpd_clients;
 use chrono::{Duration, Utc};
 use futures::future::BoxFuture;
 use futures::FutureExt;
+use internal_dns::resolver::Resolver;
+use internal_dns::ServiceName;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db::DataStore;
 use serde_json::json;
@@ -19,15 +24,15 @@ use std::sync::Arc;
 /// from ipv4_nat_entry table
 pub struct Ipv4NatGarbageCollector {
     datastore: Arc<DataStore>,
-    dpd_clients: Vec<Arc<dpd_client::Client>>,
+    resolver: Resolver,
 }
 
 impl Ipv4NatGarbageCollector {
     pub fn new(
         datastore: Arc<DataStore>,
-        dpd_clients: Vec<Arc<dpd_client::Client>>,
+        resolver: Resolver,
     ) -> Ipv4NatGarbageCollector {
-        Ipv4NatGarbageCollector { datastore, dpd_clients }
+        Ipv4NatGarbageCollector { datastore, resolver }
     }
 }
 
@@ -60,7 +65,30 @@ impl BackgroundTask for Ipv4NatGarbageCollector {
                 }
             };
 
-            for client in &self.dpd_clients {
+            let switch_zone_addresses = match self
+                .resolver
+                .lookup_all_ipv6(ServiceName::Dendrite)
+                .await
+            {
+                Ok(addrs) => addrs,
+                Err(e) => {
+                    error!(log, "failed to resolve addresses for Dendrite services"; "error" => %e);
+                    return json!({
+                        "error":
+                            format!(
+                                "failed to resolve addresses for Dendrite services: {:#}",
+                                e
+                            )
+                        });
+                },
+            };
+
+            let mappings =
+                map_switch_zone_addrs(log, switch_zone_addresses).await;
+
+            let dpd_clients = build_dpd_clients(&mappings, log);
+
+            for (_location, client) in dpd_clients {
                 let response = client.ipv4_nat_generation().await;
                 match response {
                     Ok(gen) => min_gen = std::cmp::min(min_gen, *gen),
diff --git a/nexus/src/app/background/networking.rs b/nexus/src/app/background/networking.rs
new file mode 100644
index 0000000000..95e0cd32b3
--- /dev/null
+++ b/nexus/src/app/background/networking.rs
@@ -0,0 +1,113 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use db::datastore::SwitchPortSettingsCombinedResult;
+use dpd_client::types::{
+    LinkCreate, LinkId, LinkSettings, PortFec, PortSettings, PortSpeed,
+};
+use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed};
+use nexus_db_queries::db;
+use omicron_common::address::DENDRITE_PORT;
+use omicron_common::{address::MGD_PORT, api::external::SwitchLocation};
+use std::{collections::HashMap, net::SocketAddrV6};
+
+pub(crate) fn build_mgd_clients(
+    mappings: HashMap<SwitchLocation, std::net::Ipv6Addr>,
+    log: &slog::Logger,
+) -> HashMap<SwitchLocation, mg_admin_client::Client> {
+    let mut clients: Vec<(SwitchLocation, mg_admin_client::Client)> = vec![];
+    for (location, addr) in &mappings {
+        let port = MGD_PORT;
+        let socketaddr =
+            std::net::SocketAddr::V6(SocketAddrV6::new(*addr, port, 0, 0));
+        let client =
+            match mg_admin_client::Client::new(&log.clone(), socketaddr) {
+                Ok(client) => client,
+                Err(e) => {
+                    error!(
+                        log,
+                        "error building mgd client";
+                        "location" => %location,
+                        "addr" => %addr,
+                        "error" => %e,
+                    );
+                    continue;
+                }
+            };
+        clients.push((*location, client));
+    }
+    clients.into_iter().collect::<HashMap<_, _>>()
+}
+
+pub(crate) fn build_dpd_clients(
+    mappings: &HashMap<SwitchLocation, std::net::Ipv6Addr>,
+    log: &slog::Logger,
+) -> HashMap<SwitchLocation, dpd_client::Client> {
+    let dpd_clients: HashMap<SwitchLocation, dpd_client::Client> = mappings
+        .iter()
+        .map(|(location, addr)| {
+            let port = DENDRITE_PORT;
+
+            let client_state = dpd_client::ClientState {
+                tag: String::from("nexus"),
+                log: log.new(o!(
+                    "component" => "DpdClient"
+                )),
+            };
+
+            let dpd_client = dpd_client::Client::new(
+                &format!("http://[{addr}]:{port}"),
+                client_state,
+            );
+            (*location, dpd_client)
+        })
+        .collect();
+    dpd_clients
+}
+
+pub(crate) fn api_to_dpd_port_settings(
+    settings: &SwitchPortSettingsCombinedResult,
+) -> Result<PortSettings, String> {
+    let mut dpd_port_settings = PortSettings { links: HashMap::new() };
+
+    //TODO breakouts
+    let link_id = LinkId(0);
+
+    for l in settings.links.iter() {
+        dpd_port_settings.links.insert(
+            link_id.to_string(),
+            LinkSettings {
+                params: LinkCreate {
+                    autoneg: l.autoneg,
+                    lane: Some(LinkId(0)),
+                    kr: false,
+                    fec: match l.fec {
+                        SwitchLinkFec::Firecode => PortFec::Firecode,
+                        SwitchLinkFec::Rs => PortFec::Rs,
+                        SwitchLinkFec::None => PortFec::None,
+                    },
+                    speed: match l.speed {
+                        SwitchLinkSpeed::Speed0G => PortSpeed::Speed0G,
+                        SwitchLinkSpeed::Speed1G => PortSpeed::Speed1G,
+                        SwitchLinkSpeed::Speed10G => PortSpeed::Speed10G,
+                        SwitchLinkSpeed::Speed25G => PortSpeed::Speed25G,
+                        SwitchLinkSpeed::Speed40G => PortSpeed::Speed40G,
+                        SwitchLinkSpeed::Speed50G => PortSpeed::Speed50G,
+                        SwitchLinkSpeed::Speed100G => PortSpeed::Speed100G,
+                        SwitchLinkSpeed::Speed200G => PortSpeed::Speed200G,
+                        SwitchLinkSpeed::Speed400G => PortSpeed::Speed400G,
+                    },
+                },
+                //TODO won't work for breakouts
+                addrs: settings
+                    .addresses
+                    .iter()
+                    .map(|a| a.address.ip())
+                    .collect(),
+            },
+        );
+    }
+
+    Ok(dpd_port_settings)
+}
diff --git a/nexus/src/app/background/sync_service_zone_nat.rs b/nexus/src/app/background/sync_service_zone_nat.rs
index a3d52f4469..e23621ed23 100644
--- a/nexus/src/app/background/sync_service_zone_nat.rs
+++ b/nexus/src/app/background/sync_service_zone_nat.rs
@@ -5,10 +5,15 @@
 //! Background task for detecting changes to service zone locations and
 //! updating the NAT rpw table accordingly
 
+use crate::app::map_switch_zone_addrs;
+
 use super::common::BackgroundTask;
+use super::networking::build_dpd_clients;
 use anyhow::Context;
 use futures::future::BoxFuture;
 use futures::FutureExt;
+use internal_dns::resolver::Resolver;
+use internal_dns::ServiceName;
 use nexus_db_model::Ipv4NatValues;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db::lookup::LookupPath;
@@ -36,15 +41,12 @@ const MIN_EXTERNAL_DNS_COUNT: usize = 1;
 /// persisted in the NAT RPW table
 pub struct ServiceZoneNatTracker {
     datastore: Arc<DataStore>,
-    dpd_clients: Vec<Arc<dpd_client::Client>>,
+    resolver: Resolver,
 }
 
 impl ServiceZoneNatTracker {
-    pub fn new(
-        datastore: Arc<DataStore>,
-        dpd_clients: Vec<Arc<dpd_client::Client>>,
-    ) -> Self {
-        Self { datastore, dpd_clients }
+    pub fn new(datastore: Arc<DataStore>, resolver: Resolver) -> Self {
+        Self { datastore, resolver }
     }
 }
 
@@ -332,7 +334,31 @@ impl BackgroundTask for ServiceZoneNatTracker {
 
             // notify dpd if we've added any new records
             if result > 0 {
-                for client in &self.dpd_clients {
+
+                let switch_zone_addresses = match self
+                    .resolver
+                    .lookup_all_ipv6(ServiceName::Dendrite)
+                    .await
+                {
+                    Ok(addrs) => addrs,
+                    Err(e) => {
+                        error!(log, "failed to resolve addresses for Dendrite services"; "error" => %e);
+                        return json!({
+                            "error":
+                                format!(
+                                    "failed to resolve addresses for Dendrite services: {:#}",
+                                    e
+                                )
+                            });
+                    },
+                };
+
+                let mappings =
+                    map_switch_zone_addrs(log, switch_zone_addresses).await;
+
+                let dpd_clients = build_dpd_clients(&mappings, log);
+
+                for (_location, client) in dpd_clients {
                     if let Err(e) = client.ipv4_nat_trigger_update().await {
                         error!(
                             &log,
diff --git a/nexus/src/app/background/sync_switch_configuration.rs b/nexus/src/app/background/sync_switch_configuration.rs
new file mode 100644
index 0000000000..4bc75bad10
--- /dev/null
+++ b/nexus/src/app/background/sync_switch_configuration.rs
@@ -0,0 +1,1418 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Background task for propagating user provided switch configurations
+//! to relevant management daemons (dendrite, mgd, sled-agent, etc.)
+
+use crate::app::{
+    background::networking::{
+        api_to_dpd_port_settings, build_dpd_clients, build_mgd_clients,
+    },
+    map_switch_zone_addrs,
+};
+
+use internal_dns::resolver::Resolver;
+use internal_dns::ServiceName;
+use ipnetwork::IpNetwork;
+use nexus_db_model::{
+    AddressLotBlock, BgpConfig, BootstoreConfig, LoopbackAddress,
+    SwitchLinkFec, SwitchLinkSpeed, SwitchPortBgpPeerConfig, INFRA_LOT,
+    NETWORK_KEY,
+};
+use uuid::Uuid;
+
+use super::common::BackgroundTask;
+use dpd_client::types::PortId;
+use futures::future::BoxFuture;
+use futures::FutureExt;
+use mg_admin_client::types::{
+    AddStaticRoute4Request, ApplyRequest, BgpPeerConfig,
+    DeleteStaticRoute4Request, Prefix4, StaticRoute4, StaticRoute4List,
+};
+use nexus_db_queries::{
+    context::OpContext,
+    db::{datastore::SwitchPortSettingsCombinedResult, DataStore},
+};
+use nexus_types::{external_api::params, identity::Resource};
+use omicron_common::OMICRON_DPD_TAG;
+use omicron_common::{
+    address::{get_sled_address, Ipv6Subnet},
+    api::external::{DataPageParams, SwitchLocation},
+};
+use serde_json::json;
+use sled_agent_client::types::{
+    BgpConfig as SledBgpConfig, BgpPeerConfig as SledBgpPeerConfig,
+    EarlyNetworkConfig, EarlyNetworkConfigBody, HostPortConfig, Ipv4Network,
+    PortConfigV1, RackNetworkConfigV1, RouteConfig as SledRouteConfig,
+};
+use std::{
+    collections::{hash_map::Entry, HashMap, HashSet},
+    net::{IpAddr, Ipv4Addr},
+    str::FromStr,
+    sync::Arc,
+};
+
+const DPD_TAG: Option<&'static str> = Some(OMICRON_DPD_TAG);
+
+// This is more of an implementation detail of the BGP implementation. It
+// defines the maximum time the peering engine will wait for external messages
+// before breaking to check for shutdown conditions.
+const BGP_SESSION_RESOLUTION: u64 = 100;
+
+pub struct SwitchPortSettingsManager {
+    datastore: Arc<DataStore>,
+    resolver: Resolver,
+}
+
+impl SwitchPortSettingsManager {
+    pub fn new(datastore: Arc<DataStore>, resolver: Resolver) -> Self {
+        Self { datastore, resolver }
+    }
+
+    async fn switch_ports<'a>(
+        &'a mut self,
+        opctx: &OpContext,
+        log: &slog::Logger,
+    ) -> Result<Vec<nexus_db_model::SwitchPort>, serde_json::Value> {
+        let port_list = match self
+            .datastore
+            .switch_port_list(opctx, &DataPageParams::max_page())
+            .await
+        {
+            Ok(port_list) => port_list,
+            Err(e) => {
+                error!(
+                    &log,
+                    "failed to enumerate switch ports";
+                    "error" => format!("{:#}", e)
+                );
+                return Err(json!({
+                    "error":
+                        format!(
+                            "failed enumerate switch ports: \
+                                {:#}",
+                            e
+                        )
+                }));
+            }
+        };
+        Ok(port_list)
+    }
+
+    async fn changes<'a>(
+        &'a mut self,
+        port_list: Vec<nexus_db_model::SwitchPort>,
+        opctx: &OpContext,
+        log: &slog::Logger,
+    ) -> Result<
+        Vec<(SwitchLocation, nexus_db_model::SwitchPort, PortSettingsChange)>,
+        serde_json::Value,
+    > {
+        let mut changes = Vec::new();
+        for port in port_list {
+            let location: SwitchLocation =
+                match port.switch_location.clone().parse() {
+                    Ok(location) => location,
+                    Err(e) => {
+                        error!(
+                            &log,
+                            "failed to parse switch location";
+                            "switch_location" => ?port.switch_location,
+                            "error" => ?e
+                        );
+                        continue;
+                    }
+                };
+
+            let id = match port.port_settings_id {
+                Some(id) => id,
+                _ => {
+                    changes.push((location, port, PortSettingsChange::Clear));
+                    continue;
+                }
+            };
+
+            info!(
+                log,
+                "fetching switch port settings";
+                "switch_location" => ?location,
+                "port" => ?port,
+            );
+
+            let settings = match self
+                .datastore
+                .switch_port_settings_get(opctx, &id.into())
+                .await
+            {
+                Ok(settings) => settings,
+                Err(e) => {
+                    error!(
+                        &log,
+                        "failed to get switch port settings";
+                        "switch_port_settings_id" => ?id,
+                        "error" => format!("{:#}", e)
+                    );
+                    return Err(json!({
+                        "error":
+                            format!(
+                                "failed to get switch port settings: \
+                                    {:#}",
+                                e
+                            )
+                    }));
+                }
+            };
+
+            changes.push((
+                location,
+                port,
+                PortSettingsChange::Apply(Box::new(settings)),
+            ));
+        }
+        Ok(changes)
+    }
+
+    async fn db_loopback_addresses<'a>(
+        &'a mut self,
+        opctx: &OpContext,
+        log: &slog::Logger,
+    ) -> Result<
+        HashSet<(SwitchLocation, IpAddr)>,
+        omicron_common::api::external::Error,
+    > {
+        let values = self
+            .datastore
+            .loopback_address_list(opctx, &DataPageParams::max_page())
+            .await?;
+
+        let mut set: HashSet<(SwitchLocation, IpAddr)> = HashSet::new();
+
+        // TODO: are we doing anything special with anycast addresses at the moment?
+        for LoopbackAddress { switch_location, address, .. } in values.iter() {
+            let location: SwitchLocation = match switch_location.parse() {
+                Ok(v) => v,
+                Err(e) => {
+                    error!(
+                        log,
+                        "failed to parse switch location for loopback address";
+                        "address" => %address,
+                        "location" => switch_location,
+                        "error" => ?e,
+                    );
+                    continue;
+                }
+            };
+            set.insert((location, address.ip()));
+        }
+
+        Ok(set)
+    }
+}
+
+enum PortSettingsChange {
+    Apply(Box<SwitchPortSettingsCombinedResult>),
+    Clear,
+}
+
+impl BackgroundTask for SwitchPortSettingsManager {
+    fn activate<'a>(
+        &'a mut self,
+        opctx: &'a OpContext,
+    ) -> BoxFuture<'a, serde_json::Value> {
+        async move {
+            let log = &opctx.log;
+
+            let racks = match self.datastore.rack_list_initialized(opctx, &DataPageParams::max_page()).await {
+                Ok(racks) => racks,
+                Err(e) => {
+                    error!(log, "failed to retrieve racks from database"; "error" => ?e);
+                    return json!({
+                        "error":
+                            format!(
+                                "failed to retrieve racks from database : \
+                                    {:#}",
+                                e
+                            )
+                    });
+                },
+            };
+
+            // TODO: https://github.com/oxidecomputer/omicron/issues/3090
+            // Here we're iterating over racks because that's technically the correct thing to do,
+            // but our logic for pulling switch ports and their related configurations
+            // *isn't* per-rack, so that's something we'll need to revisit in the future.
+            for rack in &racks {
+
+                // lookup switch zones via DNS
+                // TODO https://github.com/oxidecomputer/omicron/issues/5201
+                let switch_zone_addresses = match self
+                    .resolver
+                    .lookup_all_ipv6(ServiceName::Dendrite)
+                    .await
+                {
+                    Ok(addrs) => addrs,
+                    Err(e) => {
+                        error!(log, "failed to resolve addresses for Dendrite services"; "error" => %e);
+                        continue;
+                    },
+                };
+
+                // TODO https://github.com/oxidecomputer/omicron/issues/5201
+                let mappings =
+                    map_switch_zone_addrs(log, switch_zone_addresses).await;
+
+                // TODO https://github.com/oxidecomputer/omicron/issues/5201
+                // build sled agent clients
+                let sled_agent_clients = build_sled_agent_clients(&mappings, log);
+
+                // TODO https://github.com/oxidecomputer/omicron/issues/5201
+                // build dpd clients
+                let dpd_clients = build_dpd_clients(&mappings, log);
+
+                // TODO https://github.com/oxidecomputer/omicron/issues/5201
+                // build mgd clients
+                let mgd_clients = build_mgd_clients(mappings, log);
+
+                let port_list = match self.switch_ports(opctx, log).await {
+                    Ok(value) => value,
+                    Err(e) => {
+                        error!(log, "failed to generate switchports for rack"; "error" => %e);
+                        continue;
+                    },
+                };
+
+                //
+                // calculate and apply switch port changes
+                //
+
+                let changes = match self.changes(port_list, opctx, log).await {
+                    Ok(value) => value,
+                    Err(e) => {
+                        error!(log, "failed to generate changeset for switchport settings"; "error" => %e);
+                        continue;
+                    },
+                };
+
+                apply_switch_port_changes(&dpd_clients, &changes, log).await;
+
+                //
+                // calculate and apply routing changes
+                //
+
+                // get the static routes on each switch
+                let current_static_routes =
+                    static_routes_on_switch(&mgd_clients, log).await;
+                info!(&log, "retrieved existing routes"; "routes" => ?current_static_routes);
+
+                // generate the complete set of static routes that should be on a given switch
+                let desired_static_routes = static_routes_in_db(&changes);
+                info!(&log, "retrieved desired routes"; "routes" => ?desired_static_routes);
+
+                // diff the current and desired routes.
+                // Add what is missing from current, remove what is not present in desired.
+                let routes_to_add = static_routes_to_add(
+                    &desired_static_routes,
+                    &current_static_routes,
+                    log,
+                );
+                info!(&log, "calculated static routes to add"; "routes" => ?routes_to_add);
+
+                let routes_to_del = static_routes_to_del(
+                    current_static_routes,
+                    desired_static_routes,
+                );
+                info!(&log, "calculated static routes to delete"; "routes" => ?routes_to_del);
+
+                // delete the unneeded routes first, just in case there is a conflicting route for
+                // one we need to add
+                info!(&log, "deleting static routes"; "routes" => ?routes_to_del);
+                delete_static_routes(&mgd_clients, routes_to_del, log).await;
+
+                // add the new routes
+                info!(&log, "adding static routes"; "routes" => ?routes_to_add);
+                add_static_routes(&mgd_clients, routes_to_add, log).await;
+
+
+                //
+                // calculate and apply loopback address changes
+                //
+
+                match self.db_loopback_addresses(opctx, log).await {
+                    Ok(desired_loopback_addresses) => {
+                        let current_loopback_addresses = switch_loopback_addresses(&dpd_clients, log).await;
+
+                        let loopbacks_to_add: Vec<(SwitchLocation, IpAddr)> = desired_loopback_addresses
+                            .difference(&current_loopback_addresses)
+                            .map(|i| (i.0, i.1))
+                            .collect();
+                        let loopbacks_to_del: Vec<(SwitchLocation, IpAddr)> = current_loopback_addresses
+                            .difference(&desired_loopback_addresses)
+                            .map(|i| (i.0, i.1))
+                            .collect();
+
+                        delete_loopback_addresses_from_switch(&loopbacks_to_del, &dpd_clients, log).await;
+                        add_loopback_addresses_to_switch(&loopbacks_to_add, dpd_clients, log).await;
+                    },
+                    Err(e) => {
+                        error!(
+                            log,
+                            "error fetching loopback addresses from db, skipping loopback config";
+                            "error" => %e
+                        );
+                    },
+                };
+
+                //
+                // calculate and apply switch zone SMF changes
+                //
+                let uplinks = uplinks(&changes);
+
+                // yeet the messages
+                for (location, config) in &uplinks {
+                    let client: &sled_agent_client::Client =
+                        match sled_agent_clients.get(location) {
+                            Some(client) => client,
+                            None => {
+                                error!(log, "sled-agent client is missing, cannot send updates"; "location" => %location);
+                                continue;
+                            },
+                        };
+
+                    info!(
+                        &log,
+                        "applying SMF config uplink updates to switch zone";
+                        "switch_location" => ?location,
+                        "config" => ?config,
+                    );
+                    if let Err(e) = client
+                        .uplink_ensure(&sled_agent_client::types::SwitchPorts {
+                            uplinks: config.clone(),
+                        })
+                        .await
+                    {
+                        error!(
+                            log,
+                            "error while applying smf updates to switch zone";
+                            "location" => %location,
+                            "error" => %e,
+                        );
+                    }
+                }
+
+                //
+                // calculate and apply BGP changes
+                //
+
+                // build a list of desired settings for each switch
+                let mut desired_bgp_configs: HashMap<
+                    SwitchLocation,
+                    Vec<ApplyRequest>,
+                > = HashMap::new();
+
+                // we currently only support one bgp config per switch
+                let mut switch_bgp_config: HashMap<SwitchLocation, (Uuid, BgpConfig)> = HashMap::new();
+
+                // Prefixes are associated to BgpConfig via the config id
+                let mut bgp_announce_prefixes: HashMap<Uuid, Vec<Prefix4>> = HashMap::new();
+
+                let mut bootstore_bgp_peer_info: Vec<(SwitchPortBgpPeerConfig, u32, Ipv4Addr)> = vec![];
+
+                for (location, port, change) in &changes {
+                    let PortSettingsChange::Apply(settings) = change else {
+                        continue;
+                    };
+
+                    // desired peer configurations for a given switch port
+                    let mut peers: HashMap<String, Vec<BgpPeerConfig>> = HashMap::new();
+
+                    for peer in &settings.bgp_peers {
+                        let bgp_config_id = peer.bgp_config_id;
+
+                        // since we only have one bgp config per switch, we only need to fetch it once
+                        let bgp_config = match switch_bgp_config.entry(*location) {
+                            Entry::Occupied(occupied_entry) => {
+                                let (existing_id, existing_config) = occupied_entry.get().clone();
+                                // verify peers don't have differing configs
+                                if existing_id != bgp_config_id {
+                                    // should we flag the switch and not do *any* updates to it?
+                                    // with the logic as-is, it will skip the config for this port and move on
+                                    error!(
+                                        log,
+                                        "peers do not have matching asn (only one asn allowed per switch)";
+                                        "switch" => ?location,
+                                        "first_config_id" => ?existing_id,
+                                        "second_config_id" => ?bgp_config_id,
+                                    );
+                                    break;
+                                }
+                                existing_config
+                            },
+                            Entry::Vacant(vacant_entry) => {
+                                // get the bgp config for this peer
+                                let config = match self
+                                    .datastore
+                                    .bgp_config_get(opctx, &bgp_config_id.into())
+                                    .await
+                                {
+                                    Ok(config) => config,
+                                    Err(e) => {
+                                        error!(
+                                            log,
+                                            "error while fetching bgp peer config from db";
+                                            "location" => %location,
+                                            "port_name" => %port.port_name,
+                                            "error" => %e,
+                                        );
+                                        continue;
+                                    },
+                                };
+                                vacant_entry.insert((bgp_config_id, config.clone()));
+                                config
+                            },
+                        };
+
+                        //
+                        // build a list of prefixes from the announcements in the bgp config
+                        //
+
+                        // Same thing as above, check to see if we've already built the announce set,
+                        // if so we'll skip this step
+                        if bgp_announce_prefixes.get(&bgp_config.bgp_announce_set_id).is_none() {
+                            let announcements = match self
+                                .datastore
+                                .bgp_announce_list(
+                                    opctx,
+                                    &params::BgpAnnounceSetSelector {
+                                        name_or_id: bgp_config
+                                            .bgp_announce_set_id
+                                            .into(),
+                                    },
+                                )
+                                .await
+                            {
+                                Ok(a) => a,
+                                Err(e) => {
+                                    error!(
+                                        log,
+                                        "error while fetching bgp announcements from db";
+                                        "location" => %location,
+                                        "bgp_announce_set_id" => %bgp_config.bgp_announce_set_id,
+                                        "error" => %e,
+                                    );
+                                    continue;
+                                },
+                            };
+
+                            let mut prefixes: Vec<Prefix4> = vec![];
+
+                            for announcement in &announcements {
+                                let value = match announcement.network.ip() {
+                                    IpAddr::V4(value) => value,
+                                    IpAddr::V6(a) => {
+                                        error!(log, "bad request, only ipv4 supported at this time"; "requested_address" => ?a);
+                                        continue;
+                                    },
+                                };
+                                prefixes.push(Prefix4 { value, length: announcement.network.prefix() });
+                            }
+                            bgp_announce_prefixes.insert(bgp_config.bgp_announce_set_id, prefixes);
+                        }
+
+                        // now that the peer passes the above validations, add it to the list for configuration
+                        let peer_config = BgpPeerConfig {
+                            name: format!("{}", peer.addr.ip()),
+                            host: format!("{}:179", peer.addr.ip()),
+                            hold_time: peer.hold_time.0.into(),
+                            idle_hold_time: peer.idle_hold_time.0.into(),
+                            delay_open: peer.delay_open.0.into(),
+                            connect_retry: peer.connect_retry.0.into(),
+                            keepalive: peer.keepalive.0.into(),
+                            resolution: BGP_SESSION_RESOLUTION,
+                            passive: false,
+                        };
+
+                        // add it to data for the bootstore
+                        // only ipv4 is supported now
+                        match peer.addr {
+                            ipnetwork::IpNetwork::V4(addr) => {
+                                bootstore_bgp_peer_info.push((peer.clone(), bgp_config.asn.0, addr.ip()));
+                            },
+                            ipnetwork::IpNetwork::V6(_) => continue, //TODO v6
+                        };
+
+                        // update the stored vec if it exists, create a new on if it doesn't exist
+                        match peers.entry(port.port_name.clone()) {
+                            Entry::Occupied(mut occupied_entry) => {
+                                occupied_entry.get_mut().push(peer_config);
+                            },
+                            Entry::Vacant(vacant_entry) => {
+                                vacant_entry.insert(vec![peer_config]);
+                            },
+                        }
+                    }
+
+                    let (config_id, request_bgp_config) = match switch_bgp_config.get(location) {
+                        Some(config) => config,
+                        None => {
+                            info!(log, "no bgp config found for switch, skipping."; "switch" => ?location);
+                            continue;
+                        },
+                    };
+
+                    let request_prefixes = match bgp_announce_prefixes.get(&request_bgp_config.bgp_announce_set_id) {
+                        Some(prefixes) => prefixes,
+                        None => {
+                            error!(
+                                log,
+                                "no prefixes to announce found for bgp config";
+                                "switch" => ?location,
+                                "announce_set_id" => ?request_bgp_config.bgp_announce_set_id,
+                                "bgp_config_id" => ?config_id,
+                            );
+                            continue;
+                        },
+                    };
+
+                    let request = ApplyRequest {
+                        asn: *request_bgp_config.asn,
+                        peers,
+                        originate: request_prefixes.clone(),
+                    };
+
+                    match desired_bgp_configs.entry(*location) {
+                        Entry::Occupied(mut occupied_entry) => {
+                            occupied_entry.get_mut().push(request);
+                        }
+                        Entry::Vacant(vacant_entry) => {
+                            vacant_entry.insert(vec![request]);
+                        }
+                    }
+                }
+
+                for (location, configs) in &desired_bgp_configs {
+                    let client = match mgd_clients.get(location) {
+                        Some(client) => client,
+                        None => {
+                            error!(log, "no mgd client found for switch"; "switch_location" => ?location);
+                            continue;
+                        },
+                    };
+                    for config in configs {
+                        info!(
+                            &log,
+                            "applying bgp config";
+                            "switch_location" => ?location,
+                            "config" => ?config,
+                        );
+                        if let Err(e) = client.inner.bgp_apply(config).await {
+                            error!(log, "error while applying bgp configuration"; "error" => ?e);
+                        }
+                    }
+                }
+
+                //
+                // calculate and apply bootstore changes
+                //
+
+                // TODO: #5232 Make ntp servers w/ generation tracking first-class citizens in the db
+                // We're using the latest bootstore config from the sled agents to get the ntp
+                // servers. We should instead be pulling this information from the db. However, it
+                // seems that we're currently not storing the ntp servers in the db as a first-class
+                // citizen, so we'll need to add that first.
+
+                // find the active sled-agent bootstore config with the highest generation
+                let mut latest_sled_agent_bootstore_config: Option<EarlyNetworkConfig> = None;
+
+                // Since we update the first scrimlet we can reach (we failover to the second one
+                // if updating the first one fails) we need to check them both.
+                for (_location, client) in &sled_agent_clients {
+                    let scrimlet_cfg  = match client.read_network_bootstore_config_cache().await {
+                        Ok(config) => config,
+                        Err(e) => {
+                            error!(log, "unable to read bootstore config from scrimlet"; "error" => ?e);
+                            continue;
+                        }
+                    };
+                    if let Some(other_config) = latest_sled_agent_bootstore_config.as_mut() {
+                        if other_config.generation < scrimlet_cfg.generation {
+                            *other_config = scrimlet_cfg.clone();
+                        }
+                    } else {
+                        latest_sled_agent_bootstore_config = Some(scrimlet_cfg.clone());
+                    }
+                }
+
+                // TODO: this will also be removed once the above is resolved
+                // Move on to the next rack if neither scrimlet is reachable.
+                // if both scrimlets are unreachable we probably have bigger problems on this rack
+                let ntp_servers = match latest_sled_agent_bootstore_config {
+                    Some(config) => {
+                        config.body.ntp_servers.clone()
+                    },
+                    None => {
+                        error!(log, "both scrimlets are unreachable, cannot update bootstore");
+                        continue;
+                    }
+                };
+
+                // build the desired bootstore config from the records we've fetched
+                let subnet = match rack.rack_subnet {
+                    Some(IpNetwork::V6(subnet)) => subnet,
+                    Some(IpNetwork::V4(_)) => {
+                        error!(log, "rack subnet must be ipv6"; "rack" => ?rack);
+                        continue;
+                    },
+                    None => {
+                        error!(log, "rack subnet not set"; "rack" => ?rack);
+                        continue;
+                    }
+                };
+
+                // TODO: @rcgoodfellow is this correct? Do we place the BgpConfig for both switches in a single Vec to send to the bootstore?
+                let bgp: Vec<SledBgpConfig> = switch_bgp_config.iter().map(|(_location, (_id, config))| {
+                    let announcements: Vec<Ipv4Network> = bgp_announce_prefixes
+                        .get(&config.bgp_announce_set_id)
+                        .expect("bgp config is present but announce set is not populated")
+                        .iter()
+                        .map(|prefix| {
+                            ipnetwork::Ipv4Network::new(prefix.value, prefix.length)
+                                .expect("Prefix4 and Ipv4Network's value types have diverged")
+                                .into()
+                        }).collect();
+
+                    SledBgpConfig {
+                        asn: config.asn.0,
+                        originate: announcements,
+                    }
+                }).collect();
+
+                let mut ports: Vec<PortConfigV1> = vec![];
+
+                for (location, port, change) in &changes {
+                    let PortSettingsChange::Apply(info) = change else {
+                        continue;
+                    };
+
+                    let port_config = PortConfigV1 {
+                        addresses: info.addresses.iter().map(|a| a.address).collect(),
+                        autoneg: info
+                            .links
+                            .get(0) //TODO breakout support
+                            .map(|l| l.autoneg)
+                            .unwrap_or(false),
+                        bgp_peers: bootstore_bgp_peer_info
+                            .iter()
+                            .map(|(p, asn, addr)| SledBgpPeerConfig {
+                                addr: *addr,
+                                asn: *asn,
+                                port: port.port_name.clone(),
+                                hold_time: Some(p.hold_time.0.into()),
+                                connect_retry: Some(p.connect_retry.0.into()),
+                                delay_open: Some(p.delay_open.0.into()),
+                                idle_hold_time: Some(p.idle_hold_time.0.into()),
+                                keepalive: Some(p.keepalive.0.into()),
+                            })
+                            .collect(),
+                        port: port.port_name.clone(),
+                        routes: info
+                            .routes
+                            .iter()
+                            .map(|r| SledRouteConfig {
+                                destination: r.dst,
+                                nexthop: r.gw.ip(),
+                            })
+                            .collect(),
+                        switch: *location,
+                        uplink_port_fec: info
+                            .links
+                            .get(0) //TODO https://github.com/oxidecomputer/omicron/issues/3062
+                            .map(|l| l.fec)
+                            .unwrap_or(SwitchLinkFec::None)
+                            .into(),
+                        uplink_port_speed: info
+                            .links
+                            .get(0) //TODO https://github.com/oxidecomputer/omicron/issues/3062
+                            .map(|l| l.speed)
+                            .unwrap_or(SwitchLinkSpeed::Speed100G)
+                            .into(),
+                    };
+                    ports.push(port_config);
+                }
+
+                let blocks = match self.datastore.address_lot_blocks_by_name(opctx, INFRA_LOT.into()).await {
+                    Ok(blocks) => blocks,
+                    Err(e) => {
+                        error!(log, "error while fetching address lot blocks from db"; "error" => %e);
+                        continue;
+                    },
+                };
+
+                // currently there should only be one block assigned. If there is more than one
+                // block, grab the first one and emit a warning.
+                if blocks.len() > 1 {
+                    warn!(log, "more than one block assigned to infra lot"; "blocks" => ?blocks);
+                }
+
+                let (infra_ip_first, infra_ip_last)= match blocks.get(0) {
+                    Some(AddressLotBlock{ first_address, last_address, ..}) => {
+                        match (first_address, last_address) {
+                            (IpNetwork::V4(first), IpNetwork::V4(last)) => (first.ip(), last.ip()),
+                            _ =>  {
+                                error!(log, "infra lot block must be ipv4"; "block" => ?blocks.get(0));
+                                continue;
+                            },
+                        }
+                    },
+                    None => {
+                        error!(log, "no blocks assigned to infra lot");
+                        continue;
+                    },
+                };
+
+                let mut desired_config = EarlyNetworkConfig {
+                    generation: 0,
+                    schema_version: 1,
+                    body: EarlyNetworkConfigBody {
+                        ntp_servers,
+                        rack_network_config: Some(RackNetworkConfigV1 {
+                            rack_subnet: subnet,
+                            infra_ip_first,
+                            infra_ip_last,
+                            ports,
+                            bgp,
+                        }),
+                    },
+                };
+
+                // should_update is a boolean value that determines whether or not we need to
+                // increment the bootstore version and push a new config to the sled agents.
+                //
+                // * If the config we've built from the switchport configuration information is
+                //   different from the last config we've cached in the db, we update the config,
+                //   cache it in the db, and apply it.
+                // * If the last cached config cannot be succesfully deserialized into our current
+                //   bootstore format, we assume that it is an older format and update the config,
+                //   cache it in the db, and apply it.
+                // * If there is no last cached config, we assume that this is the first time this
+                //   rpw has run for the given rack, so we update the config, cache it in the db,
+                //   and apply it.
+                // * If we cannot fetch the latest version due to a db error, something is broken
+                //   so we don't do anything.
+                let bootstore_needs_update = match self.datastore.get_latest_bootstore_config(opctx, NETWORK_KEY.into()).await {
+                    Ok(Some(BootstoreConfig { data, .. })) => {
+                        match serde_json::from_value::<EarlyNetworkConfig>(data.clone()) {
+                            Ok(config) => {
+                                if config.body.ntp_servers != desired_config.body.ntp_servers {
+                                    info!(
+                                        log,
+                                        "ntp servers have changed";
+                                        "old" => ?config.body.ntp_servers,
+                                        "new" => ?desired_config.body.ntp_servers,
+                                    );
+                                    true
+                                } else if config.body.rack_network_config != desired_config.body.rack_network_config {
+                                    info!(
+                                        log,
+                                        "rack network config has changed";
+                                        "old" => ?config.body.rack_network_config,
+                                        "new" => ?desired_config.body.rack_network_config,
+                                    );
+                                    true
+                                } else {
+                                    false
+                                }
+                            },
+                            Err(e) => {
+                                error!(
+                                    log,
+                                    "bootstore config does not deserialized to current EarlyNetworkConfig format";
+                                    "key" => %NETWORK_KEY,
+                                    "value" => %data,
+                                    "error" => %e,
+                                );
+                                true
+                            },
+                        }
+                    },
+                    Ok(None) => {
+                        warn!(
+                            log,
+                            "no bootstore config found in db";
+                            "key" => %NETWORK_KEY,
+                        );
+                        true
+                    },
+                    Err(e) => {
+                        error!(
+                            log,
+                            "error while fetching last applied bootstore config";
+                            "key" => %NETWORK_KEY,
+                            "error" => %e,
+                        );
+                        continue;
+                    },
+                };
+
+                if bootstore_needs_update {
+                    let generation = match self.datastore
+                        .bump_bootstore_generation(opctx, NETWORK_KEY.into())
+                        .await {
+                        Ok(value) => value,
+                            Err(e) => {
+                                error!(
+                                    log,
+                                    "error while fetching next bootstore generation from db";
+                                    "key" => %NETWORK_KEY,
+                                    "error" => %e,
+                                );
+                                continue;
+                            },
+                        };
+
+                    desired_config.generation = generation as u64;
+                    info!(
+                        &log,
+                        "updating bootstore config";
+                        "config" => ?desired_config,
+                    );
+
+                    // spush the updates to both scrimlets
+                    // if both scrimlets are down, bootstore updates aren't happening anyway
+                    let mut one_succeeded = false;
+                    for (location, client) in &sled_agent_clients {
+                        if let Err(e) = client.write_network_bootstore_config(&desired_config).await {
+                            error!(
+                                log,
+                                "error updating bootstore";
+                                "location" => %location,
+                                "config" => ?desired_config,
+                                "error" => %e,
+                            )
+                        } else {
+                            one_succeeded = true;
+                        }
+                    }
+
+                    // if at least one succeeded, record this update in the db
+                    if one_succeeded {
+                        let config = BootstoreConfig {
+                            key: NETWORK_KEY.into(),
+                            generation: desired_config.generation as i64,
+                            data: serde_json::to_value(&desired_config).unwrap(),
+                            time_created: chrono::Utc::now(),
+                            time_deleted: None,
+                        };
+                        if let Err(e) = self.datastore.ensure_bootstore_config(opctx, config.clone()).await {
+                            // if this fails, worst case scenario is that we will send the bootstore
+                            // information it already has on the next run
+                            error!(
+                                log,
+                                "error while caching bootstore config in db";
+                                "config" => ?config,
+                                "error" => %e,
+                            );
+                        }
+                    }
+                }
+            }
+            json!({})
+        }
+        .boxed()
+    }
+}
+
+async fn add_loopback_addresses_to_switch(
+    loopbacks_to_add: &[(SwitchLocation, IpAddr)],
+    dpd_clients: HashMap<SwitchLocation, dpd_client::Client>,
+    log: &slog::Logger,
+) {
+    for (location, address) in loopbacks_to_add {
+        let client = match dpd_clients.get(location) {
+            Some(v) => v,
+            None => {
+                error!(log, "dpd_client is missing, cannot create loopback addresses"; "location" => %location);
+                continue;
+            }
+        };
+
+        if let Err(e) =
+            client.ensure_loopback_created(log, *address, OMICRON_DPD_TAG).await
+        {
+            error!(log, "error while creating loopback address"; "error" => %e);
+        };
+    }
+}
+
+async fn delete_loopback_addresses_from_switch(
+    loopbacks_to_del: &[(SwitchLocation, IpAddr)],
+    dpd_clients: &HashMap<SwitchLocation, dpd_client::Client>,
+    log: &slog::Logger,
+) {
+    for (location, address) in loopbacks_to_del {
+        let client = match dpd_clients.get(location) {
+            Some(v) => v,
+            None => {
+                error!(log, "dpd_client is missing, cannot delete loopback addresses"; "location" => %location);
+                continue;
+            }
+        };
+
+        if let Err(e) = client.ensure_loopback_deleted(log, *address).await {
+            error!(log, "error while deleting loopback address"; "error" => %e);
+        };
+    }
+}
+
+async fn switch_loopback_addresses(
+    dpd_clients: &HashMap<SwitchLocation, dpd_client::Client>,
+    log: &slog::Logger,
+) -> HashSet<(SwitchLocation, IpAddr)> {
+    let mut current_loopback_addresses: HashSet<(SwitchLocation, IpAddr)> =
+        HashSet::new();
+
+    for (location, client) in dpd_clients {
+        let ipv4_loopbacks = match client.loopback_ipv4_list().await {
+            Ok(v) => v,
+            Err(e) => {
+                error!(
+                    log,
+                    "error fetching ipv4 loopback addresses from switch";
+                    "location" => %location,
+                    "error" => %e,
+                );
+                continue;
+            }
+        };
+
+        let ipv6_loopbacks = match client.loopback_ipv6_list().await {
+            Ok(v) => v,
+            Err(e) => {
+                error!(
+                    log,
+                    "error fetching ipv6 loopback addresses from switch";
+                    "location" => %location,
+                    "error" => %e,
+                );
+                continue;
+            }
+        };
+
+        for entry in ipv4_loopbacks.iter() {
+            current_loopback_addresses
+                .insert((*location, IpAddr::V4(entry.addr)));
+        }
+
+        for entry in ipv6_loopbacks.iter() {
+            current_loopback_addresses
+                .insert((*location, IpAddr::V6(entry.addr)));
+        }
+    }
+    current_loopback_addresses
+}
+
+fn uplinks(
+    changes: &[(
+        SwitchLocation,
+        nexus_db_model::SwitchPort,
+        PortSettingsChange,
+    )],
+) -> HashMap<SwitchLocation, Vec<HostPortConfig>> {
+    let mut uplinks: HashMap<SwitchLocation, Vec<HostPortConfig>> =
+        HashMap::new();
+    for (location, port, change) in changes {
+        let PortSettingsChange::Apply(config) = change else {
+            continue;
+        };
+        let config = HostPortConfig {
+            port: port.port_name.clone(),
+            addrs: config.addresses.iter().map(|a| a.address).collect(),
+        };
+
+        match uplinks.entry(*location) {
+            Entry::Occupied(mut occupied_entry) => {
+                occupied_entry.get_mut().push(config);
+            }
+            Entry::Vacant(vacant_entry) => {
+                vacant_entry.insert(vec![config]);
+            }
+        }
+    }
+    uplinks
+}
+
+fn build_sled_agent_clients(
+    mappings: &HashMap<SwitchLocation, std::net::Ipv6Addr>,
+    log: &slog::Logger,
+) -> HashMap<SwitchLocation, sled_agent_client::Client> {
+    let sled_agent_clients: HashMap<SwitchLocation, sled_agent_client::Client> =
+        mappings
+            .iter()
+            .map(|(location, addr)| {
+                // build sled agent address from switch zone address
+                let addr = get_sled_address(Ipv6Subnet::new(*addr));
+                let client = sled_agent_client::Client::new(
+                    &format!("http://{}", addr),
+                    log.clone(),
+                );
+                (*location, client)
+            })
+            .collect();
+    sled_agent_clients
+}
+
+fn static_routes_to_del(
+    current_static_routes: HashMap<
+        SwitchLocation,
+        HashSet<(Ipv4Addr, Prefix4)>,
+    >,
+    desired_static_routes: HashMap<
+        SwitchLocation,
+        HashSet<(Ipv4Addr, Prefix4)>,
+    >,
+) -> HashMap<SwitchLocation, DeleteStaticRoute4Request> {
+    let mut routes_to_del: HashMap<SwitchLocation, DeleteStaticRoute4Request> =
+        HashMap::new();
+
+    // find routes to remove
+    for (switch_location, routes_on_switch) in &current_static_routes {
+        if let Some(routes_wanted) = desired_static_routes.get(switch_location)
+        {
+            // if it's on the switch but not desired (in our db), it should be removed
+            let stale_routes = routes_on_switch
+                .difference(routes_wanted)
+                .map(|(nexthop, prefix)| StaticRoute4 {
+                    nexthop: *nexthop,
+                    prefix: prefix.clone(),
+                })
+                .collect::<Vec<StaticRoute4>>();
+
+            routes_to_del.insert(
+                *switch_location,
+                DeleteStaticRoute4Request {
+                    routes: StaticRoute4List { list: stale_routes },
+                },
+            );
+        } else {
+            // if no desired routes are present, all routes on this switch should be deleted
+            let stale_routes = routes_on_switch
+                .iter()
+                .map(|(nexthop, prefix)| StaticRoute4 {
+                    nexthop: *nexthop,
+                    prefix: prefix.clone(),
+                })
+                .collect::<Vec<StaticRoute4>>();
+
+            let req = DeleteStaticRoute4Request {
+                routes: StaticRoute4List { list: stale_routes },
+            };
+
+            routes_to_del.insert(*switch_location, req);
+            continue;
+        };
+    }
+    routes_to_del
+}
+
+fn static_routes_to_add(
+    desired_static_routes: &HashMap<
+        SwitchLocation,
+        HashSet<(Ipv4Addr, Prefix4)>,
+    >,
+    current_static_routes: &HashMap<
+        SwitchLocation,
+        HashSet<(Ipv4Addr, Prefix4)>,
+    >,
+    log: &slog::Logger,
+) -> HashMap<SwitchLocation, AddStaticRoute4Request> {
+    let mut routes_to_add: HashMap<SwitchLocation, AddStaticRoute4Request> =
+        HashMap::new();
+
+    // find routes to add
+    for (switch_location, routes_wanted) in desired_static_routes {
+        let routes_on_switch = match current_static_routes.get(&switch_location)
+        {
+            Some(routes) => routes,
+            None => {
+                warn!(
+                    &log,
+                    "no discovered routes from switch. it is possible that an earlier api call failed.";
+                    "switch_location" => ?switch_location,
+                );
+                continue;
+            }
+        };
+        let missing_routes = routes_wanted
+            .difference(routes_on_switch)
+            .map(|(nexthop, prefix)| StaticRoute4 {
+                nexthop: *nexthop,
+                prefix: prefix.clone(),
+            })
+            .collect::<Vec<StaticRoute4>>();
+
+        routes_to_add.insert(
+            *switch_location,
+            AddStaticRoute4Request {
+                routes: StaticRoute4List { list: missing_routes },
+            },
+        );
+    }
+    routes_to_add
+}
+
+fn static_routes_in_db(
+    changes: &[(
+        SwitchLocation,
+        nexus_db_model::SwitchPort,
+        PortSettingsChange,
+    )],
+) -> HashMap<SwitchLocation, HashSet<(Ipv4Addr, Prefix4)>> {
+    let mut routes_from_db: HashMap<
+        SwitchLocation,
+        HashSet<(Ipv4Addr, Prefix4)>,
+    > = HashMap::new();
+
+    for (location, _port, change) in changes {
+        // we only need to check for ports that have a configuration present. No config == no routes.
+        let PortSettingsChange::Apply(settings) = change else {
+            continue;
+        };
+        let mut routes = HashSet::new();
+        for route in &settings.routes {
+            // convert to appropriate types for comparison and insertion
+            let nexthop = match route.gw.ip() {
+                IpAddr::V4(v4) => v4,
+                IpAddr::V6(_) => continue,
+            };
+            let prefix = match route.dst.ip() {
+                IpAddr::V4(v4) => {
+                    Prefix4 { value: v4, length: route.dst.prefix() }
+                }
+                IpAddr::V6(_) => continue,
+            };
+            routes.insert((nexthop, prefix));
+        }
+
+        match routes_from_db.entry(*location) {
+            Entry::Occupied(mut occupied_entry) => {
+                occupied_entry.get_mut().extend(routes);
+            }
+            Entry::Vacant(vacant_entry) => {
+                vacant_entry.insert(routes);
+            }
+        }
+    }
+    routes_from_db
+}
+
+// apply changes for each port
+// if we encounter an error, we log it and keep going instead of bailing
+async fn apply_switch_port_changes(
+    dpd_clients: &HashMap<SwitchLocation, dpd_client::Client>,
+    changes: &[(
+        SwitchLocation,
+        nexus_db_model::SwitchPort,
+        PortSettingsChange,
+    )],
+    log: &slog::Logger,
+) {
+    for (location, switch_port, change) in changes {
+        let client = match dpd_clients.get(&location) {
+            Some(client) => client,
+            None => {
+                error!(
+                    &log,
+                    "no DPD client for switch location";
+                    "switch_location" => ?location
+                );
+                continue;
+            }
+        };
+
+        let port_name = switch_port.port_name.clone();
+
+        let dpd_port_id = match PortId::from_str(port_name.as_str()) {
+            Ok(port_id) => port_id,
+            Err(e) => {
+                error!(
+                    &log,
+                    "failed to parse switch port id";
+                    "db_switch_port_name" => ?switch_port.port_name,
+                    "switch_location" => ?location,
+                    "error" => format!("{:#}", e)
+                );
+                continue;
+            }
+        };
+
+        match change {
+            PortSettingsChange::Apply(settings) => {
+                let dpd_port_settings = match api_to_dpd_port_settings(
+                    &settings,
+                ) {
+                    Ok(settings) => settings,
+                    Err(e) => {
+                        error!(
+                            &log,
+                            "failed to convert switch port settings";
+                            "switch_port_id" => ?port_name,
+                            "switch_location" => ?location,
+                            "switch_port_settings_id" => ?settings.settings.id(),
+                            "error" => format!("{:#}", e)
+                        );
+                        continue;
+                    }
+                };
+
+                // apply settings via dpd client
+                info!(
+                    &log,
+                    "applying settings to switch port";
+                    "switch_location" => ?location,
+                    "port_id" => ?dpd_port_id,
+                    "settings" => ?dpd_port_settings,
+                );
+                match client
+                    .port_settings_apply(
+                        &dpd_port_id,
+                        DPD_TAG,
+                        &dpd_port_settings,
+                    )
+                    .await
+                {
+                    Ok(_) => {}
+                    Err(e) => {
+                        error!(
+                            &log,
+                            "failed to apply switch port settings";
+                            "switch_port_id" => ?port_name,
+                            "switch_location" => ?location,
+                            "error" => format!("{:#}", e)
+                        );
+                    }
+                }
+            }
+            PortSettingsChange::Clear => {
+                // clear settings via dpd client
+                info!(
+                    &log,
+                    "clearing switch port settings";
+                    "switch_location" => ?location,
+                    "port_id" => ?dpd_port_id,
+                );
+                match client.port_settings_clear(&dpd_port_id, DPD_TAG).await {
+                    Ok(_) => {}
+                    Err(e) => {
+                        error!(
+                            &log,
+                            "failed to clear switch port settings";
+                            "switch_port_id" => ?port_name,
+                            "switch_location" => ?location,
+                            "error" => format!("{:#}", e)
+                        );
+                    }
+                }
+            }
+        }
+    }
+}
+
+async fn static_routes_on_switch<'a>(
+    mgd_clients: &HashMap<SwitchLocation, mg_admin_client::Client>,
+    log: &slog::Logger,
+) -> HashMap<SwitchLocation, HashSet<(Ipv4Addr, Prefix4)>> {
+    let mut routes_on_switch = HashMap::new();
+
+    for (location, client) in mgd_clients {
+        let static_routes: HashSet<(Ipv4Addr, Prefix4)> =
+            match client.inner.static_list_v4_routes().await {
+                Ok(routes) => routes
+                    .list
+                    .iter()
+                    .map(|r| (r.nexthop, r.prefix.clone()))
+                    .collect(),
+                Err(_) => {
+                    error!(
+                        &log,
+                        "unable to retrieve routes from switch";
+                        "switch_location" => ?location,
+                    );
+                    continue;
+                }
+            };
+        routes_on_switch.insert(*location, static_routes);
+    }
+    routes_on_switch
+}
+
+async fn delete_static_routes(
+    mgd_clients: &HashMap<SwitchLocation, mg_admin_client::Client>,
+    routes_to_del: HashMap<SwitchLocation, DeleteStaticRoute4Request>,
+    log: &slog::Logger,
+) {
+    for (switch_location, request) in routes_to_del {
+        let client = match mgd_clients.get(&switch_location) {
+            Some(client) => client,
+            None => {
+                error!(
+                    &log,
+                    "mgd client not found for switch location";
+                    "switch_location" => ?switch_location,
+                );
+                continue;
+            }
+        };
+
+        info!(
+            &log,
+            "removing static v4 routes";
+            "switch_location" => ?switch_location,
+            "request" => ?request,
+        );
+        if let Err(e) = client.inner.static_remove_v4_route(&request).await {
+            error!(
+                &log,
+                "failed to delete routes from mgd";
+                "switch_location" => ?switch_location,
+                "request" => ?request,
+                "error" => format!("{:#}", e)
+            );
+        };
+    }
+}
+
+async fn add_static_routes<'a>(
+    mgd_clients: &HashMap<SwitchLocation, mg_admin_client::Client>,
+    routes_to_add: HashMap<SwitchLocation, AddStaticRoute4Request>,
+    log: &slog::Logger,
+) {
+    for (switch_location, request) in routes_to_add {
+        let client = match mgd_clients.get(&switch_location) {
+            Some(client) => client,
+            None => {
+                error!(
+                    &log,
+                    "mgd client not found for switch location";
+                    "switch_location" => ?switch_location,
+                );
+                continue;
+            }
+        };
+
+        info!(
+            &log,
+            "adding static v4 routes";
+            "switch_location" => ?switch_location,
+            "request" => ?request,
+        );
+        if let Err(e) = client.inner.static_add_v4_route(&request).await {
+            error!(
+                &log,
+                "failed to add routes to mgd";
+                "switch_location" => ?switch_location,
+                "request" => ?request,
+                "error" => format!("{:#}", e)
+            );
+        };
+    }
+}
diff --git a/nexus/src/app/bfd.rs b/nexus/src/app/bfd.rs
index 2d95ad9a58..22b9fc82ef 100644
--- a/nexus/src/app/bfd.rs
+++ b/nexus/src/app/bfd.rs
@@ -7,15 +7,18 @@ use mg_admin_client::types::BfdPeerState;
 use nexus_db_queries::context::OpContext;
 use nexus_types::external_api::shared::{BfdState, BfdStatus};
 use omicron_common::api::{external::Error, internal::shared::SwitchLocation};
-use std::sync::Arc;
 
 impl super::Nexus {
-    fn mg_client_for_switch_location(
+    async fn mg_client_for_switch_location(
         &self,
         switch: SwitchLocation,
-    ) -> Result<Arc<mg_admin_client::Client>, Error> {
-        let mg_client: Arc<mg_admin_client::Client> = self
-            .mg_clients
+    ) -> Result<mg_admin_client::Client, Error> {
+        let mg_client: mg_admin_client::Client = self
+            .mg_clients()
+            .await
+            .map_err(|e| {
+                Error::internal_error(&format!("failed to get mg clients: {e}"))
+            })?
             .get(&switch)
             .ok_or_else(|| {
                 Error::not_found_by_name(
@@ -64,7 +67,7 @@ impl super::Nexus {
         // be updated for multirack.
         let mut result = Vec::new();
         for s in &[SwitchLocation::Switch0, SwitchLocation::Switch1] {
-            let mg_client = self.mg_client_for_switch_location(*s)?;
+            let mg_client = self.mg_client_for_switch_location(*s).await?;
             let status = mg_client
                 .inner
                 .get_bfd_peers()
diff --git a/nexus/src/app/bgp.rs b/nexus/src/app/bgp.rs
index 51d22dbced..9e609712e2 100644
--- a/nexus/src/app/bgp.rs
+++ b/nexus/src/app/bgp.rs
@@ -8,8 +8,8 @@ use nexus_db_model::{BgpAnnounceSet, BgpAnnouncement, BgpConfig};
 use nexus_db_queries::context::OpContext;
 use omicron_common::api::external::http_pagination::PaginatedBy;
 use omicron_common::api::external::{
-    BgpImportedRouteIpv4, BgpPeerStatus, CreateResult, DeleteResult, Ipv4Net,
-    ListResultVec, LookupResult, NameOrId,
+    self, BgpImportedRouteIpv4, BgpPeerStatus, CreateResult, DeleteResult,
+    Ipv4Net, ListResultVec, LookupResult, NameOrId,
 };
 
 impl super::Nexus {
@@ -88,7 +88,11 @@ impl super::Nexus {
     ) -> ListResultVec<BgpPeerStatus> {
         opctx.authorize(authz::Action::Read, &authz::FLEET).await?;
         let mut result = Vec::new();
-        for (switch, client) in &self.mg_clients {
+        for (switch, client) in &self.mg_clients().await.map_err(|e| {
+            external::Error::internal_error(&format!(
+                "failed to get mg clients: {e}"
+            ))
+        })? {
             let router_info = match client.inner.get_routers().await {
                 Ok(result) => result.into_inner(),
                 Err(e) => {
@@ -126,7 +130,11 @@ impl super::Nexus {
     ) -> ListResultVec<BgpImportedRouteIpv4> {
         opctx.authorize(authz::Action::Read, &authz::FLEET).await?;
         let mut result = Vec::new();
-        for (switch, client) in &self.mg_clients {
+        for (switch, client) in &self.mg_clients().await.map_err(|e| {
+            external::Error::internal_error(&format!(
+                "failed to get mg clients: {e}"
+            ))
+        })? {
             let imported: Vec<BgpImportedRouteIpv4> = match client
                 .inner
                 .get_imported4(&mg_admin_client::types::GetImported4Request {
diff --git a/nexus/src/app/instance_network.rs b/nexus/src/app/instance_network.rs
index 741b5b8b6d..c345809f4d 100644
--- a/nexus/src/app/instance_network.rs
+++ b/nexus/src/app/instance_network.rs
@@ -28,7 +28,6 @@ use sled_agent_client::types::DeleteVirtualNetworkInterfaceHost;
 use sled_agent_client::types::SetVirtualNetworkInterfaceHost;
 use std::collections::HashSet;
 use std::str::FromStr;
-use std::sync::Arc;
 use uuid::Uuid;
 
 impl super::Nexus {
@@ -464,7 +463,7 @@ impl super::Nexus {
         probe_id: Uuid,
         sled_ip_address: std::net::Ipv6Addr,
         ip_index_filter: Option<usize>,
-        dpd_client: &Arc<dpd_client::Client>,
+        dpd_client: &dpd_client::Client,
     ) -> Result<(), Error> {
         let log = &self.log;
 
@@ -738,7 +737,12 @@ impl super::Nexus {
                        "instance_id" => ?instance_id,
                        "switch" => switch.to_string());
 
-            let client_result = self.dpd_clients.get(switch).ok_or_else(|| {
+            let clients = self.dpd_clients().await.map_err(|e| {
+                Error::internal_error(&format!(
+                    "failed to get dpd clients: {e}"
+                ))
+            })?;
+            let client_result = clients.get(switch).ok_or_else(|| {
                 Error::internal_error(&format!(
                     "unable to find dendrite client for {switch}"
                 ))
@@ -823,7 +827,13 @@ impl super::Nexus {
                        "probe_id" => %probe_id,
                        "switch" => switch.to_string());
 
-            let client_result = self.dpd_clients.get(switch).ok_or_else(|| {
+            let dpd_clients = self.dpd_clients().await.map_err(|e| {
+                Error::internal_error(&format!(
+                    "unable to get dpd_clients: {e}"
+                ))
+            })?;
+
+            let client_result = dpd_clients.get(switch).ok_or_else(|| {
                 Error::internal_error(&format!(
                     "unable to find dendrite client for {switch}"
                 ))
diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs
index d387998f6a..9d94e08a5d 100644
--- a/nexus/src/app/mod.rs
+++ b/nexus/src/app/mod.rs
@@ -30,6 +30,7 @@ use omicron_common::api::external::Error;
 use omicron_common::api::internal::shared::SwitchLocation;
 use slog::Logger;
 use std::collections::HashMap;
+use std::net::SocketAddrV6;
 use std::net::{IpAddr, Ipv6Addr};
 use std::sync::Arc;
 use uuid::Uuid;
@@ -181,12 +182,6 @@ pub struct Nexus {
     // https://github.com/oxidecomputer/omicron/issues/3732
     external_dns_servers: Vec<IpAddr>,
 
-    /// Mapping of SwitchLocations to their respective Dendrite Clients
-    dpd_clients: HashMap<SwitchLocation, Arc<dpd_client::Client>>,
-
-    /// Map switch location to maghemite admin clients.
-    mg_clients: HashMap<SwitchLocation, Arc<mg_admin_client::Client>>,
-
     /// Background tasks
     background_tasks: background::BackgroundTasks,
 
@@ -372,8 +367,6 @@ impl Nexus {
             &background_ctx,
             Arc::clone(&db_datastore),
             &config.pkg.background_tasks,
-            &dpd_clients,
-            &mg_clients,
             config.deployment.id,
             resolver.clone(),
             saga_request,
@@ -422,8 +415,6 @@ impl Nexus {
                 .deployment
                 .external_dns_servers
                 .clone(),
-            dpd_clients,
-            mg_clients,
             background_tasks,
             default_region_allocation_strategy: config
                 .pkg
@@ -875,6 +866,81 @@ impl Nexus {
             }
         }
     }
+
+    pub(crate) async fn dpd_clients(
+        &self,
+    ) -> Result<HashMap<SwitchLocation, dpd_client::Client>, String> {
+        let mappings = self.switch_zone_address_mappings().await?;
+        let clients: HashMap<SwitchLocation, dpd_client::Client> = mappings
+            .iter()
+            .map(|(location, addr)| {
+                let port = DENDRITE_PORT;
+
+                let client_state = dpd_client::ClientState {
+                    tag: String::from("nexus"),
+                    log: self.log.new(o!(
+                        "component" => "DpdClient"
+                    )),
+                };
+
+                let dpd_client = dpd_client::Client::new(
+                    &format!("http://[{addr}]:{port}"),
+                    client_state,
+                );
+                (*location, dpd_client)
+            })
+            .collect();
+        Ok(clients)
+    }
+
+    pub(crate) async fn mg_clients(
+        &self,
+    ) -> Result<HashMap<SwitchLocation, mg_admin_client::Client>, String> {
+        let mappings = self.switch_zone_address_mappings().await?;
+        let mut clients: Vec<(SwitchLocation, mg_admin_client::Client)> =
+            vec![];
+        for (location, addr) in &mappings {
+            let port = MGD_PORT;
+            let socketaddr =
+                std::net::SocketAddr::V6(SocketAddrV6::new(*addr, port, 0, 0));
+            let client = match mg_admin_client::Client::new(
+                &self.log.clone(),
+                socketaddr,
+            ) {
+                Ok(client) => client,
+                Err(e) => {
+                    error!(
+                        self.log,
+                        "error building mgd client";
+                        "location" => %location,
+                        "addr" => %addr,
+                        "error" => %e,
+                    );
+                    continue;
+                }
+            };
+            clients.push((*location, client));
+        }
+        Ok(clients.into_iter().collect::<HashMap<_, _>>())
+    }
+
+    async fn switch_zone_address_mappings(
+        &self,
+    ) -> Result<HashMap<SwitchLocation, Ipv6Addr>, String> {
+        let switch_zone_addresses = match self
+            .resolver()
+            .await
+            .lookup_all_ipv6(ServiceName::Dendrite)
+            .await
+        {
+            Ok(addrs) => addrs,
+            Err(e) => {
+                error!(self.log, "failed to resolve addresses for Dendrite services"; "error" => %e);
+                return Err(e.to_string());
+            }
+        };
+        Ok(map_switch_zone_addrs(&self.log, switch_zone_addresses).await)
+    }
 }
 
 /// For unimplemented endpoints, indicates whether the resource identified
diff --git a/nexus/src/app/probe.rs b/nexus/src/app/probe.rs
index 0fce9d3431..e85c040a28 100644
--- a/nexus/src/app/probe.rs
+++ b/nexus/src/app/probe.rs
@@ -72,11 +72,18 @@ impl super::Nexus {
             self.boundary_switches(&self.opctx_alloc).await?;
 
         for switch in &boundary_switches {
-            let dpd_client = self.dpd_clients.get(switch).ok_or_else(|| {
+            let dpd_clients = self.dpd_clients().await.map_err(|e| {
+                Error::internal_error(&format!(
+                    "failed to get dpd_clients: {e}"
+                ))
+            })?;
+
+            let dpd_client = dpd_clients.get(switch).ok_or_else(|| {
                 Error::internal_error(&format!(
                     "could not find dpd client for {switch}"
                 ))
             })?;
+
             self.probe_ensure_dpd_config(
                 opctx,
                 probe.id(),
diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs
index 35f3337625..4a4a61142e 100644
--- a/nexus/src/app/rack.rs
+++ b/nexus/src/app/rack.rs
@@ -12,7 +12,7 @@ use gateway_client::types::SpType;
 use ipnetwork::{IpNetwork, Ipv6Network};
 use nexus_db_model::DnsGroup;
 use nexus_db_model::InitialDnsGroup;
-use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed};
+use nexus_db_model::INFRA_LOT;
 use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db;
@@ -54,19 +54,14 @@ use omicron_common::api::external::Name;
 use omicron_common::api::external::NameOrId;
 use omicron_common::api::internal::shared::ExternalPortDiscovery;
 use sled_agent_client::types::AddSledRequest;
-use sled_agent_client::types::EarlyNetworkConfigBody;
 use sled_agent_client::types::StartSledAgentRequest;
 use sled_agent_client::types::StartSledAgentRequestBody;
-use sled_agent_client::types::{
-    BgpConfig, BgpPeerConfig as SledBgpPeerConfig, EarlyNetworkConfig,
-    PortConfigV1, RackNetworkConfigV1, RouteConfig as SledRouteConfig,
-};
+
 use slog_error_chain::InlineErrorChain;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::collections::HashMap;
 use std::net::IpAddr;
-use std::net::Ipv4Addr;
 use std::num::NonZeroU32;
 use std::str::FromStr;
 use uuid::Uuid;
@@ -222,47 +217,6 @@ impl super::Nexus {
 
         let rack_network_config = &request.rack_network_config;
 
-        self.db_datastore
-            .rack_set_initialized(
-                opctx,
-                RackInit {
-                    rack_subnet: rack_network_config.rack_subnet.into(),
-                    rack_id,
-                    blueprint,
-                    services: request.services,
-                    datasets,
-                    service_ip_pool_ranges,
-                    internal_dns,
-                    external_dns,
-                    recovery_silo,
-                    recovery_silo_fq_dns_name,
-                    recovery_user_id: request.recovery_silo.user_name,
-                    recovery_user_password_hash: request
-                        .recovery_silo
-                        .user_password_hash
-                        .into(),
-                    dns_update,
-                },
-            )
-            .await?;
-
-        // Plumb the firewall rules for the built-in services
-        self.plumb_service_firewall_rules(opctx, &[]).await?;
-
-        // We've potentially updated the list of DNS servers and the DNS
-        // configuration for both internal and external DNS, plus the Silo
-        // certificates.  Activate the relevant background tasks.
-        for task in &[
-            &self.background_tasks.task_internal_dns_config,
-            &self.background_tasks.task_internal_dns_servers,
-            &self.background_tasks.task_external_dns_config,
-            &self.background_tasks.task_external_dns_servers,
-            &self.background_tasks.task_external_endpoints,
-            &self.background_tasks.task_inventory_collection,
-        ] {
-            self.background_tasks.activate(task);
-        }
-
         // TODO - https://github.com/oxidecomputer/omicron/pull/3359
         // register all switches found during rack initialization
         // identify requested switch from config and associate
@@ -342,12 +296,11 @@ impl super::Nexus {
         // going forward via self.run_saga()? Note that self.create_runnable_saga and
         // self.execute_saga are currently not available within this scope.
         info!(self.log, "Recording Rack Network Configuration");
-        let address_lot_name =
-            Name::from_str("initial-infra").map_err(|e| {
-                Error::internal_error(&format!(
-                    "unable to use `initial-infra` as `Name`: {e}"
-                ))
-            })?;
+        let address_lot_name = Name::from_str(INFRA_LOT).map_err(|e| {
+            Error::internal_error(&format!(
+                "unable to use `initial-infra` as `Name`: {e}"
+            ))
+        })?;
         let identity = IdentityMetadataCreateParams {
             name: address_lot_name.clone(),
             description: "initial infrastructure ip address lot".to_string(),
@@ -391,7 +344,8 @@ impl super::Nexus {
             let address_lot_name: Name =
                 format!("as{}-lot", bgp_config.asn).parse().unwrap();
 
-            self.db_datastore
+            match self
+                .db_datastore
                 .address_lot_create(
                     &opctx,
                     &AddressLotCreate {
@@ -414,14 +368,19 @@ impl super::Nexus {
                     },
                 )
                 .await
-                .map_err(|e| {
-                    Error::internal_error(&format!(
-                        "unable to create address lot for BGP as {}: {}",
-                        bgp_config.asn, e
-                    ))
-                })?;
+            {
+                Ok(_) => Ok(()),
+                Err(e) => match e {
+                    Error::ObjectAlreadyExists { .. } => Ok(()),
+                    _ => Err(Error::internal_error(&format!(
+                        "unable to create address lot for BGP as {}: {e}",
+                        bgp_config.asn
+                    ))),
+                },
+            }?;
 
-            self.db_datastore
+            match self
+                .db_datastore
                 .bgp_create_announce_set(
                     &opctx,
                     &BgpAnnounceSetCreate {
@@ -447,14 +406,19 @@ impl super::Nexus {
                     },
                 )
                 .await
-                .map_err(|e| {
-                    Error::internal_error(&format!(
-                        "unable to create bgp announce set for as {}: {}",
-                        bgp_config.asn, e
-                    ))
-                })?;
+            {
+                Ok(_) => Ok(()),
+                Err(e) => match e {
+                    Error::ObjectAlreadyExists { .. } => Ok(()),
+                    _ => Err(Error::internal_error(&format!(
+                        "unable to create bgp announce set for as {}: {e}",
+                        bgp_config.asn
+                    ))),
+                },
+            }?;
 
-            self.db_datastore
+            match self
+                .db_datastore
                 .bgp_config_set(
                     &opctx,
                     &BgpConfigCreate {
@@ -471,12 +435,16 @@ impl super::Nexus {
                     },
                 )
                 .await
-                .map_err(|e| {
-                    Error::internal_error(&format!(
-                        "unable to set bgp config for as {}: {}",
-                        bgp_config.asn, e
-                    ))
-                })?;
+            {
+                Ok(_) => Ok(()),
+                Err(e) => match e {
+                    Error::ObjectAlreadyExists { .. } => Ok(()),
+                    _ => Err(Error::internal_error(&format!(
+                        "unable to set bgp config for as {}: {e}",
+                        bgp_config.asn
+                    ))),
+                },
+            }?;
         }
 
         for (idx, uplink_config) in rack_network_config.ports.iter().enumerate()
@@ -615,6 +583,47 @@ impl super::Nexus {
 
         self.initial_bootstore_sync(&opctx).await?;
 
+        self.db_datastore
+            .rack_set_initialized(
+                opctx,
+                RackInit {
+                    rack_subnet: rack_network_config.rack_subnet.into(),
+                    rack_id,
+                    blueprint,
+                    services: request.services,
+                    datasets,
+                    service_ip_pool_ranges,
+                    internal_dns,
+                    external_dns,
+                    recovery_silo,
+                    recovery_silo_fq_dns_name,
+                    recovery_user_id: request.recovery_silo.user_name,
+                    recovery_user_password_hash: request
+                        .recovery_silo
+                        .user_password_hash
+                        .into(),
+                    dns_update,
+                },
+            )
+            .await?;
+
+        // Plumb the firewall rules for the built-in services
+        self.plumb_service_firewall_rules(opctx, &[]).await?;
+
+        // We've potentially updated the list of DNS servers and the DNS
+        // configuration for both internal and external DNS, plus the Silo
+        // certificates.  Activate the relevant background tasks.
+        for task in &[
+            &self.background_tasks.task_internal_dns_config,
+            &self.background_tasks.task_internal_dns_servers,
+            &self.background_tasks.task_external_dns_config,
+            &self.background_tasks.task_external_dns_servers,
+            &self.background_tasks.task_external_endpoints,
+            &self.background_tasks.task_inventory_collection,
+        ] {
+            self.background_tasks.activate(task);
+        }
+
         Ok(())
     }
 
@@ -673,115 +682,6 @@ impl super::Nexus {
         Ok(())
     }
 
-    pub(crate) async fn bootstore_network_config(
-        &self,
-        opctx: &OpContext,
-    ) -> Result<EarlyNetworkConfig, Error> {
-        let rack = self.rack_lookup(opctx, &self.rack_id).await?;
-        let subnet = rack_subnet(rack.rack_subnet)?;
-
-        let db_ports = self.active_port_settings(opctx).await?;
-        let mut ports = Vec::new();
-        let mut bgp = Vec::new();
-        for (port, info) in &db_ports {
-            let mut peer_info = Vec::new();
-            for p in &info.bgp_peers {
-                let bgp_config =
-                    self.bgp_config_get(&opctx, p.bgp_config_id.into()).await?;
-                let announcements = self
-                    .bgp_announce_list(
-                        &opctx,
-                        &params::BgpAnnounceSetSelector {
-                            name_or_id: bgp_config.bgp_announce_set_id.into(),
-                        },
-                    )
-                    .await?;
-                let addr = match p.addr {
-                    ipnetwork::IpNetwork::V4(addr) => addr,
-                    ipnetwork::IpNetwork::V6(_) => continue, //TODO v6
-                };
-                peer_info.push((p, bgp_config.asn.0, addr.ip()));
-                bgp.push(BgpConfig {
-                    asn: bgp_config.asn.0,
-                    originate: announcements
-                        .iter()
-                        .filter_map(|a| match a.network {
-                            IpNetwork::V4(net) => Some(net.into()),
-                            //TODO v6
-                            _ => None,
-                        })
-                        .collect(),
-                });
-            }
-
-            let p = PortConfigV1 {
-                routes: info
-                    .routes
-                    .iter()
-                    .map(|r| SledRouteConfig {
-                        destination: r.dst,
-                        nexthop: r.gw.ip(),
-                    })
-                    .collect(),
-                addresses: info.addresses.iter().map(|a| a.address).collect(),
-                bgp_peers: peer_info
-                    .iter()
-                    .map(|(p, asn, addr)| SledBgpPeerConfig {
-                        addr: *addr,
-                        asn: *asn,
-                        port: port.port_name.clone(),
-                        hold_time: Some(p.hold_time.0.into()),
-                        connect_retry: Some(p.connect_retry.0.into()),
-                        delay_open: Some(p.delay_open.0.into()),
-                        idle_hold_time: Some(p.idle_hold_time.0.into()),
-                        keepalive: Some(p.keepalive.0.into()),
-                    })
-                    .collect(),
-                switch: port.switch_location.parse().unwrap(),
-                port: port.port_name.clone(),
-                uplink_port_fec: info
-                    .links
-                    .get(0) //TODO https://github.com/oxidecomputer/omicron/issues/3062
-                    .map(|l| l.fec)
-                    .unwrap_or(SwitchLinkFec::None)
-                    .into(),
-                uplink_port_speed: info
-                    .links
-                    .get(0) //TODO https://github.com/oxidecomputer/omicron/issues/3062
-                    .map(|l| l.speed)
-                    .unwrap_or(SwitchLinkSpeed::Speed100G)
-                    .into(),
-                autoneg: info
-                    .links
-                    .get(0) //TODO breakout support
-                    .map(|l| l.autoneg)
-                    .unwrap_or(false),
-            };
-
-            ports.push(p);
-        }
-
-        let result = EarlyNetworkConfig {
-            generation: 0,
-            schema_version: 1,
-            body: EarlyNetworkConfigBody {
-                ntp_servers: Vec::new(), //TODO
-                rack_network_config: Some(RackNetworkConfigV1 {
-                    rack_subnet: subnet,
-                    //TODO: We need to remove these. They are inconsistent with
-                    // a generic set of addresses on ports that may not be
-                    // contiguous.
-                    infra_ip_first: Ipv4Addr::UNSPECIFIED,
-                    infra_ip_last: Ipv4Addr::UNSPECIFIED,
-                    ports,
-                    bgp,
-                }),
-            },
-        };
-
-        Ok(result)
-    }
-
     /// Return the list of sleds that are inserted into an initialized rack
     /// but not yet initialized as part of a rack.
     //
diff --git a/nexus/src/app/sagas/loopback_address_create.rs b/nexus/src/app/sagas/loopback_address_create.rs
deleted file mode 100644
index c32a5f387d..0000000000
--- a/nexus/src/app/sagas/loopback_address_create.rs
+++ /dev/null
@@ -1,178 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-use super::{NexusActionContext, NEXUS_DPD_TAG};
-use crate::app::sagas::{
-    declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError,
-};
-use crate::external_api::params;
-use anyhow::Error;
-use nexus_db_queries::authn;
-use nexus_db_queries::authz;
-use nexus_db_queries::db::model::LoopbackAddress;
-use omicron_common::api::internal::shared::SwitchLocation;
-use omicron_common::retry_until_known_result;
-use serde::{Deserialize, Serialize};
-use std::sync::Arc;
-use steno::ActionError;
-
-#[derive(Debug, Deserialize, Serialize)]
-pub(crate) struct Params {
-    pub serialized_authn: authn::saga::Serialized,
-    pub loopback_address: params::LoopbackAddressCreate,
-}
-
-declare_saga_actions! {
-    loopback_address_create;
-    CREATE_LOOPBACK_ADDRESS_RECORD -> "created_loopback_address_record" {
-        + slc_loopback_address_create_record
-        - slc_loopback_address_delete_record
-    }
-    CREATE_LOOPBACK_ADDRESS -> "create_loopback_address" {
-        + slc_loopback_address_create
-    }
-}
-
-#[derive(Debug)]
-pub(crate) struct SagaLoopbackAddressCreate;
-impl NexusSaga for SagaLoopbackAddressCreate {
-    const NAME: &'static str = "loopback-address-create";
-    type Params = Params;
-
-    fn register_actions(registry: &mut ActionRegistry) {
-        loopback_address_create_register_actions(registry);
-    }
-
-    fn make_saga_dag(
-        _params: &Self::Params,
-        mut builder: steno::DagBuilder,
-    ) -> Result<steno::Dag, SagaInitError> {
-        builder.append(create_loopback_address_record_action());
-        builder.append(create_loopback_address_action());
-
-        Ok(builder.build()?)
-    }
-}
-
-async fn slc_loopback_address_create_record(
-    sagactx: NexusActionContext,
-) -> Result<LoopbackAddress, ActionError> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let address_lot_lookup = nexus
-        .address_lot_lookup(&opctx, params.loopback_address.address_lot.clone())
-        .map_err(ActionError::action_failed)?;
-    let (.., authz_address_lot) = address_lot_lookup
-        .lookup_for(authz::Action::CreateChild)
-        .await
-        .map_err(|e| ActionError::action_failed(e.to_string()))?;
-
-    // Just a check to make sure a valid rack id was passed in.
-    nexus
-        .rack_lookup(&opctx, &params.loopback_address.rack_id)
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    // If there is a failure down the road, this record will get cleaned up by
-    // the unwind action slc_loopback_address_delete_record. In the case that
-    // the saga is retried, we will just retry with a new id, returning that to
-    // the caller if the saga retry is successful. Having intermediate ids here
-    // is ok.
-    let value = nexus
-        .db_datastore
-        .loopback_address_create(
-            &opctx,
-            &params.loopback_address,
-            None,
-            &authz_address_lot,
-        )
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    Ok(value)
-}
-
-async fn slc_loopback_address_delete_record(
-    sagactx: NexusActionContext,
-) -> Result<(), Error> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let loopback_address_lookup = nexus.loopback_address_lookup(
-        &opctx,
-        params.loopback_address.rack_id,
-        params.loopback_address.switch_location.clone().into(),
-        params.loopback_address.address.into(),
-    )?;
-
-    let (.., authz_loopback_address) =
-        loopback_address_lookup.lookup_for(authz::Action::Delete).await?;
-
-    nexus
-        .db_datastore
-        .loopback_address_delete(&opctx, &authz_loopback_address)
-        .await?;
-
-    Ok(())
-}
-
-async fn slc_loopback_address_create(
-    sagactx: NexusActionContext,
-) -> Result<(), ActionError> {
-    let params = sagactx.saga_params::<Params>()?;
-    let log = sagactx.user_data().log();
-
-    let dpd_client: Arc<dpd_client::Client> =
-        select_dendrite_client(&sagactx).await?;
-
-    retry_until_known_result(log, || async {
-        dpd_client
-            .ensure_loopback_created(
-                log,
-                params.loopback_address.address,
-                NEXUS_DPD_TAG,
-            )
-            .await
-    })
-    .await
-    .map_err(|e| ActionError::action_failed(e.to_string()))
-}
-
-pub(crate) async fn select_dendrite_client(
-    sagactx: &NexusActionContext,
-) -> Result<Arc<dpd_client::Client>, ActionError> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-
-    let switch_location: SwitchLocation = params
-        .loopback_address
-        .switch_location
-        .as_str()
-        .parse()
-        .map_err(ActionError::action_failed)?;
-    let dpd_client: Arc<dpd_client::Client> = osagactx
-        .nexus()
-        .dpd_clients
-        .get(&switch_location)
-        .ok_or_else(|| {
-            ActionError::action_failed(format!(
-                "requested switch not available: {switch_location}"
-            ))
-        })?
-        .clone();
-    Ok(dpd_client)
-}
diff --git a/nexus/src/app/sagas/loopback_address_delete.rs b/nexus/src/app/sagas/loopback_address_delete.rs
deleted file mode 100644
index 822a360acf..0000000000
--- a/nexus/src/app/sagas/loopback_address_delete.rs
+++ /dev/null
@@ -1,190 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-use super::NexusActionContext;
-use crate::app::sagas::{
-    declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError,
-};
-use crate::external_api::params;
-use anyhow::{anyhow, Error};
-use nexus_db_queries::authn;
-use nexus_db_queries::authz;
-use nexus_db_queries::db::model::{LoopbackAddress, Name};
-use nexus_types::identity::Asset;
-use omicron_common::api::external::{IpNet, NameOrId};
-use omicron_common::retry_until_known_result;
-use serde::{Deserialize, Serialize};
-use std::sync::Arc;
-use steno::ActionError;
-use uuid::Uuid;
-
-#[derive(Debug, Deserialize, Serialize)]
-pub(crate) struct Params {
-    pub serialized_authn: authn::saga::Serialized,
-    pub rack_id: Uuid,
-    pub switch_location: Name,
-    pub address: IpNet,
-}
-
-declare_saga_actions! {
-    loopback_address_delete;
-    DELETE_LOOPBACK_ADDRESS_RECORD -> "deleted_loopback_address_record" {
-        + slc_loopback_address_delete_record
-        - slc_loopback_address_undelete_record
-    }
-    DELETE_LOOPBACK_ADDRESS -> "delete_loopback_address" {
-        + slc_loopback_address_delete
-    }
-}
-
-#[derive(Debug)]
-pub(crate) struct SagaLoopbackAddressDelete;
-impl NexusSaga for SagaLoopbackAddressDelete {
-    const NAME: &'static str = "loopback-address-delete";
-    type Params = Params;
-
-    fn register_actions(registry: &mut ActionRegistry) {
-        loopback_address_delete_register_actions(registry);
-    }
-
-    fn make_saga_dag(
-        _params: &Self::Params,
-        mut builder: steno::DagBuilder,
-    ) -> Result<steno::Dag, SagaInitError> {
-        builder.append(delete_loopback_address_record_action());
-        builder.append(delete_loopback_address_action());
-
-        Ok(builder.build()?)
-    }
-}
-
-async fn slc_loopback_address_delete_record(
-    sagactx: NexusActionContext,
-) -> Result<LoopbackAddress, ActionError> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let loopback_address_lookup = nexus
-        .loopback_address_lookup(
-            &opctx,
-            params.rack_id,
-            params.switch_location,
-            params.address,
-        )
-        .map_err(ActionError::action_failed)?;
-
-    let (.., authz_loopback_address) = loopback_address_lookup
-        .lookup_for(authz::Action::Delete)
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    let value = nexus
-        .db_datastore
-        .loopback_address_get(&opctx, &authz_loopback_address)
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    nexus
-        .db_datastore
-        .loopback_address_delete(&opctx, &authz_loopback_address)
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    Ok(value)
-}
-
-async fn slc_loopback_address_undelete_record(
-    sagactx: NexusActionContext,
-) -> Result<(), Error> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let value =
-        sagactx.lookup::<LoopbackAddress>("deleted_loopback_address_record")?;
-
-    let address_lot_id = nexus
-        .db_datastore
-        .address_lot_id_for_block_id(&opctx, value.address_lot_block_id)
-        .await?;
-
-    let arg = params::LoopbackAddressCreate {
-        address_lot: NameOrId::Id(address_lot_id),
-        rack_id: value.rack_id,
-        switch_location: value
-            .switch_location
-            .parse()
-            .map_err(|e| anyhow!("bad switch location name: {}", e))?,
-        address: value.address.ip(),
-        mask: value.address.prefix(),
-        anycast: value.anycast,
-    };
-
-    let address_lot_lookup = nexus
-        .address_lot_lookup(&opctx, arg.address_lot.clone())
-        .map_err(ActionError::action_failed)?;
-    let (.., authz_address_lot) = address_lot_lookup
-        .lookup_for(authz::Action::Modify)
-        .await
-        .map_err(|e| ActionError::action_failed(e.to_string()))?;
-
-    // Just a check to make sure a valid rack id was passed in.
-    nexus
-        .rack_lookup(&opctx, &arg.rack_id)
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    nexus
-        .db_datastore
-        .loopback_address_create(
-            &opctx,
-            &arg,
-            Some(value.id()),
-            &authz_address_lot,
-        )
-        .await?;
-
-    Ok(())
-}
-
-async fn slc_loopback_address_delete(
-    sagactx: NexusActionContext,
-) -> Result<(), ActionError> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let log = sagactx.user_data().log();
-    let switch = &params
-        .switch_location
-        .as_str()
-        .parse()
-        .map_err(|e| ActionError::action_failed(format!("{e:#?}")))?;
-
-    let dpd_client: Arc<dpd_client::Client> = osagactx
-        .nexus()
-        .dpd_clients
-        .get(&switch)
-        .ok_or_else(|| {
-            ActionError::action_failed(format!(
-                "unable to retrieve dendrite client for {switch}"
-            ))
-        })?
-        .clone();
-
-    retry_until_known_result(log, || async {
-        dpd_client.ensure_loopback_deleted(log, params.address.ip()).await
-    })
-    .await
-    .map_err(|e| ActionError::action_failed(e.to_string()))
-}
diff --git a/nexus/src/app/sagas/mod.rs b/nexus/src/app/sagas/mod.rs
index 01b01c4571..e725c1f093 100644
--- a/nexus/src/app/sagas/mod.rs
+++ b/nexus/src/app/sagas/mod.rs
@@ -31,14 +31,9 @@ pub mod instance_ip_attach;
 pub mod instance_ip_detach;
 pub mod instance_migrate;
 pub mod instance_start;
-pub mod loopback_address_create;
-pub mod loopback_address_delete;
 pub mod project_create;
 pub mod snapshot_create;
 pub mod snapshot_delete;
-pub mod switch_port_settings_apply;
-pub mod switch_port_settings_clear;
-pub mod switch_port_settings_common;
 pub mod test_saga;
 pub mod volume_delete;
 pub mod volume_remove_rop;
@@ -145,20 +140,6 @@ fn make_action_registry() -> ActionRegistry {
     <instance_start::SagaInstanceStart as NexusSaga>::register_actions(
         &mut registry,
     );
-    <loopback_address_create::SagaLoopbackAddressCreate
-        as NexusSaga>::register_actions(
-        &mut registry,
-    );
-    <loopback_address_delete::SagaLoopbackAddressDelete
-        as NexusSaga>::register_actions(
-        &mut registry,
-    );
-    <switch_port_settings_apply::SagaSwitchPortSettingsApply as NexusSaga>::register_actions(
-        &mut registry,
-    );
-    <switch_port_settings_clear::SagaSwitchPortSettingsClear as NexusSaga>::register_actions(
-        &mut registry,
-    );
     <project_create::SagaProjectCreate as NexusSaga>::register_actions(
         &mut registry,
     );
@@ -321,8 +302,6 @@ macro_rules! declare_saga_actions {
     };
 }
 
-use omicron_common::OMICRON_DPD_TAG as NEXUS_DPD_TAG;
-
 pub(crate) use __action_name;
 pub(crate) use __emit_action;
 pub(crate) use __stringify_ident;
diff --git a/nexus/src/app/sagas/switch_port_settings_apply.rs b/nexus/src/app/sagas/switch_port_settings_apply.rs
deleted file mode 100644
index 44f2f77ea1..0000000000
--- a/nexus/src/app/sagas/switch_port_settings_apply.rs
+++ /dev/null
@@ -1,586 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-use super::{NexusActionContext, NEXUS_DPD_TAG};
-use crate::app::sagas::switch_port_settings_common::{
-    api_to_dpd_port_settings, ensure_switch_port_bgp_settings,
-    ensure_switch_port_uplink, select_dendrite_client, select_mg_client,
-    switch_sled_agent, write_bootstore_config,
-};
-use crate::app::sagas::{
-    declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError,
-};
-use anyhow::Error;
-use db::datastore::SwitchPortSettingsCombinedResult;
-use dpd_client::types::PortId;
-use mg_admin_client::types::{
-    AddStaticRoute4Request, DeleteStaticRoute4Request, Prefix4, StaticRoute4,
-    StaticRoute4List,
-};
-use nexus_db_model::NETWORK_KEY;
-use nexus_db_queries::db::datastore::UpdatePrecondition;
-use nexus_db_queries::{authn, db};
-use omicron_common::api::external::{self, NameOrId};
-use omicron_common::api::internal::shared::SwitchLocation;
-use omicron_common::retry_until_known_result;
-use serde::{Deserialize, Serialize};
-use std::net::IpAddr;
-use std::str::FromStr;
-use std::sync::Arc;
-use steno::ActionError;
-use uuid::Uuid;
-
-// switch port settings apply saga: input parameters
-
-#[derive(Debug, Deserialize, Serialize)]
-pub(crate) struct Params {
-    pub serialized_authn: authn::saga::Serialized,
-    pub switch_port_id: Uuid,
-    pub switch_port_settings_id: Uuid,
-    pub switch_port_name: String,
-}
-
-// switch port settings apply: actions
-
-declare_saga_actions! {
-    switch_port_settings_apply;
-    ASSOCIATE_SWITCH_PORT -> "original_switch_port_settings_id" {
-        + spa_associate_switch_port
-        - spa_disassociate_switch_port
-    }
-    GET_SWITCH_PORT_SETTINGS -> "switch_port_settings" {
-        + spa_get_switch_port_settings
-    }
-    ENSURE_SWITCH_PORT_SETTINGS -> "ensure_switch_port_settings" {
-        + spa_ensure_switch_port_settings
-        - spa_undo_ensure_switch_port_settings
-    }
-    ENSURE_SWITCH_PORT_UPLINK -> "ensure_switch_port_uplink" {
-        + spa_ensure_switch_port_uplink
-        - spa_undo_ensure_switch_port_uplink
-    }
-    ENSURE_SWITCH_ROUTES -> "ensure_switch_routes" {
-        + spa_ensure_switch_routes
-        - spa_undo_ensure_switch_routes
-    }
-    ENSURE_SWITCH_PORT_BGP_SETTINGS -> "ensure_switch_port_bgp_settings" {
-        + spa_ensure_switch_port_bgp_settings
-        - spa_undo_ensure_switch_port_bgp_settings
-    }
-    ENSURE_SWITCH_PORT_BOOTSTORE_NETWORK_SETTINGS -> "ensure_switch_port_bootstore_network_settings" {
-        + spa_ensure_switch_port_bootstore_network_settings
-        - spa_undo_ensure_switch_port_bootstore_network_settings
-    }
-}
-
-// switch port settings apply saga: definition
-
-#[derive(Debug)]
-pub(crate) struct SagaSwitchPortSettingsApply;
-
-impl NexusSaga for SagaSwitchPortSettingsApply {
-    const NAME: &'static str = "switch-port-settings-apply";
-    type Params = Params;
-
-    fn register_actions(registry: &mut ActionRegistry) {
-        switch_port_settings_apply_register_actions(registry);
-    }
-
-    fn make_saga_dag(
-        _params: &Self::Params,
-        mut builder: steno::DagBuilder,
-    ) -> Result<steno::Dag, SagaInitError> {
-        builder.append(associate_switch_port_action());
-        builder.append(get_switch_port_settings_action());
-        builder.append(ensure_switch_port_settings_action());
-        builder.append(ensure_switch_port_uplink_action());
-        builder.append(ensure_switch_routes_action());
-        builder.append(ensure_switch_port_bgp_settings_action());
-        builder.append(ensure_switch_port_bootstore_network_settings_action());
-        Ok(builder.build()?)
-    }
-}
-
-async fn spa_associate_switch_port(
-    sagactx: NexusActionContext,
-) -> Result<Option<Uuid>, ActionError> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    // first get the current association so we fall back to this on failure
-    let port =
-        nexus.get_switch_port(&opctx, params.switch_port_id).await.map_err(
-            |e| ActionError::action_failed(format!("get switch port: {e}")),
-        )?;
-
-    // update the switch port settings association
-    nexus
-        .set_switch_port_settings_id(
-            &opctx,
-            params.switch_port_id,
-            Some(params.switch_port_settings_id),
-            UpdatePrecondition::DontCare,
-        )
-        .await
-        .map_err(|e| {
-            ActionError::action_failed(format!(
-                "set switch port settings id {e}"
-            ))
-        })?;
-
-    Ok(port.port_settings_id)
-}
-
-async fn spa_get_switch_port_settings(
-    sagactx: NexusActionContext,
-) -> Result<SwitchPortSettingsCombinedResult, ActionError> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let port_settings = nexus
-        .switch_port_settings_get(
-            &opctx,
-            &NameOrId::Id(params.switch_port_settings_id),
-        )
-        .await
-        .map_err(|e| {
-            ActionError::action_failed(format!("get switch port settings: {e}"))
-        })?;
-
-    Ok(port_settings)
-}
-
-async fn spa_ensure_switch_port_settings(
-    sagactx: NexusActionContext,
-) -> Result<(), ActionError> {
-    let params = sagactx.saga_params::<Params>()?;
-    let log = sagactx.user_data().log();
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let settings = sagactx
-        .lookup::<SwitchPortSettingsCombinedResult>("switch_port_settings")?;
-
-    let port_id: PortId =
-        PortId::from_str(&params.switch_port_name).map_err(|e| {
-            ActionError::action_failed(format!("parse port id: {e}"))
-        })?;
-
-    let dpd_client: Arc<dpd_client::Client> =
-        select_dendrite_client(&sagactx, &opctx, params.switch_port_id).await?;
-
-    let dpd_port_settings =
-        api_to_dpd_port_settings(&settings).map_err(|e| {
-            ActionError::action_failed(format!(
-                "translate api port settings to dpd port settings: {e}",
-            ))
-        })?;
-
-    retry_until_known_result(log, || async {
-        dpd_client
-            .port_settings_apply(
-                &port_id,
-                Some(NEXUS_DPD_TAG),
-                &dpd_port_settings,
-            )
-            .await
-    })
-    .await
-    .map_err(|e| match e {
-        progenitor_client::Error::ErrorResponse(ref er) => {
-            if er.status().is_client_error() {
-                ActionError::action_failed(format!(
-                    "bad request: dpd port settings apply {}",
-                    er.message,
-                ))
-            } else {
-                ActionError::action_failed(format!(
-                    "dpd port settings apply {e}"
-                ))
-            }
-        }
-        _ => ActionError::action_failed(format!("dpd port settings apply {e}")),
-    })?;
-
-    Ok(())
-}
-
-async fn spa_ensure_switch_routes(
-    sagactx: NexusActionContext,
-) -> Result<(), ActionError> {
-    let params = sagactx.saga_params::<Params>()?;
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let settings = sagactx
-        .lookup::<SwitchPortSettingsCombinedResult>("switch_port_settings")?;
-
-    let mut rq = AddStaticRoute4Request {
-        routes: StaticRoute4List { list: Vec::new() },
-    };
-    for r in settings.routes {
-        let nexthop = match r.gw.ip() {
-            IpAddr::V4(v4) => v4,
-            IpAddr::V6(_) => continue,
-        };
-        let prefix = match r.dst.ip() {
-            IpAddr::V4(v4) => Prefix4 { value: v4, length: r.dst.prefix() },
-            IpAddr::V6(_) => continue,
-        };
-        let sr = StaticRoute4 { nexthop, prefix };
-        rq.routes.list.push(sr);
-    }
-
-    let mg_client: Arc<mg_admin_client::Client> =
-        select_mg_client(&sagactx, &opctx, params.switch_port_id).await?;
-
-    mg_client.inner.static_add_v4_route(&rq).await.map_err(|e| {
-        ActionError::action_failed(format!("mgd static route add {e}"))
-    })?;
-
-    Ok(())
-}
-
-async fn spa_undo_ensure_switch_routes(
-    sagactx: NexusActionContext,
-) -> Result<(), Error> {
-    let params = sagactx.saga_params::<Params>()?;
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-    let settings = sagactx
-        .lookup::<SwitchPortSettingsCombinedResult>("switch_port_settings")?;
-
-    let mut rq = DeleteStaticRoute4Request {
-        routes: StaticRoute4List { list: Vec::new() },
-    };
-
-    for r in settings.routes {
-        let nexthop = match r.gw.ip() {
-            IpAddr::V4(v4) => v4,
-            IpAddr::V6(_) => continue,
-        };
-        let prefix = match r.gw.ip() {
-            IpAddr::V4(v4) => Prefix4 { value: v4, length: r.gw.prefix() },
-            IpAddr::V6(_) => continue,
-        };
-        let sr = StaticRoute4 { nexthop, prefix };
-        rq.routes.list.push(sr);
-    }
-
-    let mg_client: Arc<mg_admin_client::Client> =
-        select_mg_client(&sagactx, &opctx, params.switch_port_id).await?;
-
-    mg_client.inner.static_remove_v4_route(&rq).await.map_err(|e| {
-        ActionError::action_failed(format!("mgd static route remove {e}"))
-    })?;
-
-    Ok(())
-}
-
-async fn spa_undo_ensure_switch_port_settings(
-    sagactx: NexusActionContext,
-) -> Result<(), Error> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-    let log = sagactx.user_data().log();
-
-    let port_id: PortId = PortId::from_str(&params.switch_port_name)
-        .map_err(|e| external::Error::internal_error(e.to_string().as_str()))?;
-
-    let orig_port_settings_id = sagactx
-        .lookup::<Option<Uuid>>("original_switch_port_settings_id")
-        .map_err(|e| external::Error::internal_error(&e.to_string()))?;
-
-    let dpd_client =
-        select_dendrite_client(&sagactx, &opctx, params.switch_port_id).await?;
-
-    let id = match orig_port_settings_id {
-        Some(id) => id,
-        None => {
-            retry_until_known_result(log, || async {
-                dpd_client
-                    .port_settings_clear(&port_id, Some(NEXUS_DPD_TAG))
-                    .await
-            })
-            .await
-            .map_err(|e| external::Error::internal_error(&e.to_string()))?;
-
-            return Ok(());
-        }
-    };
-
-    let settings = nexus
-        .switch_port_settings_get(&opctx, &NameOrId::Id(id))
-        .await
-        .map_err(|e| {
-            ActionError::action_failed(format!("switch port settings get: {e}"))
-        })?;
-
-    let dpd_port_settings =
-        api_to_dpd_port_settings(&settings).map_err(|e| {
-            ActionError::action_failed(format!(
-                "translate api to dpd port settings {e}"
-            ))
-        })?;
-
-    retry_until_known_result(log, || async {
-        dpd_client
-            .port_settings_apply(
-                &port_id,
-                Some(NEXUS_DPD_TAG),
-                &dpd_port_settings,
-            )
-            .await
-    })
-    .await
-    .map_err(|e| external::Error::internal_error(&e.to_string()))?;
-
-    Ok(())
-}
-
-async fn spa_undo_ensure_switch_port_bgp_settings(
-    sagactx: NexusActionContext,
-) -> Result<(), Error> {
-    use mg_admin_client::types::DeleteNeighborRequest;
-
-    let osagactx = sagactx.user_data();
-    let nexus = osagactx.nexus();
-    let params = sagactx.saga_params::<Params>()?;
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let settings = sagactx
-        .lookup::<SwitchPortSettingsCombinedResult>("switch_port_settings")
-        .map_err(|e| {
-            ActionError::action_failed(format!(
-                "lookup switch port settings (bgp undo): {e}"
-            ))
-        })?;
-
-    let mg_client: Arc<mg_admin_client::Client> =
-        select_mg_client(&sagactx, &opctx, params.switch_port_id)
-            .await
-            .map_err(|e| {
-                ActionError::action_failed(format!(
-                    "select mg client (undo): {e}"
-                ))
-            })?;
-
-    for peer in settings.bgp_peers {
-        let config = nexus
-            .bgp_config_get(&opctx, peer.bgp_config_id.into())
-            .await
-            .map_err(|e| {
-                ActionError::action_failed(format!("delete bgp config: {e}"))
-            })?;
-
-        mg_client
-            .inner
-            .delete_neighbor(&DeleteNeighborRequest {
-                asn: *config.asn,
-                addr: peer.addr.ip(),
-            })
-            .await
-            .map_err(|e| {
-                ActionError::action_failed(format!("delete neighbor: {e}"))
-            })?;
-    }
-
-    Ok(())
-}
-
-async fn spa_ensure_switch_port_bootstore_network_settings(
-    sagactx: NexusActionContext,
-) -> Result<(), ActionError> {
-    let osagactx = sagactx.user_data();
-    let nexus = osagactx.nexus();
-    let params = sagactx.saga_params::<Params>()?;
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    // Just choosing the sled agent associated with switch0 for no reason.
-    let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?;
-
-    let mut config =
-        nexus.bootstore_network_config(&opctx).await.map_err(|e| {
-            ActionError::action_failed(format!(
-                "read nexus bootstore network config: {e}"
-            ))
-        })?;
-
-    let generation = nexus
-        .datastore()
-        .bump_bootstore_generation(&opctx, NETWORK_KEY.into())
-        .await
-        .map_err(|e| {
-            ActionError::action_failed(format!(
-                "bump bootstore network generation number: {e}"
-            ))
-        })?;
-
-    config.generation = generation as u64;
-    write_bootstore_config(&sa, &config).await?;
-
-    Ok(())
-}
-
-async fn spa_undo_ensure_switch_port_bootstore_network_settings(
-    sagactx: NexusActionContext,
-) -> Result<(), Error> {
-    // The overall saga update failed but the bootstore udpate succeeded.
-    // Between now and then other updates may have happened which prevent us
-    // from simply undoing the changes we did before, as we may inadvertently
-    // roll back changes at the intersection of this failed update and other
-    // succesful updates. The only thing we can really do here is attempt a
-    // complete update of the bootstore network settings based on the current
-    // state in the Nexus databse which, we assume to be consistent at any point
-    // in time.
-
-    let nexus = sagactx.user_data().nexus();
-    let params = sagactx.saga_params::<Params>()?;
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    // Just choosing the sled agent associated with switch0 for no reason.
-    let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?;
-
-    let config = nexus.bootstore_network_config(&opctx).await?;
-    write_bootstore_config(&sa, &config).await?;
-
-    Ok(())
-}
-
-async fn spa_ensure_switch_port_uplink(
-    sagactx: NexusActionContext,
-) -> Result<(), ActionError> {
-    let params = sagactx.saga_params::<Params>()?;
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-    ensure_switch_port_uplink(
-        sagactx,
-        &opctx,
-        false,
-        None,
-        params.switch_port_id,
-        params.switch_port_name,
-    )
-    .await
-}
-
-async fn spa_undo_ensure_switch_port_uplink(
-    sagactx: NexusActionContext,
-) -> Result<(), Error> {
-    let params = sagactx.saga_params::<Params>()?;
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-    Ok(ensure_switch_port_uplink(
-        sagactx,
-        &opctx,
-        true,
-        None,
-        params.switch_port_id,
-        params.switch_port_name,
-    )
-    .await?)
-}
-
-// a common route representation for dendrite and port settings
-#[derive(Debug, Clone, Eq, Hash, PartialEq, Serialize, Deserialize)]
-pub(crate) struct Route {
-    pub dst: IpAddr,
-    pub masklen: u8,
-    pub nexthop: Option<IpAddr>,
-}
-
-async fn spa_disassociate_switch_port(
-    sagactx: NexusActionContext,
-) -> Result<(), Error> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    // set the port settings id back to what it was before the saga started
-    let orig_port_settings_id =
-        sagactx.lookup::<Option<Uuid>>("original_switch_port_settings_id")?;
-
-    nexus
-        .set_switch_port_settings_id(
-            &opctx,
-            params.switch_port_id,
-            orig_port_settings_id,
-            UpdatePrecondition::Value(params.switch_port_settings_id),
-        )
-        .await
-        .map_err(|e| {
-            ActionError::action_failed(format!(
-                "set switch port settings id for disassociate: {e}"
-            ))
-        })?;
-
-    Ok(())
-}
-
-async fn spa_ensure_switch_port_bgp_settings(
-    sagactx: NexusActionContext,
-) -> Result<(), ActionError> {
-    let settings = sagactx
-        .lookup::<SwitchPortSettingsCombinedResult>("switch_port_settings")
-        .map_err(|e| {
-            ActionError::action_failed(format!(
-                "lookup switch port settings: {e}"
-            ))
-        })?;
-
-    let params = sagactx.saga_params::<Params>()?;
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    ensure_switch_port_bgp_settings(
-        sagactx,
-        &opctx,
-        settings,
-        params.switch_port_name.clone(),
-        params.switch_port_id,
-    )
-    .await
-}
diff --git a/nexus/src/app/sagas/switch_port_settings_clear.rs b/nexus/src/app/sagas/switch_port_settings_clear.rs
deleted file mode 100644
index 2e35530ef1..0000000000
--- a/nexus/src/app/sagas/switch_port_settings_clear.rs
+++ /dev/null
@@ -1,546 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-use super::{NexusActionContext, NEXUS_DPD_TAG};
-use crate::app::sagas::switch_port_settings_common::{
-    api_to_dpd_port_settings, apply_bootstore_update, bootstore_update,
-    ensure_switch_port_bgp_settings, ensure_switch_port_uplink,
-    read_bootstore_config, select_dendrite_client, select_mg_client,
-    switch_sled_agent, write_bootstore_config,
-};
-use crate::app::sagas::{
-    declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError,
-};
-use anyhow::Error;
-use dpd_client::types::PortId;
-use mg_admin_client::types::{
-    AddStaticRoute4Request, DeleteNeighborRequest, DeleteStaticRoute4Request,
-    Prefix4, StaticRoute4, StaticRoute4List,
-};
-use nexus_db_model::NETWORK_KEY;
-use nexus_db_queries::authn;
-use nexus_db_queries::db::datastore::UpdatePrecondition;
-use omicron_common::api::external::{self, NameOrId, SwitchLocation};
-use omicron_common::retry_until_known_result;
-use serde::{Deserialize, Serialize};
-use std::net::IpAddr;
-use std::str::FromStr;
-use std::sync::Arc;
-use steno::ActionError;
-use uuid::Uuid;
-
-#[derive(Debug, Deserialize, Serialize)]
-pub(crate) struct Params {
-    pub serialized_authn: authn::saga::Serialized,
-    pub switch_port_id: Uuid,
-    pub port_name: String,
-}
-
-declare_saga_actions! {
-    switch_port_settings_clear;
-    DISASSOCIATE_SWITCH_PORT -> "original_switch_port_settings" {
-        + spa_disassociate_switch_port
-        - spa_reassociate_switch_port
-    }
-    CLEAR_SWITCH_PORT_SETTINGS -> "switch_port_settings" {
-        + spa_clear_switch_port_settings
-        - spa_undo_clear_switch_port_settings
-    }
-    CLEAR_SWITCH_PORT_ROUTES -> "clear_switch_port_routes" {
-        + spa_clear_switch_port_routes
-        - spa_undo_clear_switch_port_routes
-    }
-    CLEAR_SWITCH_PORT_UPLINK -> "clear_switch_port_uplink" {
-        + spa_clear_switch_port_uplink
-        - spa_undo_clear_switch_port_uplink
-    }
-    CLEAR_SWITCH_PORT_BGP_SETTINGS -> "clear_switch_port_bgp_settings" {
-        + spa_clear_switch_port_bgp_settings
-        - spa_undo_clear_switch_port_bgp_settings
-    }
-    CLEAR_SWITCH_PORT_BOOTSTORE_NETWORK_SETTINGS -> "clear_switch_port_bootstore_network_settings" {
-        + spa_clear_switch_port_bootstore_network_settings
-        - spa_undo_clear_switch_port_bootstore_network_settings
-    }
-}
-
-#[derive(Debug)]
-pub(crate) struct SagaSwitchPortSettingsClear;
-impl NexusSaga for SagaSwitchPortSettingsClear {
-    const NAME: &'static str = "switch-port-settings-clear";
-    type Params = Params;
-
-    fn register_actions(registry: &mut ActionRegistry) {
-        switch_port_settings_clear_register_actions(registry);
-    }
-
-    fn make_saga_dag(
-        _params: &Self::Params,
-        mut builder: steno::DagBuilder,
-    ) -> Result<steno::Dag, SagaInitError> {
-        builder.append(disassociate_switch_port_action());
-        builder.append(clear_switch_port_settings_action());
-        builder.append(clear_switch_port_uplink_action());
-        builder.append(clear_switch_port_bgp_settings_action());
-        builder.append(clear_switch_port_bootstore_network_settings_action());
-        Ok(builder.build()?)
-    }
-}
-
-async fn spa_disassociate_switch_port(
-    sagactx: NexusActionContext,
-) -> Result<Option<Uuid>, ActionError> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    // first get the current association so we fall back to this on failure
-    let port = nexus
-        .get_switch_port(&opctx, params.switch_port_id)
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    // update the switch port settings association
-    nexus
-        .set_switch_port_settings_id(
-            &opctx,
-            params.switch_port_id,
-            None,
-            UpdatePrecondition::DontCare,
-        )
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    Ok(port.port_settings_id)
-}
-
-async fn spa_reassociate_switch_port(
-    sagactx: NexusActionContext,
-) -> Result<(), anyhow::Error> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    // set the port settings id back to what it was before the saga started
-    let orig_port_settings_id =
-        sagactx.lookup::<Option<Uuid>>("original_switch_port_settings")?;
-
-    nexus
-        .set_switch_port_settings_id(
-            &opctx,
-            params.switch_port_id,
-            orig_port_settings_id,
-            UpdatePrecondition::Null,
-        )
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    Ok(())
-}
-
-async fn spa_clear_switch_port_settings(
-    sagactx: NexusActionContext,
-) -> Result<(), ActionError> {
-    let params = sagactx.saga_params::<Params>()?;
-    let log = sagactx.user_data().log();
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let port_id: PortId = PortId::from_str(&params.port_name)
-        .map_err(|e| ActionError::action_failed(e.to_string()))?;
-
-    let dpd_client =
-        select_dendrite_client(&sagactx, &opctx, params.switch_port_id).await?;
-
-    retry_until_known_result(log, || async {
-        dpd_client.port_settings_clear(&port_id, Some(NEXUS_DPD_TAG)).await
-    })
-    .await
-    .map_err(|e| ActionError::action_failed(e.to_string()))?;
-
-    Ok(())
-}
-
-async fn spa_undo_clear_switch_port_settings(
-    sagactx: NexusActionContext,
-) -> Result<(), Error> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-    let log = sagactx.user_data().log();
-
-    let port_id: PortId = PortId::from_str(&params.port_name)
-        .map_err(|e| external::Error::internal_error(e.to_string().as_str()))?;
-
-    let orig_port_settings_id = sagactx
-        .lookup::<Option<Uuid>>("original_switch_port_settings_id")
-        .map_err(|e| external::Error::internal_error(&e.to_string()))?;
-
-    let id = match orig_port_settings_id {
-        Some(id) => id,
-        None => return Ok(()),
-    };
-
-    let settings = nexus
-        .switch_port_settings_get(&opctx, &NameOrId::Id(id))
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    let dpd_client =
-        select_dendrite_client(&sagactx, &opctx, params.switch_port_id).await?;
-
-    let dpd_port_settings = api_to_dpd_port_settings(&settings)
-        .map_err(ActionError::action_failed)?;
-
-    retry_until_known_result(log, || async {
-        dpd_client
-            .port_settings_apply(
-                &port_id,
-                Some(NEXUS_DPD_TAG),
-                &dpd_port_settings,
-            )
-            .await
-    })
-    .await
-    .map_err(|e| external::Error::internal_error(&e.to_string()))?;
-
-    Ok(())
-}
-
-async fn spa_clear_switch_port_uplink(
-    sagactx: NexusActionContext,
-) -> Result<(), ActionError> {
-    let params = sagactx.saga_params::<Params>()?;
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-    ensure_switch_port_uplink(
-        sagactx,
-        &opctx,
-        true,
-        None,
-        params.switch_port_id,
-        params.port_name.clone(),
-    )
-    .await
-}
-
-async fn spa_undo_clear_switch_port_uplink(
-    sagactx: NexusActionContext,
-) -> Result<(), Error> {
-    let id = sagactx
-        .lookup::<Option<Uuid>>("original_switch_port_settings_id")
-        .map_err(|e| external::Error::internal_error(&e.to_string()))?;
-    let params = sagactx.saga_params::<Params>()?;
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    Ok(ensure_switch_port_uplink(
-        sagactx,
-        &opctx,
-        false,
-        id,
-        params.switch_port_id,
-        params.port_name.clone(),
-    )
-    .await?)
-}
-
-async fn spa_clear_switch_port_bgp_settings(
-    sagactx: NexusActionContext,
-) -> Result<(), ActionError> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let orig_port_settings_id = sagactx
-        .lookup::<Option<Uuid>>("original_switch_port_settings_id")
-        .map_err(|e| {
-            ActionError::action_failed(format!(
-                "original port settings id lookup: {e}"
-            ))
-        })?;
-
-    let id = match orig_port_settings_id {
-        Some(id) => id,
-        None => return Ok(()),
-    };
-
-    let settings = nexus
-        .switch_port_settings_get(&opctx, &NameOrId::Id(id))
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    let mg_client: Arc<mg_admin_client::Client> =
-        select_mg_client(&sagactx, &opctx, params.switch_port_id)
-            .await
-            .map_err(|e| {
-                ActionError::action_failed(format!(
-                    "select mg client (undo): {e}"
-                ))
-            })?;
-
-    for peer in settings.bgp_peers {
-        let config = nexus
-            .bgp_config_get(&opctx, peer.bgp_config_id.into())
-            .await
-            .map_err(|e| {
-                ActionError::action_failed(format!("delete bgp config: {e}"))
-            })?;
-
-        mg_client
-            .inner
-            .delete_neighbor(&DeleteNeighborRequest {
-                asn: *config.asn,
-                addr: peer.addr.ip(),
-            })
-            .await
-            .map_err(|e| {
-                ActionError::action_failed(format!("delete neighbor: {e}"))
-            })?;
-    }
-
-    Ok(())
-}
-
-async fn spa_undo_clear_switch_port_bgp_settings(
-    sagactx: NexusActionContext,
-) -> Result<(), Error> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let orig_port_settings_id =
-        sagactx.lookup::<Option<Uuid>>("original_switch_port_settings_id")?;
-
-    let id = match orig_port_settings_id {
-        Some(id) => id,
-        None => return Ok(()),
-    };
-
-    let settings =
-        nexus.switch_port_settings_get(&opctx, &NameOrId::Id(id)).await?;
-
-    Ok(ensure_switch_port_bgp_settings(
-        sagactx,
-        &opctx,
-        settings,
-        params.port_name.clone(),
-        params.switch_port_id,
-    )
-    .await?)
-}
-
-async fn spa_clear_switch_port_routes(
-    sagactx: NexusActionContext,
-) -> Result<(), ActionError> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let orig_port_settings_id =
-        sagactx.lookup::<Option<Uuid>>("original_switch_port_settings_id")?;
-
-    let id = match orig_port_settings_id {
-        Some(id) => id,
-        None => return Ok(()),
-    };
-
-    let settings = nexus
-        .switch_port_settings_get(&opctx, &NameOrId::Id(id))
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    let mut rq = DeleteStaticRoute4Request {
-        routes: StaticRoute4List { list: Vec::new() },
-    };
-
-    for r in settings.routes {
-        let nexthop = match r.gw.ip() {
-            IpAddr::V4(v4) => v4,
-            IpAddr::V6(_) => continue,
-        };
-        let prefix = match r.gw.ip() {
-            IpAddr::V4(v4) => Prefix4 { value: v4, length: r.gw.prefix() },
-            IpAddr::V6(_) => continue,
-        };
-        let sr = StaticRoute4 { nexthop, prefix };
-        rq.routes.list.push(sr);
-    }
-
-    let mg_client: Arc<mg_admin_client::Client> =
-        select_mg_client(&sagactx, &opctx, params.switch_port_id).await?;
-
-    mg_client.inner.static_remove_v4_route(&rq).await.map_err(|e| {
-        ActionError::action_failed(format!("mgd static route remove {e}"))
-    })?;
-
-    Ok(())
-}
-
-async fn spa_undo_clear_switch_port_routes(
-    sagactx: NexusActionContext,
-) -> Result<(), Error> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let orig_port_settings_id =
-        sagactx.lookup::<Option<Uuid>>("original_switch_port_settings_id")?;
-
-    let id = match orig_port_settings_id {
-        Some(id) => id,
-        None => return Ok(()),
-    };
-
-    let settings = nexus
-        .switch_port_settings_get(&opctx, &NameOrId::Id(id))
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    let mut rq = AddStaticRoute4Request {
-        routes: StaticRoute4List { list: Vec::new() },
-    };
-
-    for r in settings.routes {
-        let nexthop = match r.gw.ip() {
-            IpAddr::V4(v4) => v4,
-            IpAddr::V6(_) => continue,
-        };
-        let prefix = match r.gw.ip() {
-            IpAddr::V4(v4) => Prefix4 { value: v4, length: r.gw.prefix() },
-            IpAddr::V6(_) => continue,
-        };
-        let sr = StaticRoute4 { nexthop, prefix };
-        rq.routes.list.push(sr);
-    }
-
-    let mg_client: Arc<mg_admin_client::Client> =
-        select_mg_client(&sagactx, &opctx, params.switch_port_id).await?;
-
-    mg_client.inner.static_add_v4_route(&rq).await.map_err(|e| {
-        ActionError::action_failed(format!("mgd static route remove {e}"))
-    })?;
-
-    Ok(())
-}
-
-async fn spa_clear_switch_port_bootstore_network_settings(
-    sagactx: NexusActionContext,
-) -> Result<(), ActionError> {
-    let nexus = sagactx.user_data().nexus();
-    let params = sagactx.saga_params::<Params>()?;
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    // Just choosing the sled agent associated with switch0 for no reason.
-    let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?;
-
-    let mut config =
-        nexus.bootstore_network_config(&opctx).await.map_err(|e| {
-            ActionError::action_failed(format!(
-                "read nexus bootstore network config: {e}"
-            ))
-        })?;
-
-    let generation = nexus
-        .datastore()
-        .bump_bootstore_generation(&opctx, NETWORK_KEY.into())
-        .await
-        .map_err(|e| {
-            ActionError::action_failed(format!(
-                "bump bootstore network generation number: {e}"
-            ))
-        })?;
-
-    config.generation = generation as u64;
-    write_bootstore_config(&sa, &config).await?;
-
-    Ok(())
-}
-
-async fn spa_undo_clear_switch_port_bootstore_network_settings(
-    sagactx: NexusActionContext,
-) -> Result<(), Error> {
-    let osagactx = sagactx.user_data();
-    let params = sagactx.saga_params::<Params>()?;
-    let nexus = osagactx.nexus();
-    let opctx = crate::context::op_context_for_saga_action(
-        &sagactx,
-        &params.serialized_authn,
-    );
-
-    let orig_port_settings_id = sagactx
-        .lookup::<Option<Uuid>>("original_switch_port_settings_id")
-        .map_err(|e| {
-            ActionError::action_failed(format!(
-                "original port settings id lookup: {e}"
-            ))
-        })?;
-
-    let id = match orig_port_settings_id {
-        Some(id) => id,
-        None => return Ok(()),
-    };
-
-    let settings = nexus
-        .switch_port_settings_get(&opctx, &NameOrId::Id(id))
-        .await
-        .map_err(ActionError::action_failed)?;
-
-    // Just choosing the sled agent associated with switch0 for no reason.
-    let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?;
-
-    // Read the current bootstore config, perform the update and write it back.
-    let mut config = read_bootstore_config(&sa).await?;
-    let update = bootstore_update(
-        &nexus,
-        &opctx,
-        params.switch_port_id,
-        &params.port_name,
-        &settings,
-    )
-    .await?;
-    apply_bootstore_update(&mut config, &update)?;
-    write_bootstore_config(&sa, &config).await?;
-
-    Ok(())
-}
diff --git a/nexus/src/app/sagas/switch_port_settings_common.rs b/nexus/src/app/sagas/switch_port_settings_common.rs
deleted file mode 100644
index 9c710d837d..0000000000
--- a/nexus/src/app/sagas/switch_port_settings_common.rs
+++ /dev/null
@@ -1,603 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-use super::NexusActionContext;
-use crate::app::map_switch_zone_addrs;
-use crate::Nexus;
-use db::datastore::SwitchPortSettingsCombinedResult;
-use dpd_client::types::{
-    LinkCreate, LinkId, LinkSettings, PortFec, PortSettings, PortSpeed,
-};
-use internal_dns::ServiceName;
-use ipnetwork::IpNetwork;
-use mg_admin_client::types::Prefix4;
-use mg_admin_client::types::{ApplyRequest, BgpPeerConfig};
-use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed};
-use nexus_db_queries::context::OpContext;
-use nexus_db_queries::db;
-use nexus_types::external_api::params;
-use omicron_common::address::SLED_AGENT_PORT;
-use omicron_common::api::external::NameOrId;
-use omicron_common::api::internal::shared::{
-    ParseSwitchLocationError, SwitchLocation,
-};
-use sled_agent_client::types::PortConfigV1;
-use sled_agent_client::types::RouteConfig;
-use sled_agent_client::types::{BgpConfig, EarlyNetworkConfig};
-use sled_agent_client::types::{
-    BgpPeerConfig as OmicronBgpPeerConfig, HostPortConfig,
-};
-use std::collections::HashMap;
-use std::net::SocketAddrV6;
-use std::net::{IpAddr, Ipv6Addr};
-use std::sync::Arc;
-use steno::ActionError;
-use uuid::Uuid;
-
-// This is more of an implementation detail of the BGP implementation. It
-// defines the maximum time the peering engine will wait for external messages
-// before breaking to check for shutdown conditions.
-const BGP_SESSION_RESOLUTION: u64 = 100;
-
-pub(crate) fn api_to_dpd_port_settings(
-    settings: &SwitchPortSettingsCombinedResult,
-) -> Result<PortSettings, String> {
-    let mut dpd_port_settings = PortSettings {
-        links: HashMap::new(),
-        v4_routes: HashMap::new(),
-        v6_routes: HashMap::new(),
-    };
-
-    //TODO breakouts
-    let link_id = LinkId(0);
-
-    for l in settings.links.iter() {
-        dpd_port_settings.links.insert(
-            link_id.to_string(),
-            LinkSettings {
-                params: LinkCreate {
-                    autoneg: l.autoneg,
-                    lane: Some(LinkId(0)),
-                    kr: false,
-                    fec: match l.fec {
-                        SwitchLinkFec::Firecode => PortFec::Firecode,
-                        SwitchLinkFec::Rs => PortFec::Rs,
-                        SwitchLinkFec::None => PortFec::None,
-                    },
-                    speed: match l.speed {
-                        SwitchLinkSpeed::Speed0G => PortSpeed::Speed0G,
-                        SwitchLinkSpeed::Speed1G => PortSpeed::Speed1G,
-                        SwitchLinkSpeed::Speed10G => PortSpeed::Speed10G,
-                        SwitchLinkSpeed::Speed25G => PortSpeed::Speed25G,
-                        SwitchLinkSpeed::Speed40G => PortSpeed::Speed40G,
-                        SwitchLinkSpeed::Speed50G => PortSpeed::Speed50G,
-                        SwitchLinkSpeed::Speed100G => PortSpeed::Speed100G,
-                        SwitchLinkSpeed::Speed200G => PortSpeed::Speed200G,
-                        SwitchLinkSpeed::Speed400G => PortSpeed::Speed400G,
-                    },
-                },
-                //TODO won't work for breakouts
-                addrs: settings
-                    .addresses
-                    .iter()
-                    .map(|a| a.address.ip())
-                    .collect(),
-            },
-        );
-    }
-
-    Ok(dpd_port_settings)
-}
-
-pub(crate) fn apply_bootstore_update(
-    config: &mut EarlyNetworkConfig,
-    update: &EarlyNetworkPortUpdate,
-) -> Result<BootstoreNetworkPortChange, ActionError> {
-    let mut change = BootstoreNetworkPortChange::default();
-
-    let rack_net_config = match &mut config.body.rack_network_config {
-        Some(cfg) => cfg,
-        None => {
-            return Err(ActionError::action_failed(
-                "rack network config not yet initialized".to_string(),
-            ))
-        }
-    };
-
-    for port in &mut rack_net_config.ports {
-        if port.port == update.port.port {
-            change.previous_port_config = Some(port.clone());
-            *port = update.port.clone();
-            break;
-        }
-    }
-    if change.previous_port_config.is_none() {
-        rack_net_config.ports.push(update.port.clone());
-    }
-
-    for updated_bgp in &update.bgp_configs {
-        let mut exists = false;
-        for resident_bgp in &mut rack_net_config.bgp {
-            if resident_bgp.asn == updated_bgp.asn {
-                change.changed_bgp_configs.push(resident_bgp.clone());
-                *resident_bgp = updated_bgp.clone();
-                exists = true;
-                break;
-            }
-        }
-        if !exists {
-            change.added_bgp_configs.push(updated_bgp.clone());
-        }
-    }
-    rack_net_config.bgp.extend_from_slice(&change.added_bgp_configs);
-
-    Ok(change)
-}
-
-pub(crate) async fn bootstore_update(
-    nexus: &Arc<Nexus>,
-    opctx: &OpContext,
-    switch_port_id: Uuid,
-    switch_port_name: &str,
-    settings: &SwitchPortSettingsCombinedResult,
-) -> Result<EarlyNetworkPortUpdate, ActionError> {
-    let switch_port =
-        nexus.get_switch_port(&opctx, switch_port_id).await.map_err(|e| {
-            ActionError::action_failed(format!(
-                "get switch port for uplink: {e}"
-            ))
-        })?;
-
-    let switch_location: SwitchLocation =
-        switch_port.switch_location.parse().map_err(
-            |e: ParseSwitchLocationError| {
-                ActionError::action_failed(format!(
-                    "get switch location for uplink: {e:?}",
-                ))
-            },
-        )?;
-
-    let mut peer_info = Vec::new();
-    let mut bgp_configs = Vec::new();
-    for p in &settings.bgp_peers {
-        let bgp_config = nexus
-            .bgp_config_get(&opctx, p.bgp_config_id.into())
-            .await
-            .map_err(|e| {
-                ActionError::action_failed(format!("get bgp config: {e}"))
-            })?;
-
-        let announcements = nexus
-            .bgp_announce_list(
-                &opctx,
-                &params::BgpAnnounceSetSelector {
-                    name_or_id: NameOrId::Id(bgp_config.bgp_announce_set_id),
-                },
-            )
-            .await
-            .map_err(|e| {
-                ActionError::action_failed(format!(
-                    "get bgp announcements: {e}"
-                ))
-            })?;
-
-        peer_info.push((p, bgp_config.asn.0));
-        bgp_configs.push(BgpConfig {
-            asn: bgp_config.asn.0,
-            originate: announcements
-                .iter()
-                .filter_map(|a| match a.network {
-                    IpNetwork::V4(net) => Some(net.into()),
-                    //TODO v6
-                    _ => None,
-                })
-                .collect(),
-        });
-    }
-
-    let update = EarlyNetworkPortUpdate {
-        port: PortConfigV1 {
-            routes: settings
-                .routes
-                .iter()
-                .map(|r| RouteConfig { destination: r.dst, nexthop: r.gw.ip() })
-                .collect(),
-            addresses: settings.addresses.iter().map(|a| a.address).collect(),
-            switch: switch_location,
-            port: switch_port_name.into(),
-            uplink_port_fec: settings
-                .links
-                .get(0)
-                .map(|l| l.fec)
-                .unwrap_or(SwitchLinkFec::None)
-                .into(),
-            uplink_port_speed: settings
-                .links
-                .get(0)
-                .map(|l| l.speed)
-                .unwrap_or(SwitchLinkSpeed::Speed100G)
-                .into(),
-            autoneg: settings.links.get(0).map(|l| l.autoneg).unwrap_or(false),
-            bgp_peers: peer_info
-                .iter()
-                .filter_map(|(p, asn)| {
-                    //TODO v6
-                    match p.addr.ip() {
-                        IpAddr::V4(addr) => Some(OmicronBgpPeerConfig {
-                            asn: *asn,
-                            port: switch_port_name.into(),
-                            addr,
-                            hold_time: Some(p.hold_time.0.into()),
-                            connect_retry: Some(p.connect_retry.0.into()),
-                            delay_open: Some(p.delay_open.0.into()),
-                            idle_hold_time: Some(p.idle_hold_time.0.into()),
-                            keepalive: Some(p.keepalive.0.into()),
-                        }),
-                        IpAddr::V6(_) => {
-                            warn!(opctx.log, "IPv6 peers not yet supported");
-                            None
-                        }
-                    }
-                })
-                .collect(),
-        },
-        bgp_configs,
-    };
-
-    Ok(update)
-}
-
-pub(crate) async fn ensure_switch_port_uplink(
-    sagactx: NexusActionContext,
-    opctx: &OpContext,
-    skip_self: bool,
-    inject: Option<Uuid>,
-    switch_port_id: Uuid,
-    switch_port_name: String,
-) -> Result<(), ActionError> {
-    let osagactx = sagactx.user_data();
-    let nexus = osagactx.nexus();
-
-    let switch_port =
-        nexus.get_switch_port(&opctx, switch_port_id).await.map_err(|e| {
-            ActionError::action_failed(format!(
-                "get switch port for uplink: {e}"
-            ))
-        })?;
-
-    let switch_location: SwitchLocation =
-        switch_port.switch_location.parse().map_err(|e| {
-            ActionError::action_failed(format!(
-                "get switch location for uplink: {e:?}",
-            ))
-        })?;
-
-    let mut uplinks: Vec<HostPortConfig> = Vec::new();
-
-    // The sled agent uplinks interface is an all or nothing interface, so we
-    // need to get all the uplink configs for all the ports.
-    let active_ports =
-        nexus.active_port_settings(&opctx).await.map_err(|e| {
-            ActionError::action_failed(format!(
-                "get active switch port settings: {e}"
-            ))
-        })?;
-
-    for (port, info) in &active_ports {
-        // Since we are undoing establishing uplinks for the settings
-        // associated with this port we skip adding this ports uplinks
-        // to the list - effectively removing them.
-        if skip_self && port.id == switch_port.id {
-            continue;
-        }
-        uplinks.push(HostPortConfig {
-            port: port.port_name.clone(),
-            addrs: info.addresses.iter().map(|a| a.address).collect(),
-        })
-    }
-
-    if let Some(id) = inject {
-        let settings = nexus
-            .switch_port_settings_get(&opctx, &id.into())
-            .await
-            .map_err(|e| {
-                ActionError::action_failed(format!(
-                    "get switch port settings for injection: {e}"
-                ))
-            })?;
-        uplinks.push(HostPortConfig {
-            port: switch_port_name.clone(),
-            addrs: settings.addresses.iter().map(|a| a.address).collect(),
-        })
-    }
-
-    let sc = switch_sled_agent(switch_location, &sagactx).await?;
-    sc.uplink_ensure(&sled_agent_client::types::SwitchPorts { uplinks })
-        .await
-        .map_err(|e| {
-            ActionError::action_failed(format!("ensure uplink: {e}"))
-        })?;
-
-    Ok(())
-}
-
-pub(crate) async fn read_bootstore_config(
-    sa: &sled_agent_client::Client,
-) -> Result<EarlyNetworkConfig, ActionError> {
-    Ok(sa
-        .read_network_bootstore_config_cache()
-        .await
-        .map_err(|e| {
-            ActionError::action_failed(format!(
-                "read bootstore network config: {e}"
-            ))
-        })?
-        .into_inner())
-}
-
-pub(crate) async fn write_bootstore_config(
-    sa: &sled_agent_client::Client,
-    config: &EarlyNetworkConfig,
-) -> Result<(), ActionError> {
-    sa.write_network_bootstore_config(config).await.map_err(|e| {
-        ActionError::action_failed(format!(
-            "write bootstore network config: {e}"
-        ))
-    })?;
-    Ok(())
-}
-
-pub(crate) async fn select_mg_client(
-    sagactx: &NexusActionContext,
-    opctx: &OpContext,
-    switch_port_id: Uuid,
-) -> Result<Arc<mg_admin_client::Client>, ActionError> {
-    let osagactx = sagactx.user_data();
-    let nexus = osagactx.nexus();
-
-    let switch_port =
-        nexus.get_switch_port(&opctx, switch_port_id).await.map_err(|e| {
-            ActionError::action_failed(format!(
-                "get switch port for mg client selection: {e}"
-            ))
-        })?;
-
-    let switch_location: SwitchLocation =
-        switch_port.switch_location.parse().map_err(
-            |e: ParseSwitchLocationError| {
-                ActionError::action_failed(format!(
-                    "get switch location for uplink: {e:?}",
-                ))
-            },
-        )?;
-
-    let mg_client: Arc<mg_admin_client::Client> = osagactx
-        .nexus()
-        .mg_clients
-        .get(&switch_location)
-        .ok_or_else(|| {
-            ActionError::action_failed(format!(
-                "requested switch not available: {switch_location}"
-            ))
-        })?
-        .clone();
-    Ok(mg_client)
-}
-
-pub(crate) async fn switch_sled_agent(
-    location: SwitchLocation,
-    sagactx: &NexusActionContext,
-) -> Result<sled_agent_client::Client, ActionError> {
-    let nexus = sagactx.user_data().nexus();
-    let sled_agent_addr = get_scrimlet_address(location, nexus).await?;
-    Ok(sled_agent_client::Client::new(
-        &format!("http://{}", sled_agent_addr),
-        sagactx.user_data().log().clone(),
-    ))
-}
-
-pub(crate) async fn ensure_switch_port_bgp_settings(
-    sagactx: NexusActionContext,
-    opctx: &OpContext,
-    settings: SwitchPortSettingsCombinedResult,
-    switch_port_name: String,
-    switch_port_id: Uuid,
-) -> Result<(), ActionError> {
-    let osagactx = sagactx.user_data();
-    let nexus = osagactx.nexus();
-    let mg_client: Arc<mg_admin_client::Client> =
-        select_mg_client(&sagactx, opctx, switch_port_id).await.map_err(
-            |e| ActionError::action_failed(format!("select mg client: {e}")),
-        )?;
-
-    let mut bgp_peer_configs = HashMap::<String, Vec<BgpPeerConfig>>::new();
-
-    let mut cfg: Option<nexus_db_model::BgpConfig> = None;
-
-    for peer in settings.bgp_peers {
-        let config = nexus
-            .bgp_config_get(&opctx, peer.bgp_config_id.into())
-            .await
-            .map_err(|e| {
-                ActionError::action_failed(format!("get bgp config: {e}"))
-            })?;
-
-        if let Some(cfg) = &cfg {
-            if config.asn != cfg.asn {
-                return Err(ActionError::action_failed(
-                    "bad request: only one AS allowed per switch".to_string(),
-                ));
-            }
-        } else {
-            cfg = Some(config);
-        }
-
-        let bpc = BgpPeerConfig {
-            name: format!("{}", peer.addr.ip()), //TODO user defined name?
-            host: format!("{}:179", peer.addr.ip()),
-            hold_time: peer.hold_time.0.into(),
-            idle_hold_time: peer.idle_hold_time.0.into(),
-            delay_open: peer.delay_open.0.into(),
-            connect_retry: peer.connect_retry.0.into(),
-            keepalive: peer.keepalive.0.into(),
-            resolution: BGP_SESSION_RESOLUTION,
-            passive: false,
-        };
-
-        match bgp_peer_configs.get_mut(&switch_port_name) {
-            Some(peers) => {
-                peers.push(bpc);
-            }
-            None => {
-                bgp_peer_configs.insert(switch_port_name.clone(), vec![bpc]);
-            }
-        }
-    }
-
-    if let Some(cfg) = &cfg {
-        let announcements = nexus
-            .bgp_announce_list(
-                &opctx,
-                &params::BgpAnnounceSetSelector {
-                    name_or_id: NameOrId::Id(cfg.bgp_announce_set_id),
-                },
-            )
-            .await
-            .map_err(|e| {
-                ActionError::action_failed(format!(
-                    "get bgp announcements: {e}"
-                ))
-            })?;
-
-        let mut prefixes = Vec::new();
-        for a in &announcements {
-            let value = match a.network.ip() {
-                IpAddr::V4(value) => Ok(value),
-                IpAddr::V6(_) => Err(ActionError::action_failed(
-                    "bad request: IPv6 announcement not yet supported"
-                        .to_string(),
-                )),
-            }?;
-            prefixes.push(Prefix4 { value, length: a.network.prefix() });
-        }
-        mg_client
-            .inner
-            .bgp_apply(&ApplyRequest {
-                asn: cfg.asn.0,
-                peers: bgp_peer_configs,
-                originate: prefixes,
-            })
-            .await
-            .map_err(|e| {
-                ActionError::action_failed(format!("apply bgp settings: {e}"))
-            })?;
-    }
-
-    Ok(())
-}
-
-pub(crate) async fn get_scrimlet_address(
-    location: SwitchLocation,
-    nexus: &Arc<Nexus>,
-) -> Result<SocketAddrV6, ActionError> {
-    /* TODO this depends on DNS entries only coming from RSS, it's broken
-            on the upgrade path
-    nexus
-        .resolver()
-        .await
-        .lookup_socket_v6(ServiceName::Scrimlet(location))
-        .await
-        .map_err(|e| e.to_string())
-        .map_err(|e| {
-            ActionError::action_failed(format!(
-                "scrimlet dns lookup failed {e}",
-            ))
-        })
-    */
-    let result = nexus
-        .resolver()
-        .await
-        .lookup_all_ipv6(ServiceName::Dendrite)
-        .await
-        .map_err(|e| {
-            ActionError::action_failed(format!(
-                "scrimlet dns lookup failed {e}",
-            ))
-        });
-
-    let mappings = match result {
-        Ok(addrs) => map_switch_zone_addrs(&nexus.log, addrs).await,
-        Err(e) => {
-            warn!(nexus.log, "Failed to lookup Dendrite address: {e}");
-            return Err(ActionError::action_failed(format!(
-                "switch mapping failed {e}",
-            )));
-        }
-    };
-
-    let addr = match mappings.get(&location) {
-        Some(addr) => addr,
-        None => {
-            return Err(ActionError::action_failed(format!(
-                "address for switch at location: {location} not found",
-            )));
-        }
-    };
-
-    let mut segments = addr.segments();
-    segments[7] = 1;
-    let addr = Ipv6Addr::from(segments);
-
-    Ok(SocketAddrV6::new(addr, SLED_AGENT_PORT, 0, 0))
-}
-
-#[derive(Clone, Debug, Default)]
-pub(crate) struct BootstoreNetworkPortChange {
-    previous_port_config: Option<PortConfigV1>,
-    changed_bgp_configs: Vec<BgpConfig>,
-    added_bgp_configs: Vec<BgpConfig>,
-}
-
-#[derive(Clone, Debug)]
-pub struct EarlyNetworkPortUpdate {
-    port: PortConfigV1,
-    bgp_configs: Vec<BgpConfig>,
-}
-
-pub(crate) async fn select_dendrite_client(
-    sagactx: &NexusActionContext,
-    opctx: &OpContext,
-    switch_port_id: Uuid,
-) -> Result<Arc<dpd_client::Client>, ActionError> {
-    let osagactx = sagactx.user_data();
-    let nexus = osagactx.nexus();
-
-    let switch_port =
-        nexus.get_switch_port(&opctx, switch_port_id).await.map_err(|e| {
-            ActionError::action_failed(format!(
-                "get switch port for dendrite client selection {e}"
-            ))
-        })?;
-
-    let switch_location: SwitchLocation =
-        switch_port.switch_location.parse().map_err(
-            |e: ParseSwitchLocationError| {
-                ActionError::action_failed(format!(
-                    "get switch location for uplink: {e:?}",
-                ))
-            },
-        )?;
-
-    let dpd_client: Arc<dpd_client::Client> = osagactx
-        .nexus()
-        .dpd_clients
-        .get(&switch_location)
-        .ok_or_else(|| {
-            ActionError::action_failed(format!(
-                "requested switch not available: {switch_location}"
-            ))
-        })?
-        .clone();
-    Ok(dpd_client)
-}
diff --git a/nexus/src/app/switch_interface.rs b/nexus/src/app/switch_interface.rs
index 0acb2b7fe7..c3ce0f553c 100644
--- a/nexus/src/app/switch_interface.rs
+++ b/nexus/src/app/switch_interface.rs
@@ -2,10 +2,8 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use crate::app::sagas;
 use crate::external_api::params;
 use db::model::{LoopbackAddress, Name};
-use nexus_db_queries::authn;
 use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db;
@@ -13,8 +11,7 @@ use nexus_db_queries::db::lookup;
 use nexus_db_queries::db::lookup::LookupPath;
 use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::{
-    CreateResult, DataPageParams, DeleteResult, Error, InternalContext, IpNet,
-    ListResultVec,
+    CreateResult, DataPageParams, DeleteResult, Error, IpNet, ListResultVec,
 };
 use std::sync::Arc;
 use uuid::Uuid;
@@ -43,21 +40,24 @@ impl super::Nexus {
 
         validate_switch_location(params.switch_location.as_str())?;
 
-        let saga_params = sagas::loopback_address_create::Params {
-            serialized_authn: authn::saga::Serialized::for_opctx(opctx),
-            loopback_address: params.clone(),
-        };
+        // Just a check to make sure a valid rack id was passed in.
+        self.rack_lookup(&opctx, &params.rack_id).await?;
 
-        let saga_output = self.execute_saga::<
-            sagas::loopback_address_create::SagaLoopbackAddressCreate>(
-                saga_params).await?;
+        let address_lot_lookup =
+            self.address_lot_lookup(&opctx, params.address_lot.clone())?;
 
-        let value = saga_output
-            .lookup_node_output::<LoopbackAddress>(
-                "created_loopback_address_record",
-            )
-            .map_err(|e| Error::internal_error(&format!("{:#}", &e)))
-            .internal_context("looking up output from loopback create saga")?;
+        let (.., authz_address_lot) =
+            address_lot_lookup.lookup_for(authz::Action::CreateChild).await?;
+
+        let value = self
+            .db_datastore
+            .loopback_address_create(&opctx, &params, None, &authz_address_lot)
+            .await?;
+
+        // eagerly propagate changes via rpw
+        self.background_tasks
+            .driver
+            .activate(&self.background_tasks.task_switch_port_settings_manager);
 
         Ok(value)
     }
@@ -69,16 +69,24 @@ impl super::Nexus {
         switch_location: Name,
         address: IpNet,
     ) -> DeleteResult {
-        let saga_params = sagas::loopback_address_delete::Params {
-            serialized_authn: authn::saga::Serialized::for_opctx(opctx),
-            address,
+        let loopback_address_lookup = self.loopback_address_lookup(
+            &opctx,
             rack_id,
             switch_location,
-        };
+            address,
+        )?;
+
+        let (.., authz_loopback_address) =
+            loopback_address_lookup.lookup_for(authz::Action::Delete).await?;
+
+        self.db_datastore
+            .loopback_address_delete(&opctx, &authz_loopback_address)
+            .await?;
 
-        self.execute_saga::<
-            sagas::loopback_address_delete::SagaLoopbackAddressDelete>(
-                saga_params).await?;
+        // eagerly propagate changes via rpw
+        self.background_tasks
+            .driver
+            .activate(&self.background_tasks.task_switch_port_settings_manager);
 
         Ok(())
     }
diff --git a/nexus/src/app/switch_port.rs b/nexus/src/app/switch_port.rs
index fc9ad2866a..c7d5272ae1 100644
--- a/nexus/src/app/switch_port.rs
+++ b/nexus/src/app/switch_port.rs
@@ -92,38 +92,26 @@ impl super::Nexus {
             )
             .await?;
 
-        // run the port settings apply saga for each port referencing the
-        // updated settings
-
         let ports = self
             .db_datastore
             .switch_ports_using_settings(opctx, switch_port_settings_id)
             .await?;
 
-        for (switch_port_id, switch_port_name) in ports.into_iter() {
-            let saga_params = sagas::switch_port_settings_apply::Params {
-                serialized_authn: authn::saga::Serialized::for_opctx(opctx),
+        for (switch_port_id, _switch_port_name) in ports.into_iter() {
+            self.set_switch_port_settings_id(
+                &opctx,
                 switch_port_id,
-                switch_port_settings_id: result.settings.id(),
-                switch_port_name: switch_port_name.to_string(),
-            };
-
-            self.execute_saga::<
-                sagas::switch_port_settings_apply::SagaSwitchPortSettingsApply
-                >(
-                    saga_params,
-                )
-                .await
-            .map_err(|e| {
-                    let msg = e.to_string();
-                    if msg.contains("bad request") {
-                        external::Error::invalid_request(&msg.to_string())
-                    } else {
-                        e
-                    }
-                })?;
+                Some(switch_port_settings_id),
+                UpdatePrecondition::DontCare,
+            )
+            .await?;
         }
 
+        // eagerly propagate changes via rpw
+        self.background_tasks
+            .driver
+            .activate(&self.background_tasks.task_switch_port_settings_manager);
+
         Ok(result)
     }
 
@@ -180,15 +168,6 @@ impl super::Nexus {
         self.db_datastore.switch_port_list(opctx, pagparams).await
     }
 
-    pub(crate) async fn get_switch_port(
-        &self,
-        opctx: &OpContext,
-        params: uuid::Uuid,
-    ) -> LookupResult<SwitchPort> {
-        opctx.authorize(authz::Action::Read, &authz::FLEET).await?;
-        self.db_datastore.switch_port_get(opctx, params).await
-    }
-
     pub(crate) async fn list_switch_ports_with_uplinks(
         &self,
         opctx: &OpContext,
@@ -242,27 +221,18 @@ impl super::Nexus {
             }
         };
 
-        let saga_params = sagas::switch_port_settings_apply::Params {
-            serialized_authn: authn::saga::Serialized::for_opctx(opctx),
+        self.set_switch_port_settings_id(
+            &opctx,
             switch_port_id,
-            switch_port_settings_id,
-            switch_port_name: port.to_string(),
-        };
-
-        self.execute_saga::<
-            sagas::switch_port_settings_apply::SagaSwitchPortSettingsApply
-        >(
-            saga_params,
+            Some(switch_port_settings_id),
+            UpdatePrecondition::DontCare,
         )
-        .await
-        .map_err(|e| {
-                let msg = e.to_string();
-                if msg.contains("bad request") {
-                    external::Error::invalid_request(&msg.to_string())
-                } else {
-                    e
-                }
-            })?;
+        .await?;
+
+        // eagerly propagate changes via rpw
+        self.background_tasks
+            .driver
+            .activate(&self.background_tasks.task_switch_port_settings_manager);
 
         Ok(())
     }
@@ -284,17 +254,20 @@ impl super::Nexus {
             )
             .await?;
 
-        let saga_params = sagas::switch_port_settings_clear::Params {
-            serialized_authn: authn::saga::Serialized::for_opctx(opctx),
+        // update the switch port settings association
+        self.set_switch_port_settings_id(
+            &opctx,
             switch_port_id,
-            port_name: port.to_string(),
-        };
-
-        self.execute_saga::<sagas::switch_port_settings_clear::SagaSwitchPortSettingsClear>(
-            saga_params,
+            None,
+            UpdatePrecondition::DontCare,
         )
         .await?;
 
+        // eagerly propagate changes via rpw
+        self.background_tasks
+            .driver
+            .activate(&self.background_tasks.task_switch_port_settings_manager);
+
         Ok(())
     }
 
@@ -323,25 +296,4 @@ impl super::Nexus {
 
         Ok(())
     }
-
-    // TODO it would likely be better to do this as a one shot db query.
-    pub(crate) async fn active_port_settings(
-        &self,
-        opctx: &OpContext,
-    ) -> LookupResult<Vec<(SwitchPort, SwitchPortSettingsCombinedResult)>> {
-        let mut ports = Vec::new();
-        let port_list =
-            self.switch_port_list(opctx, &DataPageParams::max_page()).await?;
-
-        for p in port_list {
-            if let Some(id) = p.port_settings_id {
-                ports.push((
-                    p.clone(),
-                    self.switch_port_settings_get(opctx, &id.into()).await?,
-                ));
-            }
-        }
-
-        LookupResult::Ok(ports)
-    }
 }
diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml
index 8d37f9e3ef..de3aa4c1f4 100644
--- a/nexus/tests/config.test.toml
+++ b/nexus/tests/config.test.toml
@@ -10,7 +10,7 @@ session_absolute_timeout_minutes = 480
 
 # List of authentication schemes to support.
 [authn]
-schemes_external = [ "spoof", "session_cookie" ]
+schemes_external = ["spoof", "session_cookie"]
 
 #
 # NOTE: for the test suite, if mode = "file", the file path MUST be the sentinel
@@ -43,7 +43,7 @@ techport_external_server_port = 0
 
 # Nexus may need to resolve external hosts (e.g. to grab IdP metadata).
 # These are the DNS servers it should use.
-external_dns_servers = [ "1.1.1.1", "9.9.9.9" ]
+external_dns_servers = ["1.1.1.1", "9.9.9.9"]
 
 [deployment.dropshot_external]
 # NOTE: for the test suite, the port MUST be 0 (in order to bind to any
@@ -103,6 +103,7 @@ phantom_disks.period_secs = 30
 blueprints.period_secs_load = 100
 blueprints.period_secs_execute = 600
 sync_service_zone_nat.period_secs = 30
+switch_port_settings_manager.period_secs = 30
 region_replacement.period_secs = 30
 
 [default_region_allocation_strategy]
diff --git a/nexus/tests/integration_tests/address_lots.rs b/nexus/tests/integration_tests/address_lots.rs
index 40c8865929..7860dd463c 100644
--- a/nexus/tests/integration_tests/address_lots.rs
+++ b/nexus/tests/integration_tests/address_lots.rs
@@ -4,7 +4,6 @@
 
 //! Integration tests for operating on Address Lots
 
-use dropshot::HttpErrorResponseBody;
 use http::method::Method;
 use http::StatusCode;
 use nexus_test_utils::http_testing::AuthnMode;
@@ -79,27 +78,6 @@ async fn test_address_lot_basic_crud(ctx: &ControlPlaneTestContext) {
         "203.0.113.20".parse::<IpAddr>().unwrap()
     );
 
-    // Verify conflict error on recreate
-    let error: HttpErrorResponseBody = NexusRequest::new(
-        RequestBuilder::new(
-            client,
-            Method::POST,
-            "/v1/system/networking/address-lot",
-        )
-        .body(Some(&params))
-        .expect_status(Some(StatusCode::BAD_REQUEST)),
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap()
-    .parsed_body()
-    .unwrap();
-    assert_eq!(
-        error.message,
-        "already exists: address-lot \"parkinglot\"".to_string()
-    );
-
     // Verify there are lots
     let lots = NexusRequest::iter_collection_authn::<AddressLot>(
         client,
diff --git a/nexus/tests/integration_tests/loopback_address.rs b/nexus/tests/integration_tests/loopback_address.rs
deleted file mode 100644
index b92e82b216..0000000000
--- a/nexus/tests/integration_tests/loopback_address.rs
+++ /dev/null
@@ -1,389 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-//! Integration tests for operating on Loopback Addresses
-
-use dropshot::HttpErrorResponseBody;
-use http::method::Method;
-use http::StatusCode;
-use nexus_test_utils::http_testing::{AuthnMode, NexusRequest, RequestBuilder};
-use nexus_test_utils_macros::nexus_test;
-use nexus_types::external_api::params::{
-    AddressLotBlockCreate, AddressLotCreate, LoopbackAddressCreate,
-};
-use nexus_types::external_api::views::Rack;
-use omicron_common::api::external::{
-    AddressLotKind, IdentityMetadataCreateParams, LoopbackAddress, NameOrId,
-};
-
-type ControlPlaneTestContext =
-    nexus_test_utils::ControlPlaneTestContext<omicron_nexus::Server>;
-
-#[nexus_test]
-async fn test_loopback_address_basic_crud(ctx: &ControlPlaneTestContext) {
-    let client = &ctx.external_client;
-
-    // Create a lot
-    let lot_params = AddressLotCreate {
-        identity: IdentityMetadataCreateParams {
-            name: "parkinglot".parse().unwrap(),
-            description: "an address parking lot".into(),
-        },
-        kind: AddressLotKind::Infra,
-        blocks: vec![AddressLotBlockCreate {
-            first_address: "203.0.113.10".parse().unwrap(),
-            last_address: "203.0.113.100".parse().unwrap(),
-        }],
-    };
-
-    NexusRequest::objects_post(
-        client,
-        "/v1/system/networking/address-lot",
-        &lot_params,
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap();
-
-    // Verify there are no loopback addresses
-    let addrs = NexusRequest::iter_collection_authn::<LoopbackAddress>(
-        client,
-        "/v1/system/networking/loopback-address",
-        "",
-        None,
-    )
-    .await
-    .expect("Failed to list loopback addresses")
-    .all_items;
-
-    assert_eq!(addrs.len(), 0, "Expected no loopback addresses");
-
-    let racks_url = "/v1/system/hardware/racks";
-    let racks: Vec<Rack> =
-        NexusRequest::iter_collection_authn(client, racks_url, "", None)
-            .await
-            .expect("failed to list racks")
-            .all_items;
-
-    let rack_id = racks[0].identity.id;
-
-    // Create a loopback address
-    let params = LoopbackAddressCreate {
-        address_lot: NameOrId::Name("parkinglot".parse().unwrap()),
-        rack_id: rack_id,
-        switch_location: "switch0".parse().unwrap(),
-        address: "203.0.113.99".parse().unwrap(),
-        mask: 24,
-        anycast: false,
-    };
-    let addr: LoopbackAddress = NexusRequest::objects_post(
-        client,
-        "/v1/system/networking/loopback-address",
-        &params,
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap()
-    .parsed_body()
-    .unwrap();
-
-    assert_eq!(addr.address.ip(), params.address);
-    assert_eq!(addr.address.prefix(), params.mask);
-    assert_eq!(addr.rack_id, params.rack_id);
-    assert_eq!(addr.switch_location, params.switch_location.to_string());
-
-    // Verify conflict error on recreate
-    let error: HttpErrorResponseBody = NexusRequest::new(
-        RequestBuilder::new(
-            client,
-            Method::POST,
-            "/v1/system/networking/loopback-address",
-        )
-        .body(Some(&params))
-        .expect_status(Some(StatusCode::BAD_REQUEST)),
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap()
-    .parsed_body()
-    .unwrap();
-    assert_eq!(error.message, "address unavailable".to_string());
-
-    // Verify there loopback addresses
-    let addrs = NexusRequest::iter_collection_authn::<LoopbackAddress>(
-        client,
-        "/v1/system/networking/loopback-address",
-        "",
-        None,
-    )
-    .await
-    .expect("Failed to list loopback addresses")
-    .all_items;
-
-    assert_eq!(addrs.len(), 1, "Expected 1 loopback address");
-    assert_eq!(addrs[0].address.ip(), params.address);
-    assert_eq!(addrs[0].address.prefix(), params.mask);
-
-    // Verify error when deleting lot while in use
-    let _error: HttpErrorResponseBody = NexusRequest::new(
-        RequestBuilder::new(
-            client,
-            Method::DELETE,
-            "/v1/system/networking/address-lot/parkinglot",
-        )
-        .expect_status(Some(StatusCode::BAD_REQUEST)),
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap()
-    .parsed_body()
-    .unwrap();
-
-    // Delete loopback address
-    NexusRequest::object_delete(
-        client,
-        &format!(
-            "{}/{}/{}/{}/{}",
-            "/v1/system/networking/loopback-address",
-            rack_id,
-            "switch0",
-            "203.0.113.99",
-            24,
-        ),
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap();
-
-    // Verify there are no addresses
-    let addrs = NexusRequest::iter_collection_authn::<LoopbackAddress>(
-        client,
-        "/v1/system/networking/loopback-address",
-        "",
-        None,
-    )
-    .await
-    .expect("Failed to list loopback addresses")
-    .all_items;
-
-    assert_eq!(addrs.len(), 0, "Expected no loopback addresses after delete");
-
-    // Verify we can now delete the address lot.
-    NexusRequest::object_delete(
-        client,
-        "/v1/system/networking/address-lot/parkinglot",
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap();
-
-    // Create again after delete should work without conflict.
-    NexusRequest::objects_post(
-        client,
-        "/v1/system/networking/address-lot",
-        &lot_params,
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap();
-
-    let _addr: LoopbackAddress = NexusRequest::objects_post(
-        client,
-        "/v1/system/networking/loopback-address",
-        &params,
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap()
-    .parsed_body()
-    .unwrap();
-}
-
-#[nexus_test]
-async fn test_anycast_loopback_address_basic_crud(
-    ctx: &ControlPlaneTestContext,
-) {
-    let client = &ctx.external_client;
-
-    // Create a lot
-    let lot_params = AddressLotCreate {
-        identity: IdentityMetadataCreateParams {
-            name: "parkinglot".parse().unwrap(),
-            description: "an address parking lot".into(),
-        },
-        kind: AddressLotKind::Infra,
-        blocks: vec![AddressLotBlockCreate {
-            first_address: "203.0.113.10".parse().unwrap(),
-            last_address: "203.0.113.100".parse().unwrap(),
-        }],
-    };
-
-    NexusRequest::objects_post(
-        client,
-        "/v1/system/networking/address-lot",
-        &lot_params,
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap();
-
-    // Verify there are no loopback addresses
-    let addrs = NexusRequest::iter_collection_authn::<LoopbackAddress>(
-        client,
-        "/v1/system/networking/loopback-address",
-        "",
-        None,
-    )
-    .await
-    .expect("Failed to list loopback addresses")
-    .all_items;
-
-    assert_eq!(addrs.len(), 0, "Expected no loopback addresses");
-
-    let racks_url = "/v1/system/hardware/racks";
-    let racks: Vec<Rack> =
-        NexusRequest::iter_collection_authn(client, racks_url, "", None)
-            .await
-            .expect("failed to list racks")
-            .all_items;
-
-    let rack_id = racks[0].identity.id;
-
-    // Create an anycast loopback address
-    let params = LoopbackAddressCreate {
-        address_lot: NameOrId::Name("parkinglot".parse().unwrap()),
-        rack_id,
-        switch_location: "switch0".parse().unwrap(),
-        address: "203.0.113.99".parse().unwrap(),
-        mask: 24,
-        anycast: true,
-    };
-    let addr: LoopbackAddress = NexusRequest::objects_post(
-        client,
-        "/v1/system/networking/loopback-address",
-        &params,
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap()
-    .parsed_body()
-    .unwrap();
-
-    assert_eq!(addr.address.ip(), params.address);
-    assert_eq!(addr.address.prefix(), params.mask);
-    assert_eq!(addr.rack_id, params.rack_id);
-    assert_eq!(addr.switch_location, params.switch_location.to_string());
-
-    // Create a second anycast record for another switch
-    let params = LoopbackAddressCreate {
-        address_lot: NameOrId::Name("parkinglot".parse().unwrap()),
-        rack_id,
-        switch_location: "switch1".parse().unwrap(),
-        address: "203.0.113.99".parse().unwrap(),
-        mask: 24,
-        anycast: true,
-    };
-    let addr: LoopbackAddress = NexusRequest::objects_post(
-        client,
-        "/v1/system/networking/loopback-address",
-        &params,
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap()
-    .parsed_body()
-    .unwrap();
-
-    assert_eq!(addr.address.ip(), params.address);
-    assert_eq!(addr.address.prefix(), params.mask);
-    assert_eq!(addr.rack_id, params.rack_id);
-    assert_eq!(addr.switch_location, params.switch_location.to_string());
-
-    // Verify there are two anycast loopback addresses
-    let addrs = NexusRequest::iter_collection_authn::<LoopbackAddress>(
-        client,
-        "/v1/system/networking/loopback-address",
-        "",
-        None,
-    )
-    .await
-    .expect("Failed to list loopback addresses")
-    .all_items;
-
-    assert_eq!(addrs.len(), 2, "Expected 2 loopback addresses");
-    assert_eq!(addrs[0].address.ip(), params.address);
-    assert_eq!(addrs[0].address.prefix(), params.mask);
-
-    // Delete anycast loopback addresses
-    NexusRequest::object_delete(
-        client,
-        &format!(
-            "{}/{}/{}/{}/{}",
-            "/v1/system/networking/loopback-address",
-            rack_id,
-            "switch0",
-            "203.0.113.99",
-            24,
-        ),
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap();
-
-    // Verify there is only one anycast loopback address
-    let addrs = NexusRequest::iter_collection_authn::<LoopbackAddress>(
-        client,
-        "/v1/system/networking/loopback-address",
-        "",
-        None,
-    )
-    .await
-    .expect("Failed to list loopback addresses")
-    .all_items;
-
-    assert_eq!(addrs.len(), 1, "Expected 1 loopback address");
-    assert_eq!(addrs[0].address.ip(), params.address);
-    assert_eq!(addrs[0].address.prefix(), params.mask);
-
-    NexusRequest::object_delete(
-        client,
-        &format!(
-            "{}/{}/{}/{}/{}",
-            "/v1/system/networking/loopback-address",
-            rack_id,
-            "switch1",
-            "203.0.113.99",
-            24,
-        ),
-    )
-    .authn_as(AuthnMode::PrivilegedUser)
-    .execute()
-    .await
-    .unwrap();
-
-    // Verify there are no addresses
-    let addrs = NexusRequest::iter_collection_authn::<LoopbackAddress>(
-        client,
-        "/v1/system/networking/loopback-address",
-        "",
-        None,
-    )
-    .await
-    .expect("Failed to list loopback addresses")
-    .all_items;
-
-    assert_eq!(addrs.len(), 0, "Expected no loopback addresses after delete");
-}
diff --git a/nexus/tests/integration_tests/mod.rs b/nexus/tests/integration_tests/mod.rs
index 84b867252f..804694c0b2 100644
--- a/nexus/tests/integration_tests/mod.rs
+++ b/nexus/tests/integration_tests/mod.rs
@@ -18,7 +18,6 @@ mod images;
 mod initialization;
 mod instances;
 mod ip_pools;
-mod loopback_address;
 mod metrics;
 mod oximeter;
 mod pantry;
diff --git a/oximeter/collector/tests/output/self-stat-schema.json b/oximeter/collector/tests/output/self-stat-schema.json
index 0caf2d27e9..8017d61880 100644
--- a/oximeter/collector/tests/output/self-stat-schema.json
+++ b/oximeter/collector/tests/output/self-stat-schema.json
@@ -39,7 +39,7 @@
       }
     ],
     "datum_type": "cumulative_u64",
-    "created": "2023-12-04T17:49:47.797495948Z"
+    "created": "2024-02-05T23:03:00.842290108Z"
   },
   "oximeter_collector:failed_collections": {
     "timeseries_name": "oximeter_collector:failed_collections",
@@ -86,6 +86,6 @@
       }
     ],
     "datum_type": "cumulative_u64",
-    "created": "2023-12-04T17:49:47.799970009Z"
+    "created": "2024-02-05T23:03:00.842943988Z"
   }
 }
\ No newline at end of file
diff --git a/package-manifest.toml b/package-manifest.toml
index ef4e6dfb06..0b54e35d4f 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -46,7 +46,7 @@ source.paths = [
   # - gimlet: Assumes control of real hardware, does not emulate hardware
   # - gimlet-standalone: A real gimlet but running in isolation from the rack as a whole
   # - non-gimlet: Used for testing on development illumos machines
-  { from = "smf/sled-agent/{{machine}}", to = "pkg" }
+  { from = "smf/sled-agent/{{machine}}", to = "pkg" },
 ]
 output.type = "tarball"
 
@@ -60,17 +60,14 @@ only_for_targets.image = "trampoline"
 source.type = "local"
 source.rust.binary_names = ["installinator"]
 source.rust.release = true
-source.paths = [ { from = "smf/installinator", to = "pkg" } ]
+source.paths = [{ from = "smf/installinator", to = "pkg" }]
 output.type = "tarball"
 
 # overlay is a set of packages that are overlaid onto all non-global zones.
 [package.overlay]
 service_name = "overlay"
 source.type = "composite"
-source.packages = [
-  "logadm.tar.gz",
-  "profile.tar.gz",
-]
+source.packages = ["logadm.tar.gz", "profile.tar.gz"]
 output.type = "zone"
 
 # The logadm package is an overlay for all non-global zones to reconfigure log
@@ -105,7 +102,7 @@ source.packages = [
   "omicron-nexus.tar.gz",
   "zone-network-setup.tar.gz",
   "zone-network-install.tar.gz",
-  "opte-interface-setup.tar.gz"
+  "opte-interface-setup.tar.gz",
 ]
 output.type = "zone"
 
@@ -133,7 +130,11 @@ output.intermediate_only = true
 service_name = "oximeter"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "oximeter-collector.tar.gz", "zone-network-setup.tar.gz", "zone-network-install.tar.gz" ]
+source.packages = [
+  "oximeter-collector.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz",
+]
 output.type = "zone"
 
 [package.oximeter-collector]
@@ -143,8 +144,8 @@ source.type = "local"
 source.rust.binary_names = ["oximeter", "clickhouse-schema-updater"]
 source.rust.release = true
 source.paths = [
-    { from = "smf/oximeter", to = "/var/svc/manifest/site/oximeter" },
-    { from = "oximeter/db/schema", to = "/opt/oxide/oximeter/schema" },
+  { from = "smf/oximeter", to = "/var/svc/manifest/site/oximeter" },
+  { from = "oximeter/db/schema", to = "/opt/oxide/oximeter/schema" },
 ]
 output.type = "zone"
 output.intermediate_only = true
@@ -157,7 +158,7 @@ source.packages = [
   "clickhouse_svc.tar.gz",
   "internal-dns-cli.tar.gz",
   "zone-network-setup.tar.gz",
-  "zone-network-install.tar.gz"
+  "zone-network-install.tar.gz",
 ]
 output.type = "zone"
 
@@ -169,7 +170,7 @@ source.paths = [
   { from = "out/clickhouse", to = "/opt/oxide/clickhouse" },
   { from = "smf/clickhouse/manifest.xml", to = "/var/svc/manifest/site/clickhouse/manifest.xml" },
   { from = "smf/clickhouse/method_script.sh", to = "/opt/oxide/lib/svc/manifest/clickhouse.sh" },
-  { from = "smf/clickhouse/config_replica.xml", to = "/opt/oxide/clickhouse/config.d/config_replica.xml" }
+  { from = "smf/clickhouse/config_replica.xml", to = "/opt/oxide/clickhouse/config.d/config_replica.xml" },
 ]
 output.type = "zone"
 output.intermediate_only = true
@@ -183,7 +184,7 @@ source.packages = [
   "clickhouse_keeper_svc.tar.gz",
   "internal-dns-cli.tar.gz",
   "zone-network-setup.tar.gz",
-  "zone-network-install.tar.gz"
+  "zone-network-install.tar.gz",
 ]
 output.type = "zone"
 
@@ -195,7 +196,7 @@ source.paths = [
   { from = "out/clickhouse", to = "/opt/oxide/clickhouse_keeper" },
   { from = "smf/clickhouse_keeper/manifest.xml", to = "/var/svc/manifest/site/clickhouse_keeper/manifest.xml" },
   { from = "smf/clickhouse_keeper/method_script.sh", to = "/opt/oxide/lib/svc/manifest/clickhouse_keeper.sh" },
-  { from = "smf/clickhouse_keeper/keeper_config.xml", to = "/opt/oxide/clickhouse_keeper/keeper_config.xml" }
+  { from = "smf/clickhouse_keeper/keeper_config.xml", to = "/opt/oxide/clickhouse_keeper/keeper_config.xml" },
 ]
 output.type = "zone"
 output.intermediate_only = true
@@ -205,11 +206,11 @@ setup_hint = "Run `./tools/ci_download_clickhouse` to download the necessary bin
 service_name = "cockroachdb"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ 
+source.packages = [
   "cockroachdb-service.tar.gz",
   "internal-dns-cli.tar.gz",
   "zone-network-setup.tar.gz",
-  "zone-network-install.tar.gz"
+  "zone-network-install.tar.gz",
 ]
 output.type = "zone"
 
@@ -245,7 +246,7 @@ source.packages = [
   "dns-server.tar.gz",
   "internal-dns-customizations.tar.gz",
   "zone-network-setup.tar.gz",
-  "zone-network-install.tar.gz"
+  "zone-network-install.tar.gz",
 ]
 output.type = "zone"
 
@@ -253,12 +254,12 @@ output.type = "zone"
 service_name = "external_dns"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ 
+source.packages = [
   "dns-server.tar.gz",
   "external-dns-customizations.tar.gz",
   "zone-network-setup.tar.gz",
   "zone-network-install.tar.gz",
-  "opte-interface-setup.tar.gz"
+  "opte-interface-setup.tar.gz",
 ]
 output.type = "zone"
 
@@ -276,7 +277,9 @@ output.intermediate_only = true
 service_name = "internal-dns-customizations"
 only_for_targets.image = "standard"
 source.type = "local"
-source.paths = [ { from = "smf/internal-dns", to = "/var/svc/manifest/site/internal_dns" } ]
+source.paths = [
+  { from = "smf/internal-dns", to = "/var/svc/manifest/site/internal_dns" },
+]
 output.intermediate_only = true
 output.type = "zone"
 
@@ -284,7 +287,9 @@ output.type = "zone"
 service_name = "external-dns-customizations"
 only_for_targets.image = "standard"
 source.type = "local"
-source.paths = [ { from = "smf/external-dns", to = "/var/svc/manifest/site/external_dns" } ]
+source.paths = [
+  { from = "smf/external-dns", to = "/var/svc/manifest/site/external_dns" },
+]
 output.intermediate_only = true
 output.type = "zone"
 
@@ -296,7 +301,7 @@ source.packages = [
   "ntp-svc.tar.gz",
   "opte-interface-setup.tar.gz",
   "zone-network-setup.tar.gz",
-  "zone-network-install.tar.gz"
+  "zone-network-install.tar.gz",
 ]
 output.type = "zone"
 
@@ -305,9 +310,9 @@ service_name = "ntp-svc"
 only_for_targets.image = "standard"
 source.type = "local"
 source.paths = [
-	{ from = "smf/ntp/manifest", to = "/var/svc/manifest/site/ntp" },
-	{ from = "smf/ntp/method", to = "/var/svc/method" },
-	{ from = "smf/ntp/etc", to = "/etc" },
+  { from = "smf/ntp/manifest", to = "/var/svc/manifest/site/ntp" },
+  { from = "smf/ntp/method", to = "/var/svc/method" },
+  { from = "smf/ntp/etc", to = "/etc" },
 ]
 output.intermediate_only = true
 output.type = "zone"
@@ -318,7 +323,7 @@ only_for_targets.image = "standard"
 source.type = "local"
 source.rust.binary_names = ["mgs"]
 source.rust.release = true
-source.paths = [ ]
+source.paths = []
 output.type = "zone"
 output.intermediate_only = true
 
@@ -327,7 +332,7 @@ service_name = "mgs"
 only_for_targets.image = "standard"
 only_for_targets.switch = "asic"
 source.type = "local"
-source.paths = [ { from = "smf/mgs", to = "/var/svc/manifest/site/mgs" } ]
+source.paths = [{ from = "smf/mgs", to = "/var/svc/manifest/site/mgs" }]
 output.intermediate_only = true
 output.type = "zone"
 
@@ -336,9 +341,9 @@ service_name = "mgs"
 only_for_targets.image = "standard"
 only_for_targets.switch = "stub"
 source.type = "local"
-source.paths = [ 
-  { from = "smf/mgs/manifest.xml", to = "/var/svc/manifest/site/mgs/manifest.xml" }, 
-  { from = "smf/mgs-sim/config.toml", to = "/var/svc/manifest/site/mgs/config.toml" } 
+source.paths = [
+  { from = "smf/mgs/manifest.xml", to = "/var/svc/manifest/site/mgs/manifest.xml" },
+  { from = "smf/mgs-sim/config.toml", to = "/var/svc/manifest/site/mgs/config.toml" },
 ]
 output.intermediate_only = true
 output.type = "zone"
@@ -348,9 +353,9 @@ service_name = "mgs"
 only_for_targets.image = "standard"
 only_for_targets.switch = "softnpu"
 source.type = "local"
-source.paths = [ 
-  { from = "smf/mgs/manifest.xml", to = "/var/svc/manifest/site/mgs/manifest.xml" }, 
-  { from = "smf/mgs-sim/config.toml", to = "/var/svc/manifest/site/mgs/config.toml" } 
+source.paths = [
+  { from = "smf/mgs/manifest.xml", to = "/var/svc/manifest/site/mgs/manifest.xml" },
+  { from = "smf/mgs-sim/config.toml", to = "/var/svc/manifest/site/mgs/config.toml" },
 ]
 output.intermediate_only = true
 output.type = "zone"
@@ -360,7 +365,10 @@ service_name = "mgs"
 only_for_targets.image = "standard"
 only_for_targets.switch = "asic"
 source.type = "composite"
-source.packages = [ "omicron-gateway.tar.gz", "omicron-gateway-asic-customizations.tar.gz" ]
+source.packages = [
+  "omicron-gateway.tar.gz",
+  "omicron-gateway-asic-customizations.tar.gz",
+]
 output.type = "zone"
 
 [package.omicron-gateway-stub]
@@ -368,7 +376,10 @@ service_name = "mgs"
 only_for_targets.image = "standard"
 only_for_targets.switch = "stub"
 source.type = "composite"
-source.packages = [ "omicron-gateway.tar.gz", "omicron-gateway-stub-customizations.tar.gz" ]
+source.packages = [
+  "omicron-gateway.tar.gz",
+  "omicron-gateway-stub-customizations.tar.gz",
+]
 output.type = "zone"
 
 [package.omicron-gateway-softnpu]
@@ -376,7 +387,10 @@ service_name = "mgs"
 only_for_targets.image = "standard"
 only_for_targets.switch = "softnpu"
 source.type = "composite"
-source.packages = [ "omicron-gateway.tar.gz", "omicron-gateway-softnpu-customizations.tar.gz" ]
+source.packages = [
+  "omicron-gateway.tar.gz",
+  "omicron-gateway-softnpu-customizations.tar.gz",
+]
 output.type = "zone"
 
 [package.wicketd]
@@ -385,7 +399,7 @@ only_for_targets.image = "standard"
 source.type = "local"
 source.rust.binary_names = ["wicketd"]
 source.rust.release = true
-source.paths = [ { from = "smf/wicketd", to = "/var/svc/manifest/site/wicketd" } ]
+source.paths = [{ from = "smf/wicketd", to = "/var/svc/manifest/site/wicketd" }]
 output.type = "zone"
 output.intermediate_only = true
 
@@ -395,9 +409,7 @@ only_for_targets.image = "standard"
 source.type = "local"
 source.rust.binary_names = ["wicket"]
 source.rust.release = true
-source.paths = [
-  { from = "wicket/zone-etc", to = "/etc" },
-]
+source.paths = [{ from = "wicket/zone-etc", to = "/etc" }]
 output.type = "zone"
 output.intermediate_only = true
 
@@ -419,7 +431,7 @@ only_for_targets.image = "standard"
 source.type = "local"
 source.rust.binary_names = ["sp-sim"]
 source.rust.release = true
-source.paths = [ { from = "smf/sp-sim", to = "/var/svc/manifest/site/sp-sim" } ]
+source.paths = [{ from = "smf/sp-sim", to = "/var/svc/manifest/site/sp-sim" }]
 output.type = "zone"
 output.intermediate_only = true
 
@@ -428,7 +440,7 @@ service_name = "sp-sim-customizations"
 only_for_targets.image = "standard"
 only_for_targets.switch = "stub"
 source.type = "composite"
-source.packages = [ "sp-sim.tar.gz" ]
+source.packages = ["sp-sim.tar.gz"]
 output.type = "zone"
 output.intermediate_only = true
 
@@ -437,7 +449,7 @@ service_name = "sp-sim-customizations"
 only_for_targets.image = "standard"
 only_for_targets.switch = "softnpu"
 source.type = "composite"
-source.packages = [ "sp-sim.tar.gz" ]
+source.packages = ["sp-sim.tar.gz"]
 output.type = "zone"
 output.intermediate_only = true
 
@@ -445,7 +457,11 @@ output.intermediate_only = true
 service_name = "crucible"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "crucible.tar.gz", "zone-network-setup.tar.gz", "zone-network-install.tar.gz" ]
+source.packages = [
+  "crucible.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz",
+]
 output.type = "zone"
 
 
@@ -453,7 +469,11 @@ output.type = "zone"
 service_name = "crucible_pantry"
 only_for_targets.image = "standard"
 source.type = "composite"
-source.packages = [ "crucible-pantry.tar.gz", "zone-network-setup.tar.gz", "zone-network-install.tar.gz" ]
+source.packages = [
+  "crucible-pantry.tar.gz",
+  "zone-network-setup.tar.gz",
+  "zone-network-install.tar.gz",
+]
 output.type = "zone"
 
 # Packages not built within Omicron, but which must be imported.
@@ -575,8 +595,8 @@ only_for_targets.image = "standard"
 # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "3618dd6017b363c5d34399273453cf50b9c9a43e"
-source.sha256 = "eb98985871f321411f7875ef7751dba85ae0dd3034877b63ccb78cedcb96e6e7"
+source.commit = "41ddeab9d43d90a51e6fc1c236dc9982fc76f922"
+source.sha256 = "8ebb889a555ce59cb0373a1ec9595536e015c951f6fc4d89308b4e3f09c83b20"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -600,8 +620,8 @@ only_for_targets.image = "standard"
 # 2. Copy the output zone image from dendrite/out to omicron/out
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "3618dd6017b363c5d34399273453cf50b9c9a43e"
-source.sha256 = "cc0429f0d9ce6df94e834cea89cabbdf4d1fbfe623369dd3eb84c5b2677414be"
+source.commit = "41ddeab9d43d90a51e6fc1c236dc9982fc76f922"
+source.sha256 = "3e8aa5483d22316e1fd629c77277190dafa875938a9ab3900e92a210c5e91e91"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -618,8 +638,8 @@ only_for_targets.image = "standard"
 # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "3618dd6017b363c5d34399273453cf50b9c9a43e"
-source.sha256 = "fa25585fb3aa1a888b76133af3060b859cbea8e53287bb1cc64e70889db37679"
+source.commit = "41ddeab9d43d90a51e6fc1c236dc9982fc76f922"
+source.sha256 = "5e5f2831f3c46253828ea237f701f1fa174061ab0bf73c200d31d09e94890ae7"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -629,7 +649,7 @@ only_for_targets.switch = "asic"
 only_for_targets.image = "standard"
 source.type = "local"
 source.paths = [
-  { from = "out/transceiver-control/root/opt/oxide/bin/xcvradm", to = "/opt/oxide/bin/xcvradm" }
+  { from = "out/transceiver-control/root/opt/oxide/bin/xcvradm", to = "/opt/oxide/bin/xcvradm" },
 ]
 output.type = "zone"
 output.intermediate_only = true
@@ -664,7 +684,7 @@ source.packages = [
   "mgd.tar.gz",
   "switch_zone_setup.tar.gz",
   "xcvradm.tar.gz",
-  "omicron-omdb.tar.gz"
+  "omicron-omdb.tar.gz",
 ]
 output.type = "zone"
 
@@ -688,7 +708,7 @@ source.packages = [
   "mgd.tar.gz",
   "switch_zone_setup.tar.gz",
   "sp-sim-stub.tar.gz",
-  "omicron-omdb.tar.gz"
+  "omicron-omdb.tar.gz",
 ]
 output.type = "zone"
 
@@ -712,7 +732,7 @@ source.packages = [
   "mgd.tar.gz",
   "switch_zone_setup.tar.gz",
   "sp-sim-softnpu.tar.gz",
-  "omicron-omdb.tar.gz"
+  "omicron-omdb.tar.gz",
 ]
 output.type = "zone"
 
@@ -721,7 +741,7 @@ service_name = "zone-network-setup"
 only_for_targets.image = "standard"
 source.type = "local"
 source.paths = [
-    { from = "smf/zone-network-setup/manifest.xml", to = "/var/svc/manifest/site/zone-network-setup/manifest.xml" },
+  { from = "smf/zone-network-setup/manifest.xml", to = "/var/svc/manifest/site/zone-network-setup/manifest.xml" },
 ]
 output.type = "zone"
 output.intermediate_only = true
@@ -740,7 +760,7 @@ service_name = "opte-interface-setup"
 only_for_targets.image = "standard"
 source.type = "local"
 source.paths = [
-    { from = "smf/opte-interface-setup/manifest.xml", to = "/var/svc/manifest/site/opte-interface-setup/manifest.xml" },
+  { from = "smf/opte-interface-setup/manifest.xml", to = "/var/svc/manifest/site/opte-interface-setup/manifest.xml" },
 ]
 output.type = "zone"
 output.intermediate_only = true
@@ -765,7 +785,5 @@ output.type = "tarball"
 [package.probe]
 service_name = "probe"
 source.type = "composite"
-source.packages = [
-  "thundermuffin.tar.gz",
-]
+source.packages = ["thundermuffin.tar.gz"]
 output.type = "zone"
diff --git a/schema/crdb/45.0.0/up01.sql b/schema/crdb/45.0.0/up01.sql
new file mode 100644
index 0000000000..429350dd29
--- /dev/null
+++ b/schema/crdb/45.0.0/up01.sql
@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS rack_initialized ON omicron.public.rack (initialized);
diff --git a/schema/crdb/45.0.0/up02.sql b/schema/crdb/45.0.0/up02.sql
new file mode 100644
index 0000000000..b1f4bdadff
--- /dev/null
+++ b/schema/crdb/45.0.0/up02.sql
@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS omicron.public.bootstore_config (
+    key TEXT NOT NULL,
+    generation INT8 NOT NULL,
+    PRIMARY KEY (key, generation),
+    data JSONB NOT NULL,
+    time_created TIMESTAMPTZ NOT NULL,
+    time_deleted TIMESTAMPTZ
+);
diff --git a/schema/crdb/45.0.0/up03.sql b/schema/crdb/45.0.0/up03.sql
new file mode 100644
index 0000000000..aa20ae144d
--- /dev/null
+++ b/schema/crdb/45.0.0/up03.sql
@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS address_lot_names ON omicron.public.address_lot(name);
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index c7a4db09f1..b56bec1ba9 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -243,7 +243,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.sled_underlay_subnet_allocation (
     subnet_octet INT2 NOT NULL UNIQUE CHECK (subnet_octet BETWEEN 33 AND 255)
 );
 
--- Add an index which allows pagination by {rack_id, sled_id} pairs. 
+-- Add an index which allows pagination by {rack_id, sled_id} pairs.
 CREATE UNIQUE INDEX IF NOT EXISTS lookup_subnet_allocation_by_rack_and_sled ON omicron.public.sled_underlay_subnet_allocation (
     rack_id,
     sled_id
@@ -889,7 +889,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.silo_quotas (
  * A view of the amount of provisioned and allocated (set by quotas) resources
  * on a given silo.
  */
-CREATE VIEW IF NOT EXISTS omicron.public.silo_utilization 
+CREATE VIEW IF NOT EXISTS omicron.public.silo_utilization
 AS SELECT
     c.id AS silo_id,
     s.name AS silo_name,
@@ -902,7 +902,7 @@ AS SELECT
     s.discoverable as silo_discoverable
 FROM
     omicron.public.virtual_provisioning_collection AS c
-    RIGHT JOIN omicron.public.silo_quotas AS q 
+    RIGHT JOIN omicron.public.silo_quotas AS q
     ON c.id = q.silo_id
     INNER JOIN omicron.public.silo AS s
     ON c.id = s.id
@@ -1006,7 +1006,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_instance_by_project ON omicron.public.i
     time_deleted IS NULL;
 
 /*
- * A special view of an instance provided to operators for insights into what's running 
+ * A special view of an instance provided to operators for insights into what's running
  * on a sled.
  *
  * This view requires the VMM table, which doesn't exist yet, so create a
@@ -1155,9 +1155,9 @@ SELECT
     digest,
     block_size,
     size_bytes
-FROM 
+FROM
     omicron.public.image
-WHERE 
+WHERE
     project_id IS NOT NULL;
 
 CREATE VIEW IF NOT EXISTS omicron.public.silo_image AS
@@ -1176,9 +1176,9 @@ SELECT
     digest,
     block_size,
     size_bytes
-FROM 
+FROM
     omicron.public.image
-WHERE 
+WHERE
     project_id IS NULL;
 
 /* Index for silo images */
@@ -3658,6 +3658,21 @@ CREATE TABLE IF NOT EXISTS omicron.public.downstairs_client_stopped_notification
     PRIMARY KEY (time, upstairs_id, downstairs_id, reason)
 );
 
+CREATE INDEX IF NOT EXISTS rack_initialized ON omicron.public.rack (initialized);
+
+-- table for tracking bootstore configuration changes over time
+-- this makes reconciliation easier and also gives us a visible history of changes
+CREATE TABLE IF NOT EXISTS omicron.public.bootstore_config (
+    key TEXT NOT NULL,
+    generation INT8 NOT NULL,
+    PRIMARY KEY (key, generation),
+    data JSONB NOT NULL,
+    time_created TIMESTAMPTZ NOT NULL,
+    time_deleted TIMESTAMPTZ
+);
+
+CREATE INDEX IF NOT EXISTS address_lot_names ON omicron.public.address_lot(name);
+
 /*
  * Metadata for the schema itself. This version number isn't great, as there's
  * nothing to ensure it gets bumped when it should be, but it's a start.
@@ -3692,7 +3707,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '44.0.0', NULL)
+    ( TRUE, NOW(), NOW(), '45.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;
diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs
index acad2b8d3c..c70758f775 100644
--- a/sled-agent/src/bootstrap/early_networking.rs
+++ b/sled-agent/src/bootstrap/early_networking.rs
@@ -565,11 +565,7 @@ impl<'a> EarlyNetworkSetup<'a> {
         port_config: &PortConfigV1,
     ) -> Result<(PortSettings, PortId), EarlyNetworkSetupError> {
         info!(self.log, "Building Port Configuration");
-        let mut dpd_port_settings = PortSettings {
-            links: HashMap::new(),
-            v4_routes: HashMap::new(),
-            v6_routes: HashMap::new(),
-        };
+        let mut dpd_port_settings = PortSettings { links: HashMap::new() };
         let link_id = LinkId(0);
 
         let mut addrs = Vec::new();
diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml
index 40ed41bfda..553cdb0aef 100644
--- a/smf/nexus/multi-sled/config-partial.toml
+++ b/smf/nexus/multi-sled/config-partial.toml
@@ -51,6 +51,7 @@ phantom_disks.period_secs = 30
 blueprints.period_secs_load = 10
 blueprints.period_secs_execute = 60
 sync_service_zone_nat.period_secs = 30
+switch_port_settings_manager.period_secs = 30
 region_replacement.period_secs = 30
 
 [default_region_allocation_strategy]
diff --git a/smf/nexus/single-sled/config-partial.toml b/smf/nexus/single-sled/config-partial.toml
index 2e259aa42f..9f7cb959d3 100644
--- a/smf/nexus/single-sled/config-partial.toml
+++ b/smf/nexus/single-sled/config-partial.toml
@@ -51,6 +51,7 @@ phantom_disks.period_secs = 30
 blueprints.period_secs_load = 10
 blueprints.period_secs_execute = 60
 sync_service_zone_nat.period_secs = 30
+switch_port_settings_manager.period_secs = 30
 region_replacement.period_secs = 30
 
 [default_region_allocation_strategy]
diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version
index 6895170e02..bbc2110a0a 100644
--- a/tools/dendrite_openapi_version
+++ b/tools/dendrite_openapi_version
@@ -1,2 +1,2 @@
-COMMIT="3618dd6017b363c5d34399273453cf50b9c9a43e"
-SHA2="aa670165e5b459fab4caba36ae4d382a09264ff5cf6a2dac0dae0a0db39a378e"
+COMMIT="41ddeab9d43d90a51e6fc1c236dc9982fc76f922"
+SHA2="50eff6d9f986b7b1af5970d11d8d01b812de37269731c6c691a244b3fdae82ae"
diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums
index 74b379f359..86cf1a56ec 100644
--- a/tools/dendrite_stub_checksums
+++ b/tools/dendrite_stub_checksums
@@ -1,3 +1,3 @@
-CIDL_SHA256_ILLUMOS="eb98985871f321411f7875ef7751dba85ae0dd3034877b63ccb78cedcb96e6e7"
-CIDL_SHA256_LINUX_DPD="cb9a1978d1fe3a3f2391757f80436d8cc87c0041161652ad2234e7cf83e9ae36"
-CIDL_SHA256_LINUX_SWADM="b7e737be56a8a815a95624f0b5c42ce1e339b07feeae7b3d7b9b4bc17c204245"
+CIDL_SHA256_ILLUMOS="8ebb889a555ce59cb0373a1ec9595536e015c951f6fc4d89308b4e3f09c83b20"
+CIDL_SHA256_LINUX_DPD="f753444cae478cdedcde743a20a9df5965ed28cddab0f9632f3c263c66cd6397"
+CIDL_SHA256_LINUX_SWADM="66eab497b955751d0704c3cd97ac5c1ed373aa656fc37ccba86ae9900b5ae96d"
diff --git a/tools/virtual_hardware.sh b/tools/virtual_hardware.sh
index 119b64ac70..ade7ac58b3 100755
--- a/tools/virtual_hardware.sh
+++ b/tools/virtual_hardware.sh
@@ -41,7 +41,7 @@ function ensure_zpools {
             fi
             success "ZFS vdev $VDEV_PATH exists"
             if [[ -z "$(zpool list -o name | grep $ZPOOL)" ]]; then
-                zpool create -f "$ZPOOL" "$VDEV_PATH"
+                zpool create -o ashift=12 -f "$ZPOOL" "$VDEV_PATH"
             fi
             success "ZFS zpool $ZPOOL exists"
         done
diff --git a/wicketd/src/preflight_check/uplink.rs b/wicketd/src/preflight_check/uplink.rs
index 31d479a5ed..984b14ff3f 100644
--- a/wicketd/src/preflight_check/uplink.rs
+++ b/wicketd/src/preflight_check/uplink.rs
@@ -11,7 +11,6 @@ use dpd_client::types::PortFec as DpdPortFec;
 use dpd_client::types::PortId;
 use dpd_client::types::PortSettings;
 use dpd_client::types::PortSpeed as DpdPortSpeed;
-use dpd_client::types::RouteSettingsV4;
 use dpd_client::Client as DpdClient;
 use dpd_client::ClientState as DpdClientState;
 use either::Either;
@@ -723,11 +722,7 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>(
                     .port_settings_apply(
                         &port_id,
                         Some(OMICRON_DPD_TAG),
-                        &PortSettings {
-                            links: HashMap::new(),
-                            v4_routes: HashMap::new(),
-                            v6_routes: HashMap::new(),
-                        },
+                        &PortSettings { links: HashMap::new() },
                     )
                     .await
                     .map_err(|err| {
@@ -765,11 +760,7 @@ fn build_port_settings(
         OmicronPortSpeed::Speed400G => DpdPortSpeed::Speed400G,
     };
 
-    let mut port_settings = PortSettings {
-        links: HashMap::new(),
-        v4_routes: HashMap::new(),
-        v6_routes: HashMap::new(),
-    };
+    let mut port_settings = PortSettings { links: HashMap::new() };
 
     let addrs = uplink.addresses.iter().map(|a| a.ip()).collect();
 
@@ -788,13 +779,10 @@ fn build_port_settings(
     );
 
     for r in &uplink.routes {
-        if let (IpNetwork::V4(dst), IpAddr::V4(nexthop)) =
+        if let (IpNetwork::V4(_dst), IpAddr::V4(_nexthop)) =
             (r.destination, r.nexthop)
         {
-            port_settings.v4_routes.insert(
-                dst.to_string(),
-                vec![RouteSettingsV4 { link_id: link_id.0, nexthop }],
-            );
+            // TODO: do we need to create config for mgd?
         }
     }
 

From 274edc009d979e4443112ab56095444219f60029 Mon Sep 17 00:00:00 2001
From: Laura Abbott <labbott@users.noreply.github.com>
Date: Mon, 18 Mar 2024 16:29:32 -0400
Subject: [PATCH 125/157] SPs v1.0.11 (#5280)

---
 tools/hubris_checksums | 16 ++++++++--------
 tools/hubris_version   |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tools/hubris_checksums b/tools/hubris_checksums
index 83a22f9a43..6f47ff9f93 100644
--- a/tools/hubris_checksums
+++ b/tools/hubris_checksums
@@ -1,8 +1,8 @@
-05e4d531c2db947379c08bda099a53055e18e86d4c7fbcba1004f7f7983da701  build-gimlet-c-image-default-v1.0.10.zip
-9cba42b8d6c4dbd1bcc805b44e14e6bac7fa590f1faaaa47664f7aa254e91602  build-gimlet-d-image-default-v1.0.10.zip
-3ace1b1d692e3c23815665b932dadb42b58abc1942fdd4564c1884a13fa0f0cb  build-gimlet-e-image-default-v1.0.10.zip
-b3b05e1926bb0f66be8587a67084c76cd760da5db0f1e288c265a03cf80637e9  build-gimlet-f-image-default-v1.0.10.zip
-2d1313c4f282c49a411c32c643347dcb63bb063ae47539b2a1e4003da5a20402  build-psc-b-image-default-v1.0.10.zip
-bb16ee95be863fe5bb7a155d07732a42fc56178ce08bcb9323f46315e9616e62  build-psc-c-image-default-v1.0.10.zip
-97763288ca52a779334422cd7f3aff892341f38f79d1022a1d7ff49b85fb8356  build-sidecar-b-image-default-v1.0.10.zip
-814098557e67f5f8720e95fb76b4f694e7edbd26cd3aa120266b4238641d7029  build-sidecar-c-image-default-v1.0.10.zip
+4ad8e4bb4e96bafe988c4b26a0e39ff147d7ba3fd0fd04460dad20c9bdd4be93  build-gimlet-c-image-default-v1.0.11.zip
+171ee7fbff449447602b84c0e4d6f25c732a00485aba929add5bf5fd6209c8cb  build-gimlet-d-image-default-v1.0.11.zip
+0b48a4487d4683b09522aea7f66ac0be2977bcc0c0c93d1c6806c2dd3f58904f  build-gimlet-e-image-default-v1.0.11.zip
+3dcac446c7451ae4cbdd1ddf4081e1d9e9461d0935f67535bd8161fb9e84625d  build-gimlet-f-image-default-v1.0.11.zip
+060757892627a8c2bddf58621778e2bef825c3cc34906a81866486c413f917a3  build-psc-b-image-default-v1.0.11.zip
+fd712fe842a91b2fa60e680d4433d4951fd105c0fa81c7441a01f9208da8621d  build-psc-c-image-default-v1.0.11.zip
+9fe7856b0e92d03cd4ea2268d327af2479ae0a53c07cb820946b6b15c215d2a4  build-sidecar-b-image-default-v1.0.11.zip
+41ff08e08384f5015fd6a3d2ad07d48f159333087e89e88d58b2bfc4b6da9ef7  build-sidecar-c-image-default-v1.0.11.zip
diff --git a/tools/hubris_version b/tools/hubris_version
index 8098ff8759..8bc373efcd 100644
--- a/tools/hubris_version
+++ b/tools/hubris_version
@@ -1 +1 @@
-TAGS=(gimlet-v1.0.10 psc-v1.0.10 sidecar-v1.0.10)
+TAGS=(gimlet-v1.0.11 psc-v1.0.11 sidecar-v1.0.11)

From 01c57c514b25654cbc06e199aa36276c6a6b0e17 Mon Sep 17 00:00:00 2001
From: David Crespo <david-crespo@users.noreply.github.com>
Date: Mon, 18 Mar 2024 16:43:22 -0500
Subject: [PATCH 126/157] Current IP pool utilization endpoint (#5258)

Closes #5189


The strangest thing here is that IPv6 `allocated` and `capacity` are
both u128 that we put in the JSON spec as `"type": "string", "format":
"uint128"`, where `uint128` is a custom format our client generators
will have to look for and handle. `capacity` is huge because IPv6 ranges
are huge. IPv4 allocated and capacity are `u32`.

~~The strangest thing here is the fact that we have to handle the
unlikely possibility that an IPv6 range is too large for us to serialize
its length in a JSON response. IPv6 range size is a `u128`, and we seem
to be capping our numbers in API responses at `u32`. My proposal for
handling this is simply to make the field nullable in the response, and
we make clear that a null value means it was too large to serialize. I
think this is fine because this should only happen if the operator does
something really bad, and it is well understood that the operator can
mess up networking with a misconfiguration. Another approach would be to
always return the size as a string, but I think given the unlikelihood
of the number being too large (and the fact that it's pathological if it
is) means returning a numeric string puts undue burden on clients to
handle it.~~
---
 common/src/address.rs                         |  33 +++
 nexus/db-model/src/utilization.rs             |  37 +++
 nexus/db-queries/src/db/datastore/ip_pool.rs  | 241 +++++++++++++++++-
 nexus/src/app/utilization.rs                  |  54 ++++
 nexus/src/external_api/http_entrypoints.rs    |  26 ++
 nexus/test-utils/src/resource_helpers.rs      |  32 +++
 nexus/tests/integration_tests/endpoints.rs    |  12 +
 nexus/tests/integration_tests/external_ips.rs |  34 +++
 nexus/tests/integration_tests/instances.rs    |  42 ++-
 nexus/tests/integration_tests/ip_pools.rs     |  42 +++
 nexus/tests/output/nexus_tags.txt             |   1 +
 nexus/types/src/external_api/views.rs         |  76 ++++++
 openapi/nexus.json                            | 103 ++++++++
 13 files changed, 726 insertions(+), 7 deletions(-)

diff --git a/common/src/address.rs b/common/src/address.rs
index 48ec7c46ed..8e12810343 100644
--- a/common/src/address.rs
+++ b/common/src/address.rs
@@ -421,6 +421,14 @@ impl IpRange {
             IpRange::V6(ip6) => IpRangeIter::V6(ip6.iter()),
         }
     }
+
+    // Has to be u128 to accommodate IPv6
+    pub fn len(&self) -> u128 {
+        match self {
+            IpRange::V4(ip4) => u128::from(ip4.len()),
+            IpRange::V6(ip6) => ip6.len(),
+        }
+    }
 }
 
 impl From<IpAddr> for IpRange {
@@ -508,6 +516,12 @@ impl Ipv4Range {
     pub fn iter(&self) -> Ipv4RangeIter {
         Ipv4RangeIter { next: Some(self.first.into()), last: self.last.into() }
     }
+
+    pub fn len(&self) -> u32 {
+        let start_num = u32::from(self.first);
+        let end_num = u32::from(self.last);
+        end_num - start_num + 1
+    }
 }
 
 impl From<Ipv4Addr> for Ipv4Range {
@@ -564,6 +578,12 @@ impl Ipv6Range {
     pub fn iter(&self) -> Ipv6RangeIter {
         Ipv6RangeIter { next: Some(self.first.into()), last: self.last.into() }
     }
+
+    pub fn len(&self) -> u128 {
+        let start_num = u128::from(self.first);
+        let end_num = u128::from(self.last);
+        end_num - start_num + 1
+    }
 }
 
 impl From<Ipv6Addr> for Ipv6Range {
@@ -781,6 +801,19 @@ mod test {
         );
     }
 
+    #[test]
+    fn test_ip_range_length() {
+        let lo = Ipv4Addr::new(10, 0, 0, 1);
+        let hi = Ipv4Addr::new(10, 0, 0, 3);
+        let range = IpRange::try_from((lo, hi)).unwrap();
+        assert_eq!(range.len(), 3);
+
+        let lo = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1);
+        let hi = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 1, 3);
+        let range = IpRange::try_from((lo, hi)).unwrap();
+        assert_eq!(range.len(), 2u128.pow(16) + 3);
+    }
+
     #[test]
     fn test_ipv6_subnet_deserialize() {
         let value = json!({
diff --git a/nexus/db-model/src/utilization.rs b/nexus/db-model/src/utilization.rs
index b0e6324bc9..e82600d64d 100644
--- a/nexus/db-model/src/utilization.rs
+++ b/nexus/db-model/src/utilization.rs
@@ -55,3 +55,40 @@ impl From<SiloUtilization> for views::Utilization {
         }
     }
 }
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Ipv4Utilization {
+    pub allocated: u32,
+    pub capacity: u32,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Ipv6Utilization {
+    pub allocated: u128,
+    pub capacity: u128,
+}
+
+// Not really a DB model, just the result of a datastore function
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct IpPoolUtilization {
+    pub ipv4: Ipv4Utilization,
+    pub ipv6: Ipv6Utilization,
+}
+
+impl From<Ipv4Utilization> for views::Ipv4Utilization {
+    fn from(util: Ipv4Utilization) -> Self {
+        Self { allocated: util.allocated, capacity: util.capacity }
+    }
+}
+
+impl From<Ipv6Utilization> for views::Ipv6Utilization {
+    fn from(util: Ipv6Utilization) -> Self {
+        Self { allocated: util.allocated, capacity: util.capacity }
+    }
+}
+
+impl From<IpPoolUtilization> for views::IpPoolUtilization {
+    fn from(util: IpPoolUtilization) -> Self {
+        Self { ipv4: util.ipv4.into(), ipv6: util.ipv6.into() }
+    }
+}
diff --git a/nexus/db-queries/src/db/datastore/ip_pool.rs b/nexus/db-queries/src/db/datastore/ip_pool.rs
index a9062e6ee3..c0f4e0eea8 100644
--- a/nexus/db-queries/src/db/datastore/ip_pool.rs
+++ b/nexus/db-queries/src/db/datastore/ip_pool.rs
@@ -51,6 +51,16 @@ use omicron_common::api::external::UpdateResult;
 use ref_cast::RefCast;
 use uuid::Uuid;
 
+pub struct IpsAllocated {
+    pub ipv4: i64,
+    pub ipv6: i64,
+}
+
+pub struct IpsCapacity {
+    pub ipv4: u32,
+    pub ipv6: u128,
+}
+
 impl DataStore {
     /// List IP Pools
     pub async fn ip_pools_list(
@@ -324,6 +334,80 @@ impl DataStore {
             })
     }
 
+    pub async fn ip_pool_allocated_count(
+        &self,
+        opctx: &OpContext,
+        authz_pool: &authz::IpPool,
+    ) -> Result<IpsAllocated, Error> {
+        opctx.authorize(authz::Action::Read, authz_pool).await?;
+
+        use db::schema::external_ip;
+        use diesel::dsl::sql;
+        use diesel::sql_types::BigInt;
+
+        let (ipv4, ipv6) = external_ip::table
+            .filter(external_ip::ip_pool_id.eq(authz_pool.id()))
+            .filter(external_ip::time_deleted.is_null())
+            .select((
+                sql::<BigInt>("count(*) FILTER (WHERE family(ip) = 4)"),
+                sql::<BigInt>("count(*) FILTER (WHERE family(ip) = 6)"),
+            ))
+            .first_async::<(i64, i64)>(
+                &*self.pool_connection_authorized(opctx).await?,
+            )
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        Ok(IpsAllocated { ipv4, ipv6 })
+    }
+
+    pub async fn ip_pool_total_capacity(
+        &self,
+        opctx: &OpContext,
+        authz_pool: &authz::IpPool,
+    ) -> Result<IpsCapacity, Error> {
+        opctx.authorize(authz::Action::Read, authz_pool).await?;
+        opctx.authorize(authz::Action::ListChildren, authz_pool).await?;
+
+        use db::schema::ip_pool_range;
+
+        let ranges = ip_pool_range::table
+            .filter(ip_pool_range::ip_pool_id.eq(authz_pool.id()))
+            .filter(ip_pool_range::time_deleted.is_null())
+            .select(IpPoolRange::as_select())
+            // This is a rare unpaginated DB query, which means we are
+            // vulnerable to a resource exhaustion attack in which someone
+            // creates a very large number of ranges in order to make this
+            // query slow. In order to mitigate that, we limit the query to the
+            // (current) max allowed page size, effectively making this query
+            // exactly as vulnerable as if it were paginated. If there are more
+            // than 10,000 ranges in a pool, we will undercount, but I have a
+            // hard time seeing that as a practical problem.
+            .limit(10000)
+            .get_results_async::<IpPoolRange>(
+                &*self.pool_connection_authorized(opctx).await?,
+            )
+            .await
+            .map_err(|e| {
+                public_error_from_diesel(
+                    e,
+                    ErrorHandler::NotFoundByResource(authz_pool),
+                )
+            })?;
+
+        let mut ipv4: u32 = 0;
+        let mut ipv6: u128 = 0;
+
+        for range in &ranges {
+            let r = IpRange::from(range);
+            match r {
+                IpRange::V4(r) => ipv4 += r.len(),
+                IpRange::V6(r) => ipv6 += r.len(),
+            }
+        }
+        Ok(IpsCapacity { ipv4, ipv6 })
+    }
+
     pub async fn ip_pool_silo_list(
         &self,
         opctx: &OpContext,
@@ -846,10 +930,14 @@ mod test {
 
     use crate::authz;
     use crate::db::datastore::test_utils::datastore_test;
-    use crate::db::model::{IpPool, IpPoolResource, IpPoolResourceType};
+    use crate::db::model::{
+        IpPool, IpPoolResource, IpPoolResourceType, Project,
+    };
     use assert_matches::assert_matches;
     use nexus_test_utils::db::test_setup_database;
+    use nexus_types::external_api::params;
     use nexus_types::identity::Resource;
+    use omicron_common::address::{IpRange, Ipv4Range, Ipv6Range};
     use omicron_common::api::external::http_pagination::PaginatedBy;
     use omicron_common::api::external::{
         DataPageParams, Error, IdentityMetadataCreateParams, LookupType,
@@ -1075,4 +1163,155 @@ mod test {
         db.cleanup().await.unwrap();
         logctx.cleanup_successful();
     }
+
+    #[tokio::test]
+    async fn test_ip_pool_utilization() {
+        let logctx = dev::test_setup_log("test_ip_utilization");
+        let mut db = test_setup_database(&logctx.log).await;
+        let (opctx, datastore) = datastore_test(&logctx, &db).await;
+
+        let authz_silo = opctx.authn.silo_required().unwrap();
+        let project = Project::new(
+            authz_silo.id(),
+            params::ProjectCreate {
+                identity: IdentityMetadataCreateParams {
+                    name: "my-project".parse().unwrap(),
+                    description: "".to_string(),
+                },
+            },
+        );
+        let (.., project) =
+            datastore.project_create(&opctx, project).await.unwrap();
+
+        // create an IP pool for the silo, add a range to it, and link it to the silo
+        let identity = IdentityMetadataCreateParams {
+            name: "my-pool".parse().unwrap(),
+            description: "".to_string(),
+        };
+        let pool = datastore
+            .ip_pool_create(&opctx, IpPool::new(&identity))
+            .await
+            .expect("Failed to create IP pool");
+        let authz_pool = authz::IpPool::new(
+            authz::FLEET,
+            pool.id(),
+            LookupType::ById(pool.id()),
+        );
+
+        // capacity of zero because there are no ranges
+        let max_ips = datastore
+            .ip_pool_total_capacity(&opctx, &authz_pool)
+            .await
+            .unwrap();
+        assert_eq!(max_ips.ipv4, 0);
+        assert_eq!(max_ips.ipv6, 0);
+
+        let range = IpRange::V4(
+            Ipv4Range::new(
+                std::net::Ipv4Addr::new(10, 0, 0, 1),
+                std::net::Ipv4Addr::new(10, 0, 0, 5),
+            )
+            .unwrap(),
+        );
+        datastore
+            .ip_pool_add_range(&opctx, &authz_pool, &range)
+            .await
+            .expect("Could not add range");
+
+        // now it has a capacity of 5 because we added the range
+        let max_ips = datastore
+            .ip_pool_total_capacity(&opctx, &authz_pool)
+            .await
+            .unwrap();
+        assert_eq!(max_ips.ipv4, 5);
+        assert_eq!(max_ips.ipv6, 0);
+
+        let link = IpPoolResource {
+            ip_pool_id: pool.id(),
+            resource_type: IpPoolResourceType::Silo,
+            resource_id: authz_silo.id(),
+            is_default: true,
+        };
+        datastore
+            .ip_pool_link_silo(&opctx, link)
+            .await
+            .expect("Could not link pool to silo");
+
+        let ip_count = datastore
+            .ip_pool_allocated_count(&opctx, &authz_pool)
+            .await
+            .unwrap();
+        assert_eq!(ip_count.ipv4, 0);
+        assert_eq!(ip_count.ipv6, 0);
+
+        let identity = IdentityMetadataCreateParams {
+            name: "my-ip".parse().unwrap(),
+            description: "".to_string(),
+        };
+        let ip = datastore
+            .allocate_floating_ip(&opctx, project.id(), identity, None, None)
+            .await
+            .expect("Could not allocate floating IP");
+        assert_eq!(ip.ip.to_string(), "10.0.0.1/32");
+
+        let ip_count = datastore
+            .ip_pool_allocated_count(&opctx, &authz_pool)
+            .await
+            .unwrap();
+        assert_eq!(ip_count.ipv4, 1);
+        assert_eq!(ip_count.ipv6, 0);
+
+        // allocating one has nothing to do with total capacity
+        let max_ips = datastore
+            .ip_pool_total_capacity(&opctx, &authz_pool)
+            .await
+            .unwrap();
+        assert_eq!(max_ips.ipv4, 5);
+        assert_eq!(max_ips.ipv6, 0);
+
+        let ipv6_range = IpRange::V6(
+            Ipv6Range::new(
+                std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 10),
+                std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 1, 20),
+            )
+            .unwrap(),
+        );
+        datastore
+            .ip_pool_add_range(&opctx, &authz_pool, &ipv6_range)
+            .await
+            .expect("Could not add range");
+
+        // now test with additional v6 range
+        let max_ips = datastore
+            .ip_pool_total_capacity(&opctx, &authz_pool)
+            .await
+            .unwrap();
+        assert_eq!(max_ips.ipv4, 5);
+        assert_eq!(max_ips.ipv6, 11 + 65536);
+
+        // add a giant range for fun
+        let ipv6_range = IpRange::V6(
+            Ipv6Range::new(
+                std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 1, 21),
+                std::net::Ipv6Addr::new(
+                    0xfd00, 0, 0, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
+                ),
+            )
+            .unwrap(),
+        );
+        datastore
+            .ip_pool_add_range(&opctx, &authz_pool, &ipv6_range)
+            .await
+            .expect("Could not add range");
+
+        let max_ips = datastore
+            .ip_pool_total_capacity(&opctx, &authz_pool)
+            .await
+            .unwrap();
+        assert_eq!(max_ips.ipv4, 5);
+        assert_eq!(max_ips.ipv6, 1208925819614629174706166);
+
+        db.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
 }
diff --git a/nexus/src/app/utilization.rs b/nexus/src/app/utilization.rs
index 526ebc9470..bd5f5d002a 100644
--- a/nexus/src/app/utilization.rs
+++ b/nexus/src/app/utilization.rs
@@ -4,6 +4,9 @@
 
 //! Insights into capacity and utilization
 
+use nexus_db_model::IpPoolUtilization;
+use nexus_db_model::Ipv4Utilization;
+use nexus_db_model::Ipv6Utilization;
 use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db;
@@ -30,4 +33,55 @@ impl super::Nexus {
     ) -> ListResultVec<db::model::SiloUtilization> {
         self.db_datastore.silo_utilization_list(opctx, pagparams).await
     }
+
+    pub async fn ip_pool_utilization_view(
+        &self,
+        opctx: &OpContext,
+        pool_lookup: &lookup::IpPool<'_>,
+    ) -> Result<IpPoolUtilization, Error> {
+        let (.., authz_pool) =
+            pool_lookup.lookup_for(authz::Action::Read).await?;
+
+        let allocated = self
+            .db_datastore
+            .ip_pool_allocated_count(opctx, &authz_pool)
+            .await?;
+        let capacity = self
+            .db_datastore
+            .ip_pool_total_capacity(opctx, &authz_pool)
+            .await?;
+
+        // we have one query for v4 and v6 allocated and one for both v4 and
+        // v6 capacity so we can do two queries instead 4, but in the response
+        // we want them paired by IP version
+        Ok(IpPoolUtilization {
+            ipv4: Ipv4Utilization {
+                // This one less trivial to justify than the u128 conversion
+                // below because an i64 could obviously be too big for u32.
+                // In addition to the fact that it is unlikely for anyone to
+                // allocate 4 billion IPs, we rely on the fact that there can
+                // only be 2^32 IPv4 addresses, period.
+                allocated: u32::try_from(allocated.ipv4).map_err(|_e| {
+                    Error::internal_error(&format!(
+                        "Failed to convert i64 {} IPv4 address count to u32",
+                        allocated.ipv4
+                    ))
+                })?,
+                capacity: capacity.ipv4,
+            },
+            ipv6: Ipv6Utilization {
+                // SQL represents counts as signed integers for historical
+                // or compatibility reasons even though they can't really be
+                // negative, and Diesel follows that. We assume here that it
+                // will be a positive i64.
+                allocated: u128::try_from(allocated.ipv6).map_err(|_e| {
+                    Error::internal_error(&format!(
+                        "Failed to convert i64 {} IPv6 address count to u128",
+                        allocated.ipv6
+                    ))
+                })?,
+                capacity: capacity.ipv6,
+            },
+        })
+    }
 }
diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs
index aa037e072f..9c8d9cef50 100644
--- a/nexus/src/external_api/http_entrypoints.rs
+++ b/nexus/src/external_api/http_entrypoints.rs
@@ -130,6 +130,7 @@ pub(crate) fn external_api() -> NexusApiDescription {
         api.register(ip_pool_update)?;
         // Variants for internal services
         api.register(ip_pool_service_view)?;
+        api.register(ip_pool_utilization_view)?;
 
         // Operator-Accessible IP Pool Range API
         api.register(ip_pool_range_list)?;
@@ -1557,6 +1558,31 @@ async fn ip_pool_update(
     apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
 }
 
+/// Fetch IP pool utilization
+#[endpoint {
+    method = GET,
+    path = "/v1/system/ip-pools/{pool}/utilization",
+    tags = ["system/networking"],
+}]
+async fn ip_pool_utilization_view(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    path_params: Path<params::IpPoolPath>,
+) -> Result<HttpResponseOk<views::IpPoolUtilization>, HttpError> {
+    let apictx = rqctx.context();
+    let handler = async {
+        let opctx = crate::context::op_context_for_external_api(&rqctx).await?;
+        let nexus = &apictx.nexus;
+        let pool_selector = path_params.into_inner().pool;
+        // We do not prevent the service pool from being fetched by name or ID
+        // like we do for update, delete, associate.
+        let pool_lookup = nexus.ip_pool_lookup(&opctx, &pool_selector)?;
+        let utilization =
+            nexus.ip_pool_utilization_view(&opctx, &pool_lookup).await?;
+        Ok(HttpResponseOk(utilization.into()))
+    };
+    apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
 /// List IP pool's linked silos
 #[endpoint {
     method = GET,
diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs
index f4b5d9fa46..b67028a996 100644
--- a/nexus/test-utils/src/resource_helpers.rs
+++ b/nexus/test-utils/src/resource_helpers.rs
@@ -631,6 +631,38 @@ pub async fn create_router(
     .unwrap()
 }
 
+pub async fn assert_ip_pool_utilization(
+    client: &ClientTestContext,
+    pool_name: &str,
+    ipv4_allocated: u32,
+    ipv4_capacity: u32,
+    ipv6_allocated: u128,
+    ipv6_capacity: u128,
+) {
+    let url = format!("/v1/system/ip-pools/{}/utilization", pool_name);
+    let utilization: views::IpPoolUtilization = object_get(client, &url).await;
+    assert_eq!(
+        utilization.ipv4.allocated, ipv4_allocated,
+        "IP pool '{}': expected {} IPv4 allocated, got {:?}",
+        pool_name, ipv4_allocated, utilization.ipv4.allocated
+    );
+    assert_eq!(
+        utilization.ipv4.capacity, ipv4_capacity,
+        "IP pool '{}': expected {} IPv4 capacity, got {:?}",
+        pool_name, ipv4_capacity, utilization.ipv4.capacity
+    );
+    assert_eq!(
+        utilization.ipv6.allocated, ipv6_allocated,
+        "IP pool '{}': expected {} IPv6 allocated, got {:?}",
+        pool_name, ipv6_allocated, utilization.ipv6.allocated
+    );
+    assert_eq!(
+        utilization.ipv6.capacity, ipv6_capacity,
+        "IP pool '{}': expected {} IPv6 capacity, got {:?}",
+        pool_name, ipv6_capacity, utilization.ipv6.capacity
+    );
+}
+
 /// Grant a role on a resource to a user
 ///
 /// * `grant_resource_url`: URL of the resource we're granting the role on
diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs
index f18b2d961d..7d2ff2596a 100644
--- a/nexus/tests/integration_tests/endpoints.rs
+++ b/nexus/tests/integration_tests/endpoints.rs
@@ -656,6 +656,8 @@ pub static DEMO_IP_POOL_PROJ_URL: Lazy<String> = Lazy::new(|| {
 });
 pub static DEMO_IP_POOL_URL: Lazy<String> =
     Lazy::new(|| format!("/v1/system/ip-pools/{}", *DEMO_IP_POOL_NAME));
+pub static DEMO_IP_POOL_UTILIZATION_URL: Lazy<String> =
+    Lazy::new(|| format!("{}/utilization", *DEMO_IP_POOL_URL));
 pub static DEMO_IP_POOL_UPDATE: Lazy<params::IpPoolUpdate> =
     Lazy::new(|| params::IpPoolUpdate {
         identity: IdentityMetadataUpdateParams {
@@ -1104,6 +1106,16 @@ pub static VERIFY_ENDPOINTS: Lazy<Vec<VerifyEndpoint>> = Lazy::new(|| {
             ],
         },
 
+        // IP pool utilization
+        VerifyEndpoint {
+            url: &DEMO_IP_POOL_UTILIZATION_URL,
+            visibility: Visibility::Protected,
+            unprivileged_access: UnprivilegedAccess::None,
+            allowed_methods: vec![
+                AllowedMethod::Get,
+            ],
+        },
+
         // IP Pool endpoint (Oxide services)
         VerifyEndpoint {
             url: &DEMO_IP_POOL_SERVICE_URL,
diff --git a/nexus/tests/integration_tests/external_ips.rs b/nexus/tests/integration_tests/external_ips.rs
index 25c74bfd7f..fa884f8731 100644
--- a/nexus/tests/integration_tests/external_ips.rs
+++ b/nexus/tests/integration_tests/external_ips.rs
@@ -17,6 +17,7 @@ use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO;
 use nexus_test_utils::http_testing::AuthnMode;
 use nexus_test_utils::http_testing::NexusRequest;
 use nexus_test_utils::http_testing::RequestBuilder;
+use nexus_test_utils::resource_helpers::assert_ip_pool_utilization;
 use nexus_test_utils::resource_helpers::create_default_ip_pool;
 use nexus_test_utils::resource_helpers::create_floating_ip;
 use nexus_test_utils::resource_helpers::create_instance_with;
@@ -151,6 +152,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) {
     // automatically linked to current silo
     create_default_ip_pool(&client).await;
 
+    assert_ip_pool_utilization(client, "default", 0, 65536, 0, 0).await;
+
     let other_pool_range = IpRange::V4(
         Ipv4Range::new(Ipv4Addr::new(10, 1, 0, 1), Ipv4Addr::new(10, 1, 0, 5))
             .unwrap(),
@@ -158,6 +161,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) {
     // not automatically linked to currently silo. see below
     create_ip_pool(&client, "other-pool", Some(other_pool_range)).await;
 
+    assert_ip_pool_utilization(client, "other-pool", 0, 5, 0, 0).await;
+
     let project = create_project(client, PROJECT_NAME).await;
 
     // Create with no chosen IP and fallback to default pool.
@@ -175,6 +180,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) {
     assert_eq!(fip.instance_id, None);
     assert_eq!(fip.ip, IpAddr::from(Ipv4Addr::new(10, 0, 0, 0)));
 
+    assert_ip_pool_utilization(client, "default", 1, 65536, 0, 0).await;
+
     // Create with chosen IP and fallback to default pool.
     let fip_name = FIP_NAMES[1];
     let ip_addr = "10.0.12.34".parse().unwrap();
@@ -191,6 +198,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) {
     assert_eq!(fip.instance_id, None);
     assert_eq!(fip.ip, ip_addr);
 
+    assert_ip_pool_utilization(client, "default", 2, 65536, 0, 0).await;
+
     // Creating with other-pool fails with 404 until it is linked to the current silo
     let fip_name = FIP_NAMES[2];
     let params = params::FloatingIpCreate {
@@ -206,6 +215,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) {
         object_create_error(client, &url, &params, StatusCode::NOT_FOUND).await;
     assert_eq!(error.message, "not found: ip-pool with name \"other-pool\"");
 
+    assert_ip_pool_utilization(client, "other-pool", 0, 5, 0, 0).await;
+
     // now link the pool and everything should work with the exact same params
     let silo_id = DEFAULT_SILO.id();
     link_ip_pool(&client, "other-pool", &silo_id, false).await;
@@ -217,6 +228,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) {
     assert_eq!(fip.instance_id, None);
     assert_eq!(fip.ip, IpAddr::from(Ipv4Addr::new(10, 1, 0, 1)));
 
+    assert_ip_pool_utilization(client, "other-pool", 1, 5, 0, 0).await;
+
     // Create with chosen IP from fleet-scoped named pool.
     let fip_name = FIP_NAMES[3];
     let ip_addr = "10.1.0.5".parse().unwrap();
@@ -232,6 +245,8 @@ async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) {
     assert_eq!(fip.project_id, project.identity.id);
     assert_eq!(fip.instance_id, None);
     assert_eq!(fip.ip, ip_addr);
+
+    assert_ip_pool_utilization(client, "other-pool", 2, 5, 0, 0).await;
 }
 
 #[nexus_test]
@@ -586,6 +601,8 @@ async fn test_external_ip_live_attach_detach(
     create_default_ip_pool(&client).await;
     let project = create_project(client, PROJECT_NAME).await;
 
+    assert_ip_pool_utilization(client, "default", 0, 65536, 0, 0).await;
+
     // Create 2 instances, and a floating IP for each instance.
     // One instance will be started, and one will be stopped.
     let mut fips = vec![];
@@ -602,6 +619,9 @@ async fn test_external_ip_live_attach_detach(
         );
     }
 
+    // 2 floating IPs have been allocated
+    assert_ip_pool_utilization(client, "default", 2, 65536, 0, 0).await;
+
     let mut instances = vec![];
     for (i, start) in [false, true].iter().enumerate() {
         let instance = instance_for_external_ips(
@@ -633,6 +653,10 @@ async fn test_external_ip_live_attach_detach(
         instances.push(instance);
     }
 
+    // the two instances above were deliberately not given ephemeral IPs, but
+    // they still always get SNAT IPs, so we went from 2 to 4
+    assert_ip_pool_utilization(client, "default", 4, 65536, 0, 0).await;
+
     // Attach a floating IP and ephemeral IP to each instance.
     let mut recorded_ephs = vec![];
     for (instance, fip) in instances.iter().zip(&fips) {
@@ -675,6 +699,10 @@ async fn test_external_ip_live_attach_detach(
         recorded_ephs.push(eph_resp);
     }
 
+    // now 6 because an ephemeral IP was added for each instance. floating IPs
+    // were attached, but they were already allocated
+    assert_ip_pool_utilization(client, "default", 6, 65536, 0, 0).await;
+
     // Detach a floating IP and ephemeral IP from each instance.
     for (instance, fip) in instances.iter().zip(&fips) {
         let instance_name = instance.identity.name.as_str();
@@ -705,6 +733,9 @@ async fn test_external_ip_live_attach_detach(
         );
     }
 
+    // 2 ephemeral go away on detachment but still 2 floating and 2 SNAT
+    assert_ip_pool_utilization(client, "default", 4, 65536, 0, 0).await;
+
     // Finally, two kind of funny tests. There is special logic in the handler
     // for the case where the floating IP is specified by name but the instance
     // by ID and vice versa, so we want to test both combinations.
@@ -763,6 +794,9 @@ async fn test_external_ip_live_attach_detach(
         fetch_instance_external_ips(client, instance_name, PROJECT_NAME).await;
     assert_eq!(eip_list.len(), 1);
     assert_eq!(eip_list[0].ip(), fips[1].ip);
+
+    // none of that changed the number of allocated IPs
+    assert_ip_pool_utilization(client, "default", 4, 65536, 0, 0).await;
 }
 
 #[nexus_test]
diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs
index 309036256f..5d8bcc2136 100644
--- a/nexus/tests/integration_tests/instances.rs
+++ b/nexus/tests/integration_tests/instances.rs
@@ -20,6 +20,7 @@ use nexus_test_interface::NexusServer;
 use nexus_test_utils::http_testing::AuthnMode;
 use nexus_test_utils::http_testing::NexusRequest;
 use nexus_test_utils::http_testing::RequestBuilder;
+use nexus_test_utils::resource_helpers::assert_ip_pool_utilization;
 use nexus_test_utils::resource_helpers::create_default_ip_pool;
 use nexus_test_utils::resource_helpers::create_disk;
 use nexus_test_utils::resource_helpers::create_floating_ip;
@@ -3905,10 +3906,14 @@ async fn test_instance_ephemeral_ip_from_correct_pool(
     create_ip_pool(&client, "pool1", Some(range1)).await;
     link_ip_pool(&client, "pool1", &DEFAULT_SILO.id(), /*default*/ true).await;
 
+    assert_ip_pool_utilization(client, "pool1", 0, 5, 0, 0).await;
+
     // second pool is associated with the silo but not default
     create_ip_pool(&client, "pool2", Some(range2)).await;
     link_ip_pool(&client, "pool2", &DEFAULT_SILO.id(), /*default*/ false).await;
 
+    assert_ip_pool_utilization(client, "pool2", 0, 5, 0, 0).await;
+
     // Create an instance with pool name blank, expect IP from default pool
     create_instance_with_pool(client, "pool1-inst", None).await;
 
@@ -3917,6 +3922,10 @@ async fn test_instance_ephemeral_ip_from_correct_pool(
         ip.ip() >= range1.first_address() && ip.ip() <= range1.last_address(),
         "Expected ephemeral IP to come from pool1"
     );
+    // 1 ephemeral + 1 snat
+    assert_ip_pool_utilization(client, "pool1", 2, 5, 0, 0).await;
+    // pool2 unaffected
+    assert_ip_pool_utilization(client, "pool2", 0, 5, 0, 0).await;
 
     // Create an instance explicitly using the non-default "other-pool".
     create_instance_with_pool(client, "pool2-inst", Some("pool2")).await;
@@ -3926,6 +3935,13 @@ async fn test_instance_ephemeral_ip_from_correct_pool(
         "Expected ephemeral IP to come from pool2"
     );
 
+    // added 1 snat because snat comes from default pool.
+    // https://github.com/oxidecomputer/omicron/issues/5043 is about changing that
+    assert_ip_pool_utilization(client, "pool1", 3, 5, 0, 0).await;
+
+    // ephemeral IP comes from specified pool
+    assert_ip_pool_utilization(client, "pool2", 1, 5, 0, 0).await;
+
     // make pool2 default and create instance with default pool. check that it now it comes from pool2
     let _: views::IpPoolSiloLink = object_put(
         client,
@@ -3941,6 +3957,11 @@ async fn test_instance_ephemeral_ip_from_correct_pool(
         "Expected ephemeral IP to come from pool2"
     );
 
+    // pool1 unchanged
+    assert_ip_pool_utilization(client, "pool1", 3, 5, 0, 0).await;
+    // +1 snat (now that pool2 is default) and +1 ephemeral
+    assert_ip_pool_utilization(client, "pool2", 3, 5, 0, 0).await;
+
     // try to delete association with pool1, but it fails because there is an
     // instance with an IP from the pool in this silo
     let pool1_silo_url =
@@ -3956,8 +3977,14 @@ async fn test_instance_ephemeral_ip_from_correct_pool(
     // stop and delete instances with IPs from pool1. perhaps surprisingly, that
     // includes pool2-inst also because the SNAT IP comes from the default pool
     // even when different pool is specified for the ephemeral IP
-    stop_instance(&cptestctx, "pool1-inst").await;
-    stop_instance(&cptestctx, "pool2-inst").await;
+    stop_and_delete_instance(&cptestctx, "pool1-inst").await;
+    stop_and_delete_instance(&cptestctx, "pool2-inst").await;
+
+    // pool1 is down to 0 because it had 1 snat + 1 ephemeral from pool1-inst
+    // and 1 snat from pool2-inst
+    assert_ip_pool_utilization(client, "pool1", 0, 5, 0, 0).await;
+    // pool2 drops one because it had 1 ephemeral from pool2-inst
+    assert_ip_pool_utilization(client, "pool2", 2, 5, 0, 0).await;
 
     // now unlink works
     object_delete(client, &pool1_silo_url).await;
@@ -3992,7 +4019,7 @@ async fn test_instance_ephemeral_ip_from_correct_pool(
     assert_eq!(error.message, "not found: ip-pool with name \"pool1\"");
 }
 
-async fn stop_instance(
+async fn stop_and_delete_instance(
     cptestctx: &ControlPlaneTestContext,
     instance_name: &str,
 ) {
@@ -4146,9 +4173,7 @@ async fn test_instance_attach_several_external_ips(
     create_ip_pool(&client, "default", Some(default_pool_range)).await;
     link_ip_pool(&client, "default", &DEFAULT_SILO.id(), true).await;
 
-    // this doesn't work as a replacement for the above. figure out why and
-    // probably delete it
-    // create_default_ip_pool(&client).await;
+    assert_ip_pool_utilization(client, "default", 0, 10, 0, 0).await;
 
     // Create several floating IPs for the instance, totalling 8 IPs.
     let mut external_ip_create =
@@ -4177,6 +4202,9 @@ async fn test_instance_attach_several_external_ips(
     )
     .await;
 
+    // 1 ephemeral + 7 floating + 1 SNAT
+    assert_ip_pool_utilization(client, "default", 9, 10, 0, 0).await;
+
     // Verify that all external IPs are visible on the instance and have
     // been allocated in order.
     let external_ips =
@@ -4331,6 +4359,8 @@ async fn test_instance_create_in_silo(cptestctx: &ControlPlaneTestContext) {
     create_ip_pool(&client, "default", None).await;
     link_ip_pool(&client, "default", &silo.identity.id, true).await;
 
+    assert_ip_pool_utilization(client, "default", 0, 65536, 0, 0).await;
+
     // Create test projects
     NexusRequest::objects_post(
         client,
diff --git a/nexus/tests/integration_tests/ip_pools.rs b/nexus/tests/integration_tests/ip_pools.rs
index f1d8825d0e..c8390e8ce0 100644
--- a/nexus/tests/integration_tests/ip_pools.rs
+++ b/nexus/tests/integration_tests/ip_pools.rs
@@ -15,6 +15,7 @@ use nexus_db_queries::db::fixed_data::silo::INTERNAL_SILO_ID;
 use nexus_test_utils::http_testing::AuthnMode;
 use nexus_test_utils::http_testing::NexusRequest;
 use nexus_test_utils::http_testing::RequestBuilder;
+use nexus_test_utils::resource_helpers::assert_ip_pool_utilization;
 use nexus_test_utils::resource_helpers::create_instance;
 use nexus_test_utils::resource_helpers::create_ip_pool;
 use nexus_test_utils::resource_helpers::create_project;
@@ -757,6 +758,47 @@ async fn create_pool(client: &ClientTestContext, name: &str) -> IpPool {
         .unwrap()
 }
 
+// This is mostly about testing the total field with huge numbers.
+// testing allocated is done in a bunch of other places. look for
+// assert_ip_pool_utilization calls
+#[nexus_test]
+async fn test_ip_pool_utilization_total(cptestctx: &ControlPlaneTestContext) {
+    let client = &cptestctx.external_client;
+
+    create_pool(client, "p0").await;
+
+    assert_ip_pool_utilization(client, "p0", 0, 0, 0, 0).await;
+
+    let add_url = "/v1/system/ip-pools/p0/ranges/add";
+
+    // add just 5 addresses to get the party started
+    let range = IpRange::V4(
+        Ipv4Range::new(
+            std::net::Ipv4Addr::new(10, 0, 0, 1),
+            std::net::Ipv4Addr::new(10, 0, 0, 5),
+        )
+        .unwrap(),
+    );
+    object_create::<IpRange, IpPoolRange>(client, &add_url, &range).await;
+
+    assert_ip_pool_utilization(client, "p0", 0, 5, 0, 0).await;
+
+    // now let's add a gigantic range just for fun
+    let big_range = IpRange::V6(
+        Ipv6Range::new(
+            std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1),
+            std::net::Ipv6Addr::new(
+                0xfd00, 0, 0, 0, 0xffff, 0xfff, 0xffff, 0xffff,
+            ),
+        )
+        .unwrap(),
+    );
+    object_create::<IpRange, IpPoolRange>(client, &add_url, &big_range).await;
+
+    assert_ip_pool_utilization(client, "p0", 0, 5, 0, 18446480190918885375)
+        .await;
+}
+
 // Data for testing overlapping IP ranges
 struct TestRange {
     // A starting IP range that should be inserted correctly
diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt
index 3b954ec6ec..ba79d75f22 100644
--- a/nexus/tests/output/nexus_tags.txt
+++ b/nexus/tests/output/nexus_tags.txt
@@ -164,6 +164,7 @@ ip_pool_silo_list                        GET      /v1/system/ip-pools/{pool}/sil
 ip_pool_silo_unlink                      DELETE   /v1/system/ip-pools/{pool}/silos/{silo}
 ip_pool_silo_update                      PUT      /v1/system/ip-pools/{pool}/silos/{silo}
 ip_pool_update                           PUT      /v1/system/ip-pools/{pool}
+ip_pool_utilization_view                 GET      /v1/system/ip-pools/{pool}/utilization
 ip_pool_view                             GET      /v1/system/ip-pools/{pool}
 networking_address_lot_block_list        GET      /v1/system/networking/address-lot/{address_lot}/blocks
 networking_address_lot_create            POST     /v1/system/networking/address-lot
diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs
index a3e87b162e..fcea302f72 100644
--- a/nexus/types/src/external_api/views.rs
+++ b/nexus/types/src/external_api/views.rs
@@ -305,6 +305,82 @@ pub struct IpPool {
     pub identity: IdentityMetadata,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+pub struct Ipv4Utilization {
+    /// The number of IPv4 addresses allocated from this pool
+    pub allocated: u32,
+    /// The total number of IPv4 addresses in the pool, i.e., the sum of the
+    /// lengths of the IPv4 ranges. Unlike IPv6 capacity, can be a 32-bit
+    /// integer because there are only 2^32 IPv4 addresses.
+    pub capacity: u32,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+pub struct Ipv6Utilization {
+    /// The number of IPv6 addresses allocated from this pool. A 128-bit integer
+    /// string to match the capacity field.
+    #[serde(with = "U128String")]
+    pub allocated: u128,
+
+    /// The total number of IPv6 addresses in the pool, i.e., the sum of the
+    /// lengths of the IPv6 ranges. An IPv6 range can contain up to 2^128
+    /// addresses, so we represent this value in JSON as a numeric string with a
+    /// custom "uint128" format.
+    #[serde(with = "U128String")]
+    pub capacity: u128,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
+pub struct IpPoolUtilization {
+    /// Number of allocated and total available IPv4 addresses in pool
+    pub ipv4: Ipv4Utilization,
+    /// Number of allocated and total available IPv6 addresses in pool
+    pub ipv6: Ipv6Utilization,
+}
+
+// Custom struct for serializing/deserializing u128 as a string. The serde
+// docs will suggest using a module (or serialize_with and deserialize_with
+// functions), but as discussed in the comments on the UserData de/serializer,
+// schemars wants this to be a type, so it has to be a struct.
+struct U128String;
+impl U128String {
+    pub fn serialize<S>(value: &u128, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_str(&value.to_string())
+    }
+
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<u128, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        s.parse().map_err(serde::de::Error::custom)
+    }
+}
+
+impl JsonSchema for U128String {
+    fn schema_name() -> String {
+        "String".to_string()
+    }
+
+    fn json_schema(
+        _: &mut schemars::gen::SchemaGenerator,
+    ) -> schemars::schema::Schema {
+        schemars::schema::SchemaObject {
+            instance_type: Some(schemars::schema::InstanceType::String.into()),
+            format: Some("uint128".to_string()),
+            ..Default::default()
+        }
+        .into()
+    }
+
+    fn is_referenceable() -> bool {
+        false
+    }
+}
+
 /// An IP pool in the context of a silo
 #[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)]
 pub struct SiloIpPool {
diff --git a/openapi/nexus.json b/openapi/nexus.json
index cacd875acc..7516a896a7 100644
--- a/openapi/nexus.json
+++ b/openapi/nexus.json
@@ -5723,6 +5723,44 @@
         }
       }
     },
+    "/v1/system/ip-pools/{pool}/utilization": {
+      "get": {
+        "tags": [
+          "system/networking"
+        ],
+        "summary": "Fetch IP pool utilization",
+        "operationId": "ip_pool_utilization_view",
+        "parameters": [
+          {
+            "in": "path",
+            "name": "pool",
+            "description": "Name or ID of the IP pool",
+            "required": true,
+            "schema": {
+              "$ref": "#/components/schemas/NameOrId"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/IpPoolUtilization"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/v1/system/ip-pools-service": {
       "get": {
         "tags": [
@@ -13652,6 +13690,31 @@
           }
         }
       },
+      "IpPoolUtilization": {
+        "type": "object",
+        "properties": {
+          "ipv4": {
+            "description": "Number of allocated and total available IPv4 addresses in pool",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Ipv4Utilization"
+              }
+            ]
+          },
+          "ipv6": {
+            "description": "Number of allocated and total available IPv6 addresses in pool",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Ipv6Utilization"
+              }
+            ]
+          }
+        },
+        "required": [
+          "ipv4",
+          "ipv6"
+        ]
+      },
       "IpRange": {
         "oneOf": [
           {
@@ -13697,6 +13760,27 @@
           "last"
         ]
       },
+      "Ipv4Utilization": {
+        "type": "object",
+        "properties": {
+          "allocated": {
+            "description": "The number of IPv4 addresses allocated from this pool",
+            "type": "integer",
+            "format": "uint32",
+            "minimum": 0
+          },
+          "capacity": {
+            "description": "The total number of IPv4 addresses in the pool, i.e., the sum of the lengths of the IPv4 ranges. Unlike IPv6 capacity, can be a 32-bit integer because there are only 2^32 IPv4 addresses.",
+            "type": "integer",
+            "format": "uint32",
+            "minimum": 0
+          }
+        },
+        "required": [
+          "allocated",
+          "capacity"
+        ]
+      },
       "Ipv6Net": {
         "example": "fd12:3456::/64",
         "title": "An IPv6 subnet",
@@ -13722,6 +13806,25 @@
           "last"
         ]
       },
+      "Ipv6Utilization": {
+        "type": "object",
+        "properties": {
+          "allocated": {
+            "description": "The number of IPv6 addresses allocated from this pool. A 128-bit integer string to match the capacity field.",
+            "type": "string",
+            "format": "uint128"
+          },
+          "capacity": {
+            "description": "The total number of IPv6 addresses in the pool, i.e., the sum of the lengths of the IPv6 ranges. An IPv6 range can contain up to 2^128 addresses, so we represent this value in JSON as a numeric string with a custom \"uint128\" format.",
+            "type": "string",
+            "format": "uint128"
+          }
+        },
+        "required": [
+          "allocated",
+          "capacity"
+        ]
+      },
       "L4PortRange": {
         "example": "22",
         "title": "A range of IP ports",

From dd4090f038605fcc5fc69d9350d4ca0ef5f5dc34 Mon Sep 17 00:00:00 2001
From: bnaecker <ben@oxide.computer>
Date: Mon, 18 Mar 2024 15:18:51 -0700
Subject: [PATCH 127/157] Update Propolis (#5281)

- Update propolis and related dep
- Pass instance metadata for vCPU statistics tracking
- Closes #4851
---
 Cargo.lock                       | 16 ++++++++--------
 Cargo.toml                       | 12 ++++++------
 package-manifest.toml            | 12 ++++++------
 sled-agent/src/instance.rs       | 10 ++--------
 sled-agent/src/params.rs         |  6 ++++++
 sled-agent/src/sim/sled_agent.rs |  5 ++---
 6 files changed, 30 insertions(+), 31 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2299d1df62..1818c89bd7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -469,7 +469,7 @@ dependencies = [
 [[package]]
 name = "bhyve_api"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c#a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c"
+source = "git+https://github.com/oxidecomputer/propolis?rev=fdf0585c6a227a7cfbee4a61a36938c3d77e4712#fdf0585c6a227a7cfbee4a61a36938c3d77e4712"
 dependencies = [
  "bhyve_api_sys",
  "libc",
@@ -479,7 +479,7 @@ dependencies = [
 [[package]]
 name = "bhyve_api_sys"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c#a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c"
+source = "git+https://github.com/oxidecomputer/propolis?rev=fdf0585c6a227a7cfbee4a61a36938c3d77e4712#fdf0585c6a227a7cfbee4a61a36938c3d77e4712"
 dependencies = [
  "libc",
  "strum 0.26.1",
@@ -1335,7 +1335,7 @@ dependencies = [
 [[package]]
 name = "crucible-agent-client"
 version = "0.0.1"
-source = "git+https://github.com/oxidecomputer/crucible?rev=1c1574fb721f98f2df1b23e3fd27d83be018882e#1c1574fb721f98f2df1b23e3fd27d83be018882e"
+source = "git+https://github.com/oxidecomputer/crucible?rev=952c7d60d22be5198b892bec8d92f4291b9160c2#952c7d60d22be5198b892bec8d92f4291b9160c2"
 dependencies = [
  "anyhow",
  "chrono",
@@ -1351,7 +1351,7 @@ dependencies = [
 [[package]]
 name = "crucible-pantry-client"
 version = "0.0.1"
-source = "git+https://github.com/oxidecomputer/crucible?rev=1c1574fb721f98f2df1b23e3fd27d83be018882e#1c1574fb721f98f2df1b23e3fd27d83be018882e"
+source = "git+https://github.com/oxidecomputer/crucible?rev=952c7d60d22be5198b892bec8d92f4291b9160c2#952c7d60d22be5198b892bec8d92f4291b9160c2"
 dependencies = [
  "anyhow",
  "chrono",
@@ -1368,7 +1368,7 @@ dependencies = [
 [[package]]
 name = "crucible-smf"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/crucible?rev=1c1574fb721f98f2df1b23e3fd27d83be018882e#1c1574fb721f98f2df1b23e3fd27d83be018882e"
+source = "git+https://github.com/oxidecomputer/crucible?rev=952c7d60d22be5198b892bec8d92f4291b9160c2#952c7d60d22be5198b892bec8d92f4291b9160c2"
 dependencies = [
  "crucible-workspace-hack",
  "libc",
@@ -6796,7 +6796,7 @@ dependencies = [
 [[package]]
 name = "propolis-client"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c#a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c"
+source = "git+https://github.com/oxidecomputer/propolis?rev=fdf0585c6a227a7cfbee4a61a36938c3d77e4712#fdf0585c6a227a7cfbee4a61a36938c3d77e4712"
 dependencies = [
  "async-trait",
  "base64 0.21.7",
@@ -6817,7 +6817,7 @@ dependencies = [
 [[package]]
 name = "propolis-mock-server"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c#a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c"
+source = "git+https://github.com/oxidecomputer/propolis?rev=fdf0585c6a227a7cfbee4a61a36938c3d77e4712#fdf0585c6a227a7cfbee4a61a36938c3d77e4712"
 dependencies = [
  "anyhow",
  "atty",
@@ -6847,7 +6847,7 @@ dependencies = [
 [[package]]
 name = "propolis_types"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/propolis?rev=a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c#a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c"
+source = "git+https://github.com/oxidecomputer/propolis?rev=fdf0585c6a227a7cfbee4a61a36938c3d77e4712#fdf0585c6a227a7cfbee4a61a36938c3d77e4712"
 dependencies = [
  "schemars",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index c27a4bef05..4a039e8c43 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -194,9 +194,9 @@ cookie = "0.18"
 criterion = { version = "0.5.1", features = [ "async_tokio" ] }
 crossbeam = "0.8"
 crossterm = { version = "0.27.0", features = ["event-stream"] }
-crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "1c1574fb721f98f2df1b23e3fd27d83be018882e" }
-crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "1c1574fb721f98f2df1b23e3fd27d83be018882e" }
-crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "1c1574fb721f98f2df1b23e3fd27d83be018882e" }
+crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "952c7d60d22be5198b892bec8d92f4291b9160c2" }
+crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "952c7d60d22be5198b892bec8d92f4291b9160c2" }
+crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "952c7d60d22be5198b892bec8d92f4291b9160c2" }
 csv = "1.3.0"
 curve25519-dalek = "4"
 datatest-stable = "0.2.3"
@@ -333,9 +333,9 @@ prettyplease = { version = "0.2.16", features = ["verbatim"] }
 proc-macro2 = "1.0"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" }
 progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" }
-bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c" }
-propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c" }
-propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c" }
+bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "fdf0585c6a227a7cfbee4a61a36938c3d77e4712" }
+propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "fdf0585c6a227a7cfbee4a61a36938c3d77e4712" }
+propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "fdf0585c6a227a7cfbee4a61a36938c3d77e4712" }
 proptest = "1.4.0"
 quote = "1.0"
 rand = "0.8.5"
diff --git a/package-manifest.toml b/package-manifest.toml
index 0b54e35d4f..83c12b9174 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -492,10 +492,10 @@ only_for_targets.image = "standard"
 # 3. Use source.type = "manual" instead of "prebuilt"
 source.type = "prebuilt"
 source.repo = "crucible"
-source.commit = "1c1574fb721f98f2df1b23e3fd27d83be018882e"
+source.commit = "952c7d60d22be5198b892bec8d92f4291b9160c2"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image/<commit>/crucible.sha256.txt
-source.sha256 = "a14ce9d6b17c6c8427d526e044bb1196ee81316674936873ac9dc1d7ca51459d"
+source.sha256 = "af200d930e6d658bdd1688a032747da7c2bafa0e6ab1b53bda7b7f571d8a19c6"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -504,10 +504,10 @@ service_name = "crucible_pantry_prebuilt"
 only_for_targets.image = "standard"
 source.type = "prebuilt"
 source.repo = "crucible"
-source.commit = "1c1574fb721f98f2df1b23e3fd27d83be018882e"
+source.commit = "952c7d60d22be5198b892bec8d92f4291b9160c2"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image/<commit>/crucible-pantry.sha256.txt
-source.sha256 = "8839a3748cd2926f61244c99344d0802e59ea4673a5f31a2aaad5d759176aa80"
+source.sha256 = "1a40e24b176e8179cc7f74ef6922cd2ee51d5eca9bfa7057fc37994cf5c4678e"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -519,10 +519,10 @@ service_name = "propolis-server"
 only_for_targets.image = "standard"
 source.type = "prebuilt"
 source.repo = "propolis"
-source.commit = "a9fdbc464db4cbd4ac0f43ffa0d045b4b02e0c1c"
+source.commit = "fdf0585c6a227a7cfbee4a61a36938c3d77e4712"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image/<commit>/propolis-server.sha256.txt
-source.sha256 = "7998a71c4e905a4bf9ef2d2863b7695d0bb4d1b34322aa3ce20d173d4a7ee77a"
+source.sha256 = "f07720e9041907f9285432251c82c0c7502bf1be9dd4df1ba6abe8f7462c2e9e"
 output.type = "zone"
 
 [package.mg-ddm-gz]
diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs
index 5ddca90403..6b69de8bb5 100644
--- a/sled-agent/src/instance.rs
+++ b/sled-agent/src/instance.rs
@@ -311,11 +311,6 @@ struct InstanceRunner {
     // Properties visible to Propolis
     properties: propolis_client::types::InstanceProperties,
 
-    // This is currently unused, but will be sent to Propolis as part of the
-    // work tracked in https://github.com/oxidecomputer/omicron/issues/4851. It
-    // will be included in the InstanceProperties above, most likely.
-    _metadata: InstanceMetadata,
-
     // The ID of the Propolis server (and zone) running this instance
     propolis_id: Uuid,
 
@@ -926,6 +921,7 @@ impl Instance {
     ///   instance manager's tracking table.
     /// * `state`: The initial state of this instance.
     /// * `services`: A set of instance manager-provided services.
+    /// * `metadata`: Instance-related metadata used to track statistics.
     pub(crate) fn new(
         log: Logger,
         id: Uuid,
@@ -1010,10 +1006,8 @@ impl Instance {
                 // TODO: we should probably make propolis aligned with
                 // InstanceCpuCount here, to avoid any casting...
                 vcpus: hardware.properties.ncpus.0 as u8,
+                metadata: metadata.into(),
             },
-            // This will be used in a follow up, tracked under
-            // https://github.com/oxidecomputer/omicron/issues/4851.
-            _metadata: metadata,
             propolis_id,
             propolis_addr,
             vnic_allocator,
diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs
index f74438a678..71bd34c091 100644
--- a/sled-agent/src/params.rs
+++ b/sled-agent/src/params.rs
@@ -93,6 +93,12 @@ pub struct InstanceMetadata {
     pub project_id: Uuid,
 }
 
+impl From<InstanceMetadata> for propolis_client::types::InstanceMetadata {
+    fn from(md: InstanceMetadata) -> Self {
+        Self { silo_id: md.silo_id, project_id: md.project_id }
+    }
+}
+
 /// The body of a request to ensure that a instance and VMM are known to a sled
 /// agent.
 #[derive(Serialize, Deserialize, JsonSchema)]
diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs
index b772a60347..45547cea64 100644
--- a/sled-agent/src/sim/sled_agent.rs
+++ b/sled-agent/src/sim/sled_agent.rs
@@ -238,9 +238,7 @@ impl SledAgent {
         hardware: InstanceHardware,
         instance_runtime: InstanceRuntimeState,
         vmm_runtime: VmmRuntimeState,
-        // This is currently unused, but will be included as part of work
-        // tracked in https://github.com/oxidecomputer/omicron/issues/4851.
-        _metadata: InstanceMetadata,
+        metadata: InstanceMetadata,
     ) -> Result<SledInstanceState, Error> {
         // respond with a fake 500 level failure if asked to ensure an instance
         // with more than 16 CPUs.
@@ -294,6 +292,7 @@ impl SledAgent {
                     bootrom_id: Uuid::default(),
                     memory: hardware.properties.memory.to_whole_mebibytes(),
                     vcpus: hardware.properties.ncpus.0 as u8,
+                    metadata: metadata.into(),
                 };
                 let body = propolis_client::types::InstanceEnsureRequest {
                     properties,

From 89e7b42a36422b55d64a42c3cc504638af9af339 Mon Sep 17 00:00:00 2001
From: Ryan Goodfellow <ryan.goodfellow@oxide.computer>
Date: Mon, 18 Mar 2024 20:30:38 -0700
Subject: [PATCH 128/157] add bgp message tracking, use real maghemite client
 (#5271)

---
 .github/buildomat/jobs/a4x2-deploy.sh         |  17 +-
 .github/buildomat/jobs/a4x2-prepare.sh        |   2 +-
 Cargo.lock                                    | 456 ++++++++++--------
 Cargo.toml                                    |   8 +-
 clients/ddm-admin-client/Cargo.toml           |   4 +-
 clients/ddm-admin-client/build.rs             | 107 ----
 clients/ddm-admin-client/src/lib.rs           |  15 +-
 clients/mg-admin-client/Cargo.toml            |  25 -
 clients/mg-admin-client/build.rs              | 103 ----
 clients/mg-admin-client/src/lib.rs            | 100 ----
 common/Cargo.toml                             |   1 +
 common/src/api/external/mod.rs                |  71 ++-
 end-to-end-tests/src/helpers/icmp.rs          |   4 +-
 installinator/Cargo.toml                      |   2 +-
 installinator/src/bootstrap.rs                |   2 +-
 installinator/src/peers.rs                    |   2 +-
 nexus/src/app/background/bfd.rs               |   7 +-
 nexus/src/app/background/networking.rs        |  18 +-
 .../background/sync_switch_configuration.rs   |  25 +-
 nexus/src/app/bfd.rs                          |   1 -
 nexus/src/app/bgp.rs                          |  48 +-
 nexus/src/app/mod.rs                          |  39 +-
 nexus/src/external_api/http_entrypoints.rs    |  26 +-
 nexus/tests/integration_tests/endpoints.rs    |  11 +
 nexus/tests/output/nexus_tags.txt             |   1 +
 openapi/nexus.json                            |  82 ++++
 package-manifest.toml                         |  12 +-
 sled-agent/Cargo.toml                         |   2 +-
 sled-agent/src/bootstrap/bootstore_setup.rs   |   2 +-
 sled-agent/src/bootstrap/early_networking.rs  |  56 +--
 sled-agent/src/bootstrap/pre_server.rs        |   2 +-
 sled-agent/src/bootstrap/server.rs            |   4 +-
 sled-agent/src/rack_setup/service.rs          |   2 +-
 sled-agent/src/services.rs                    |   2 +-
 sled-agent/src/sled_agent.rs                  |   4 +-
 tools/maghemite_ddm_openapi_version           |   2 +-
 tools/maghemite_mg_openapi_version            |   4 +-
 tools/maghemite_mgd_checksums                 |   4 +-
 tools/update_maghemite.sh                     |  45 --
 wicketd/Cargo.toml                            |   2 +-
 wicketd/src/bootstrap_addrs.rs                |   2 +-
 workspace-hack/Cargo.toml                     |  28 +-
 42 files changed, 622 insertions(+), 728 deletions(-)
 delete mode 100644 clients/ddm-admin-client/build.rs
 delete mode 100644 clients/mg-admin-client/Cargo.toml
 delete mode 100644 clients/mg-admin-client/build.rs
 delete mode 100644 clients/mg-admin-client/src/lib.rs

diff --git a/.github/buildomat/jobs/a4x2-deploy.sh b/.github/buildomat/jobs/a4x2-deploy.sh
index a07f953281..0e37d4ae25 100755
--- a/.github/buildomat/jobs/a4x2-deploy.sh
+++ b/.github/buildomat/jobs/a4x2-deploy.sh
@@ -13,7 +13,7 @@
 #:  "%/out/dhcp-server.log",
 #: ]
 #: skip_clone = true
-#: enable = false
+#: enable = true
 #:
 #: [dependencies.a4x2]
 #: job = "a4x2-prepare"
@@ -87,7 +87,7 @@ pfexec mv propolis-server /usr/bin/
 #
 export DISK=${DISK:-c1t1d0}
 pfexec diskinfo
-pfexec zpool create -f cpool $DISK
+pfexec zpool create -o ashift=12 -f cpool $DISK
 pfexec zfs create -o mountpoint=/ci cpool/ci
 
 if [[ $(curl -s http://catacomb.eng.oxide.computer:12346/trim-me) =~ "true" ]]; then
@@ -115,7 +115,7 @@ ls -R
 #
 buildomat_url=https://buildomat.eng.oxide.computer
 testbed_artifact_path=public/file/oxidecomputer/testbed/topo/
-testbed_rev=677559e30b4dfc65c374b24336ac23d40102de81
+testbed_rev=67454d38958bcf51830850aec36600df84b7d8a0
 curl -fOL $buildomat_url/$testbed_artifact_path/$testbed_rev/a4x2
 chmod +x a4x2
 
@@ -193,13 +193,16 @@ pfexec route add 198.51.100.0/24 $customer_edge_addr
 #
 # Run the communications test program
 #
+# TODO tighten up packet loss tolerance. For now it's more or less ok for it to
+# just run with _some_ comms. The program will fail if there are no comms to a
+# given sled.
 cp /input/a4x2/out/commtest .
 chmod +x commtest
-pfexec ./commtest http://198.51.100.23 run \
+NO_COLOR=1 pfexec ./commtest http://198.51.100.23 run \
     --ip-pool-begin 198.51.100.40 \
     --ip-pool-end 198.51.100.70 \
-    --icmp-loss-tolerance 10 \
-    --test-duration 300s \
-    --packet-rate 30
+    --icmp-loss-tolerance 500 \
+    --test-duration 200s \
+    --packet-rate 10
 
 cp connectivity-report.json /out/
diff --git a/.github/buildomat/jobs/a4x2-prepare.sh b/.github/buildomat/jobs/a4x2-prepare.sh
index bc88ddd4c0..79fa037139 100755
--- a/.github/buildomat/jobs/a4x2-prepare.sh
+++ b/.github/buildomat/jobs/a4x2-prepare.sh
@@ -20,7 +20,7 @@
 #: access_repos = [
 #:	"oxidecomputer/testbed",
 #: ]
-#: enable = false
+#: enable = true
 
 source ./env.sh
 
diff --git a/Cargo.lock b/Cargo.lock
index 1818c89bd7..0bb79eece8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -174,7 +174,7 @@ dependencies = [
  "omicron-workspace-hack",
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -275,7 +275,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -297,7 +297,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -308,7 +308,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -361,7 +361,7 @@ dependencies = [
  "quote",
  "serde",
  "serde_tokenstream 0.2.0",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -466,6 +466,26 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "bgp"
+version = "0.1.0"
+source = "git+https://github.com/oxidecomputer/maghemite?rev=1d8b818b0fd000bc6d2739575c8868b0781dfe41#1d8b818b0fd000bc6d2739575c8868b0781dfe41"
+dependencies = [
+ "anyhow",
+ "chrono",
+ "mg-common",
+ "nom",
+ "num_enum 0.7.2",
+ "rdb",
+ "schemars",
+ "serde",
+ "sled",
+ "slog",
+ "slog-async",
+ "slog-bunyan",
+ "thiserror",
+]
+
 [[package]]
 name = "bhyve_api"
 version = "0.0.0"
@@ -504,7 +524,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.51",
+ "syn 2.0.52",
  "which",
 ]
 
@@ -649,7 +669,7 @@ dependencies = [
  "slog-term",
  "thiserror",
  "tokio",
- "uuid",
+ "uuid 1.7.0",
  "vsss-rs",
  "zeroize",
 ]
@@ -662,13 +682,13 @@ dependencies = [
  "omicron-common",
  "omicron-workspace-hack",
  "progenitor",
- "regress 0.9.0",
+ "regress",
  "reqwest",
  "schemars",
  "serde",
  "sled-hardware-types",
  "slog",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -1024,7 +1044,7 @@ dependencies = [
  "heck 0.4.1",
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1362,7 +1382,7 @@ dependencies = [
  "schemars",
  "serde",
  "serde_json",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -1478,7 +1498,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1526,7 +1546,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim 0.10.0",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1548,7 +1568,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
 dependencies = [
  "darling_core 0.20.3",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1582,30 +1602,21 @@ dependencies = [
  "quote",
  "serde",
  "serde_tokenstream 0.2.0",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
 name = "ddm-admin-client"
 version = "0.1.0"
+source = "git+https://github.com/oxidecomputer/maghemite?rev=1d8b818b0fd000bc6d2739575c8868b0781dfe41#1d8b818b0fd000bc6d2739575c8868b0781dfe41"
 dependencies = [
- "anyhow",
- "either",
- "omicron-common",
- "omicron-workspace-hack",
- "omicron-zone-package",
+ "mg-common",
+ "percent-encoding",
  "progenitor",
- "progenitor-client",
- "quote",
  "reqwest",
- "rustfmt-wrapper",
  "serde",
  "serde_json",
- "sled-hardware-types",
  "slog",
- "thiserror",
- "tokio",
- "toml 0.8.10",
 ]
 
 [[package]]
@@ -1634,7 +1645,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1667,7 +1678,7 @@ checksum = "5fe87ce4529967e0ba1dcf8450bab64d97dfd5010a6256187ffe2e43e6f0e049"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1688,7 +1699,7 @@ checksum = "62d671cc41a825ebabc75757b62d3d168c577f9149b2d49ece1dad1f72119d25"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1743,7 +1754,7 @@ dependencies = [
  "darling 0.20.3",
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1783,7 +1794,7 @@ dependencies = [
  "pq-sys",
  "r2d2",
  "serde_json",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -1794,7 +1805,7 @@ dependencies = [
  "diesel",
  "serde",
  "usdt 0.5.0",
- "uuid",
+ "uuid 1.7.0",
  "version_check",
 ]
 
@@ -1807,7 +1818,7 @@ dependencies = [
  "diesel_table_macro_syntax",
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1816,7 +1827,7 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5"
 dependencies = [
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1898,7 +1909,7 @@ source = "git+https://github.com/oxidecomputer/dlpi-sys#1d587ea98cf2d36f1b1624b0
 dependencies = [
  "libc",
  "libdlpi-sys",
- "num_enum",
+ "num_enum 0.5.11",
  "pretty-hex 0.2.1",
  "thiserror",
  "tokio",
@@ -1939,7 +1950,7 @@ dependencies = [
  "trust-dns-proto",
  "trust-dns-resolver",
  "trust-dns-server",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -2008,7 +2019,7 @@ dependencies = [
  "progenitor-client",
  "quote",
  "rand 0.8.5",
- "regress 0.9.0",
+ "regress",
  "reqwest",
  "rustfmt-wrapper",
  "schemars",
@@ -2016,7 +2027,7 @@ dependencies = [
  "serde_json",
  "slog",
  "toml 0.8.10",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -2060,7 +2071,7 @@ dependencies = [
  "tokio-rustls 0.25.0",
  "toml 0.8.10",
  "usdt 0.3.5",
- "uuid",
+ "uuid 1.7.0",
  "version_check",
  "waitgroup",
 ]
@@ -2074,7 +2085,7 @@ dependencies = [
  "quote",
  "serde",
  "serde_tokenstream 0.2.0",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -2241,7 +2252,7 @@ dependencies = [
  "tokio",
  "toml 0.8.10",
  "trust-dns-resolver",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -2456,7 +2467,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -2573,7 +2584,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -2641,7 +2652,7 @@ dependencies = [
  "termios",
  "tokio",
  "tokio-tungstenite",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -2659,7 +2670,7 @@ dependencies = [
  "serde",
  "serde_json",
  "slog",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -2674,7 +2685,7 @@ dependencies = [
  "smoltcp 0.9.1",
  "static_assertions",
  "strum_macros 0.25.2",
- "uuid",
+ "uuid 1.7.0",
  "zerocopy 0.6.4",
 ]
 
@@ -2704,7 +2715,7 @@ dependencies = [
  "tlvc 0.3.1 (git+https://github.com/oxidecomputer/tlvc.git?branch=main)",
  "tokio",
  "usdt 0.3.5",
- "uuid",
+ "uuid 1.7.0",
  "version_check",
  "zip",
 ]
@@ -2722,7 +2733,7 @@ dependencies = [
  "slog",
  "sp-sim",
  "tokio",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -3419,7 +3430,7 @@ source = "git+https://github.com/oxidecomputer/illumos-devinfo?branch=main#4323b
 dependencies = [
  "anyhow",
  "libc",
- "num_enum",
+ "num_enum 0.5.11",
 ]
 
 [[package]]
@@ -3449,7 +3460,7 @@ dependencies = [
  "opte-ioctl",
  "oxide-vpc",
  "oxlog",
- "regress 0.9.0",
+ "regress",
  "schemars",
  "serde",
  "serde_json",
@@ -3458,7 +3469,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "toml 0.8.10",
- "uuid",
+ "uuid 1.7.0",
  "zone",
 ]
 
@@ -3542,7 +3553,6 @@ dependencies = [
  "camino",
  "cancel-safe-futures",
  "clap 4.5.1",
- "ddm-admin-client",
  "display-error-chain",
  "futures",
  "hex",
@@ -3555,6 +3565,7 @@ dependencies = [
  "itertools 0.12.1",
  "libc",
  "omicron-common",
+ "omicron-ddm-admin-client",
  "omicron-test-utils",
  "omicron-workspace-hack",
  "partial-io",
@@ -3576,7 +3587,7 @@ dependencies = [
  "tokio-stream",
  "tufaceous-lib",
  "update-engine",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -3586,14 +3597,14 @@ dependencies = [
  "installinator-common",
  "omicron-workspace-hack",
  "progenitor",
- "regress 0.9.0",
+ "regress",
  "reqwest",
  "schemars",
  "serde",
  "serde_json",
  "slog",
  "update-engine",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -3617,7 +3628,7 @@ dependencies = [
  "serde_json",
  "slog",
  "subprocess",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -3675,7 +3686,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "trust-dns-resolver",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -3712,7 +3723,7 @@ dependencies = [
  "serde",
  "test-strategy",
  "thiserror",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -3832,7 +3843,7 @@ version = "0.1.0"
 source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
 dependencies = [
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -3907,7 +3918,7 @@ dependencies = [
  "libc",
  "libefi-sys",
  "thiserror",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -3942,7 +3953,7 @@ dependencies = [
  "colored",
  "dlpi",
  "libc",
- "num_enum",
+ "num_enum 0.5.11",
  "nvpair",
  "nvpair-sys",
  "rusty-doors",
@@ -4026,9 +4037,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.20"
+version = "0.4.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
+checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
 
 [[package]]
 name = "lpc55_areas"
@@ -4175,23 +4186,31 @@ dependencies = [
 [[package]]
 name = "mg-admin-client"
 version = "0.1.0"
+source = "git+https://github.com/oxidecomputer/maghemite?rev=1d8b818b0fd000bc6d2739575c8868b0781dfe41#1d8b818b0fd000bc6d2739575c8868b0781dfe41"
 dependencies = [
  "anyhow",
- "either",
- "omicron-common",
- "omicron-workspace-hack",
- "omicron-zone-package",
+ "bgp",
+ "chrono",
+ "percent-encoding",
  "progenitor",
- "progenitor-client",
- "quote",
+ "rdb",
  "reqwest",
- "rustfmt-wrapper",
+ "schemars",
  "serde",
  "serde_json",
  "slog",
+]
+
+[[package]]
+name = "mg-common"
+version = "0.1.0"
+source = "git+https://github.com/oxidecomputer/maghemite?rev=1d8b818b0fd000bc6d2739575c8868b0781dfe41#1d8b818b0fd000bc6d2739575c8868b0781dfe41"
+dependencies = [
+ "anstyle",
+ "clap 4.5.1",
+ "schemars",
+ "serde",
  "thiserror",
- "tokio",
- "toml 0.8.10",
 ]
 
 [[package]]
@@ -4261,7 +4280,7 @@ dependencies = [
  "cfg-if",
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -4347,7 +4366,7 @@ checksum = "6a5ff2b31594942586c1520da8f1e5c705729ec67b3c2ad0fe459f0b576e4d9a"
 dependencies = [
  "schemars",
  "serde",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -4372,13 +4391,13 @@ dependencies = [
  "omicron-uuid-kinds",
  "omicron-workspace-hack",
  "progenitor",
- "regress 0.9.0",
+ "regress",
  "reqwest",
  "schemars",
  "serde",
  "serde_json",
  "slog",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -4398,7 +4417,7 @@ dependencies = [
  "serde_with",
  "tokio-postgres",
  "toml 0.8.10",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -4436,7 +4455,7 @@ dependencies = [
  "steno",
  "strum 0.26.1",
  "thiserror",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -4518,7 +4537,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "usdt 0.5.0",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -4557,7 +4576,7 @@ dependencies = [
  "strum 0.26.1",
  "thiserror",
  "tokio",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -4567,7 +4586,7 @@ dependencies = [
  "omicron-workspace-hack",
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -4582,7 +4601,7 @@ dependencies = [
  "reqwest",
  "sled-agent-client",
  "slog",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -4618,7 +4637,7 @@ dependencies = [
  "slog",
  "slog-error-chain",
  "tokio",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -4644,7 +4663,7 @@ dependencies = [
  "sled-agent-client",
  "slog",
  "thiserror",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -4668,7 +4687,7 @@ dependencies = [
  "omicron-common",
  "omicron-workspace-hack",
  "slog",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -4712,7 +4731,7 @@ dependencies = [
  "tokio",
  "tokio-util",
  "trust-dns-resolver",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -4721,7 +4740,7 @@ version = "0.1.0"
 dependencies = [
  "omicron-workspace-hack",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -4750,7 +4769,7 @@ dependencies = [
  "steno",
  "strum 0.26.1",
  "thiserror",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -4883,7 +4902,7 @@ checksum = "9e6a0fd4f737c707bd9086cc16c925f294943eb62eb71499e9fd4cf71f8b9f4e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -4943,7 +4962,16 @@ version = "0.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9"
 dependencies = [
- "num_enum_derive",
+ "num_enum_derive 0.5.11",
+]
+
+[[package]]
+name = "num_enum"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845"
+dependencies = [
+ "num_enum_derive 0.7.2",
 ]
 
 [[package]]
@@ -4958,6 +4986,18 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "num_enum_derive"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.52",
+]
+
 [[package]]
 name = "num_threads"
 version = "0.1.6"
@@ -5054,6 +5094,7 @@ dependencies = [
  "ipnetwork",
  "libc",
  "macaddr",
+ "mg-admin-client",
  "omicron-uuid-kinds",
  "omicron-workspace-hack",
  "once_cell",
@@ -5062,7 +5103,7 @@ dependencies = [
  "progenitor-client",
  "proptest",
  "rand 0.8.5",
- "regress 0.9.0",
+ "regress",
  "reqwest",
  "schemars",
  "semver 1.0.22",
@@ -5077,7 +5118,31 @@ dependencies = [
  "test-strategy",
  "thiserror",
  "tokio",
- "uuid",
+ "uuid 1.7.0",
+]
+
+[[package]]
+name = "omicron-ddm-admin-client"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "ddm-admin-client",
+ "either",
+ "omicron-common",
+ "omicron-workspace-hack",
+ "omicron-zone-package",
+ "progenitor",
+ "progenitor-client",
+ "quote",
+ "reqwest",
+ "rustfmt-wrapper",
+ "serde",
+ "serde_json",
+ "sled-hardware-types",
+ "slog",
+ "thiserror",
+ "tokio",
+ "toml 0.8.10",
 ]
 
 [[package]]
@@ -5154,7 +5219,7 @@ dependencies = [
  "tokio-stream",
  "tokio-tungstenite",
  "toml 0.8.10",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -5276,7 +5341,7 @@ dependencies = [
  "tufaceous",
  "tufaceous-lib",
  "update-common",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -5332,7 +5397,7 @@ dependencies = [
  "textwrap 0.16.1",
  "tokio",
  "unicode-width",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -5410,7 +5475,6 @@ dependencies = [
  "chrono",
  "clap 4.5.1",
  "crucible-agent-client",
- "ddm-admin-client",
  "derive_more",
  "display-error-chain",
  "dns-server",
@@ -5441,6 +5505,7 @@ dependencies = [
  "nexus-config",
  "nexus-types",
  "omicron-common",
+ "omicron-ddm-admin-client",
  "omicron-test-utils",
  "omicron-workspace-hack",
  "once_cell",
@@ -5483,7 +5548,7 @@ dependencies = [
  "tokio-util",
  "toml 0.8.10",
  "usdt 0.5.0",
- "uuid",
+ "uuid 1.7.0",
  "zeroize",
  "zone",
 ]
@@ -5623,7 +5688,7 @@ dependencies = [
  "string_cache",
  "subtle",
  "syn 1.0.109",
- "syn 2.0.51",
+ "syn 2.0.52",
  "time",
  "time-macros",
  "tokio",
@@ -5640,7 +5705,7 @@ dependencies = [
  "unicode-normalization",
  "usdt 0.3.5",
  "usdt-impl 0.5.0",
- "uuid",
+ "uuid 1.7.0",
  "yasna",
  "zerocopy 0.7.32",
  "zeroize",
@@ -5743,7 +5808,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -5858,14 +5923,14 @@ dependencies = [
  "omicron-workspace-hack",
  "progenitor",
  "rand 0.8.5",
- "regress 0.9.0",
+ "regress",
  "reqwest",
  "serde",
  "serde_json",
  "thiserror",
  "tokio",
  "trust-dns-resolver",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -5902,7 +5967,7 @@ dependencies = [
  "strum 0.26.1",
  "thiserror",
  "trybuild",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -5917,7 +5982,7 @@ dependencies = [
  "reqwest",
  "serde",
  "slog",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -5957,7 +6022,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "toml 0.8.10",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -6000,7 +6065,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "usdt 0.5.0",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -6021,7 +6086,7 @@ dependencies = [
  "slog-term",
  "thiserror",
  "tokio",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -6031,7 +6096,7 @@ dependencies = [
  "omicron-workspace-hack",
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -6052,7 +6117,7 @@ dependencies = [
  "slog-dtrace",
  "thiserror",
  "tokio",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -6064,7 +6129,7 @@ dependencies = [
  "chrono",
  "clap 4.5.1",
  "omicron-workspace-hack",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -6196,7 +6261,7 @@ dependencies = [
  "regex",
  "regex-syntax 0.8.2",
  "structmeta 0.3.0",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -6343,7 +6408,7 @@ dependencies = [
  "pest_meta",
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -6413,7 +6478,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -6586,7 +6651,7 @@ dependencies = [
  "postgres-protocol",
  "serde",
  "serde_json",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -6674,7 +6739,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5"
 dependencies = [
  "proc-macro2",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -6731,8 +6796,8 @@ dependencies = [
 
 [[package]]
 name = "progenitor"
-version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor?branch=main#08bbafc251d3d6828ca5921d1658c3d12d64fe7c"
+version = "0.6.0"
+source = "git+https://github.com/oxidecomputer/progenitor?branch=main#90d3282f488a17f9c85e25c26845fef2d92af435"
 dependencies = [
  "progenitor-client",
  "progenitor-impl",
@@ -6742,8 +6807,8 @@ dependencies = [
 
 [[package]]
 name = "progenitor-client"
-version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor?branch=main#08bbafc251d3d6828ca5921d1658c3d12d64fe7c"
+version = "0.6.0"
+source = "git+https://github.com/oxidecomputer/progenitor?branch=main#90d3282f488a17f9c85e25c26845fef2d92af435"
 dependencies = [
  "bytes",
  "futures-core",
@@ -6756,8 +6821,8 @@ dependencies = [
 
 [[package]]
 name = "progenitor-impl"
-version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor?branch=main#08bbafc251d3d6828ca5921d1658c3d12d64fe7c"
+version = "0.6.0"
+source = "git+https://github.com/oxidecomputer/progenitor?branch=main#90d3282f488a17f9c85e25c26845fef2d92af435"
 dependencies = [
  "getopts",
  "heck 0.4.1",
@@ -6770,7 +6835,7 @@ dependencies = [
  "schemars",
  "serde",
  "serde_json",
- "syn 2.0.51",
+ "syn 2.0.52",
  "thiserror",
  "typify",
  "unicode-ident",
@@ -6778,8 +6843,8 @@ dependencies = [
 
 [[package]]
 name = "progenitor-macro"
-version = "0.5.0"
-source = "git+https://github.com/oxidecomputer/progenitor?branch=main#08bbafc251d3d6828ca5921d1658c3d12d64fe7c"
+version = "0.6.0"
+source = "git+https://github.com/oxidecomputer/progenitor?branch=main#90d3282f488a17f9c85e25c26845fef2d92af435"
 dependencies = [
  "openapiv3",
  "proc-macro2",
@@ -6790,7 +6855,7 @@ dependencies = [
  "serde_json",
  "serde_tokenstream 0.2.0",
  "serde_yaml",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -6811,7 +6876,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "tokio-tungstenite",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -6841,7 +6906,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "tokio-tungstenite",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -7110,6 +7175,22 @@ dependencies = [
  "yasna",
 ]
 
+[[package]]
+name = "rdb"
+version = "0.1.0"
+source = "git+https://github.com/oxidecomputer/maghemite?rev=1d8b818b0fd000bc6d2739575c8868b0781dfe41#1d8b818b0fd000bc6d2739575c8868b0781dfe41"
+dependencies = [
+ "anyhow",
+ "ciborium",
+ "mg-common",
+ "schemars",
+ "serde",
+ "serde_json",
+ "sled",
+ "slog",
+ "thiserror",
+]
+
 [[package]]
 name = "rdrand"
 version = "0.4.0"
@@ -7145,7 +7226,7 @@ dependencies = [
  "subprocess",
  "swrite",
  "tabled",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -7223,7 +7304,7 @@ checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -7267,16 +7348,6 @@ version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
 
-[[package]]
-name = "regress"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f5f39ba4513916c1b2657b72af6ec671f091cd637992f58d0ede5cae4e5dea0"
-dependencies = [
- "hashbrown 0.14.3",
- "memchr",
-]
-
 [[package]]
 name = "regress"
 version = "0.9.0"
@@ -7472,7 +7543,7 @@ dependencies = [
  "regex",
  "relative-path",
  "rustc_version 0.4.0",
- "syn 2.0.51",
+ "syn 2.0.52",
  "unicode-ident",
 ]
 
@@ -7821,7 +7892,7 @@ dependencies = [
  "serde",
  "thiserror",
  "url",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -7863,7 +7934,8 @@ dependencies = [
  "schemars_derive",
  "serde",
  "serde_json",
- "uuid",
+ "uuid 0.8.2",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -7901,7 +7973,7 @@ checksum = "7f81c2fde025af7e69b1d1420531c8a8811ca898919db177141a85313b1cb932"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -8039,7 +8111,7 @@ checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -8100,7 +8172,7 @@ checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -8132,7 +8204,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -8174,7 +8246,7 @@ dependencies = [
  "darling 0.20.3",
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -8354,12 +8426,12 @@ dependencies = [
  "omicron-common",
  "omicron-workspace-hack",
  "progenitor",
- "regress 0.9.0",
+ "regress",
  "reqwest",
  "schemars",
  "serde",
  "slog",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -8389,7 +8461,7 @@ dependencies = [
  "thiserror",
  "tofino",
  "tokio",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -8427,7 +8499,7 @@ dependencies = [
  "slog",
  "thiserror",
  "tokio",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -8505,7 +8577,7 @@ source = "git+https://github.com/oxidecomputer/slog-error-chain?branch=main#15f6
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -8760,7 +8832,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -8804,7 +8876,7 @@ dependencies = [
  "slog",
  "thiserror",
  "tokio",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -8867,7 +8939,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "structmeta-derive 0.2.0",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -8879,7 +8951,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "structmeta-derive 0.3.0",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -8890,7 +8962,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -8901,7 +8973,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -8966,7 +9038,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -8979,7 +9051,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -9026,9 +9098,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.51"
+version = "2.0.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ab617d94515e94ae53b8406c628598680aa0c9587474ecbe58188f7b345d66c"
+checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -9224,7 +9296,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "structmeta 0.2.0",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -9264,7 +9336,7 @@ checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -9454,7 +9526,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -9720,7 +9792,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -9946,8 +10018,8 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
 
 [[package]]
 name = "typify"
-version = "0.0.15"
-source = "git+https://github.com/oxidecomputer/typify#131fe0ef3722d3034a61a8ab2994c29f9c978903"
+version = "0.0.16"
+source = "git+https://github.com/oxidecomputer/typify#c5ebe0a2bf08ad8a743be5b593b1a8526a3fff4a"
 dependencies = [
  "typify-impl",
  "typify-macro",
@@ -9955,25 +10027,25 @@ dependencies = [
 
 [[package]]
 name = "typify-impl"
-version = "0.0.15"
-source = "git+https://github.com/oxidecomputer/typify#131fe0ef3722d3034a61a8ab2994c29f9c978903"
+version = "0.0.16"
+source = "git+https://github.com/oxidecomputer/typify#c5ebe0a2bf08ad8a743be5b593b1a8526a3fff4a"
 dependencies = [
  "heck 0.4.1",
  "log",
  "proc-macro2",
  "quote",
- "regress 0.8.0",
+ "regress",
  "schemars",
  "serde_json",
- "syn 2.0.51",
+ "syn 2.0.52",
  "thiserror",
  "unicode-ident",
 ]
 
 [[package]]
 name = "typify-macro"
-version = "0.0.15"
-source = "git+https://github.com/oxidecomputer/typify#131fe0ef3722d3034a61a8ab2994c29f9c978903"
+version = "0.0.16"
+source = "git+https://github.com/oxidecomputer/typify#c5ebe0a2bf08ad8a743be5b593b1a8526a3fff4a"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -9981,7 +10053,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_tokenstream 0.2.0",
- "syn 2.0.51",
+ "syn 2.0.52",
  "typify-impl",
 ]
 
@@ -10148,7 +10220,7 @@ dependencies = [
  "tokio",
  "tokio-stream",
  "unicode-width",
- "uuid",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -10215,7 +10287,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_tokenstream 0.2.0",
- "syn 2.0.51",
+ "syn 2.0.52",
  "usdt-impl 0.5.0",
 ]
 
@@ -10253,7 +10325,7 @@ dependencies = [
  "quote",
  "serde",
  "serde_json",
- "syn 2.0.51",
+ "syn 2.0.52",
  "thiserror",
  "thread-id",
  "version_check",
@@ -10283,7 +10355,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_tokenstream 0.2.0",
- "syn 2.0.51",
+ "syn 2.0.52",
  "usdt-impl 0.5.0",
 ]
 
@@ -10299,6 +10371,12 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
 
+[[package]]
+name = "uuid"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
+
 [[package]]
 name = "uuid"
 version = "1.7.0"
@@ -10441,7 +10519,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
  "wasm-bindgen-shared",
 ]
 
@@ -10475,7 +10553,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -10633,7 +10711,6 @@ dependencies = [
  "camino",
  "camino-tempfile",
  "clap 4.5.1",
- "ddm-admin-client",
  "debug-ignore",
  "display-error-chain",
  "dpd-client",
@@ -10662,6 +10739,7 @@ dependencies = [
  "maplit",
  "omicron-certificates",
  "omicron-common",
+ "omicron-ddm-admin-client",
  "omicron-passwords",
  "omicron-test-utils",
  "omicron-workspace-hack",
@@ -10689,7 +10767,7 @@ dependencies = [
  "tufaceous-lib",
  "update-common",
  "update-engine",
- "uuid",
+ "uuid 1.7.0",
  "wicket",
  "wicket-common",
  "wicketd-client",
@@ -10704,14 +10782,14 @@ dependencies = [
  "ipnetwork",
  "omicron-workspace-hack",
  "progenitor",
- "regress 0.9.0",
+ "regress",
  "reqwest",
  "schemars",
  "serde",
  "serde_json",
  "slog",
  "update-engine",
- "uuid",
+ "uuid 1.7.0",
  "wicket-common",
 ]
 
@@ -11027,7 +11105,7 @@ checksum = "56097d5b91d711293a42be9289403896b68654625021732067eac7a4ca388a1f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -11038,7 +11116,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -11058,7 +11136,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.51",
+ "syn 2.0.52",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 4a039e8c43..802f965965 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,7 +10,6 @@ members = [
     "clients/dpd-client",
     "clients/gateway-client",
     "clients/installinator-artifact-client",
-    "clients/mg-admin-client",
     "clients/nexus-client",
     "clients/oxide-client",
     "clients/oximeter-client",
@@ -96,7 +95,6 @@ default-members = [
     "clients/oximeter-client",
     "clients/sled-agent-client",
     "clients/wicketd-client",
-    "clients/mg-admin-client",
     "common",
     "dev-tools/crdb-seed",
     "dev-tools/omdb",
@@ -201,7 +199,7 @@ csv = "1.3.0"
 curve25519-dalek = "4"
 datatest-stable = "0.2.3"
 display-error-chain = "0.2.0"
-ddm-admin-client = { path = "clients/ddm-admin-client" }
+omicron-ddm-admin-client = { path = "clients/ddm-admin-client" }
 db-macros = { path = "nexus/db-macros" }
 debug-ignore = "1.0.5"
 derive_more = "0.99.17"
@@ -267,7 +265,8 @@ maplit = "1.0.2"
 mime_guess = "2.0.4"
 mockall = "0.12"
 newtype_derive = "0.1.6"
-mg-admin-client = { path = "clients/mg-admin-client" }
+mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "1d8b818b0fd000bc6d2739575c8868b0781dfe41" }
+ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "1d8b818b0fd000bc6d2739575c8868b0781dfe41" }
 multimap = "0.10.0"
 nexus-client = { path = "clients/nexus-client" }
 nexus-config = { path = "nexus-config" }
@@ -669,6 +668,7 @@ branch = "oxide/omicron"
 # See also: uuid-kinds/README.adoc.
 [patch."https://github.com/oxidecomputer/omicron"]
 omicron-uuid-kinds = { path = "uuid-kinds" }
+omicron-common = { path = "common" }
 
 [patch.crates-io.zone]
 git = 'https://github.com/oxidecomputer/zone'
diff --git a/clients/ddm-admin-client/Cargo.toml b/clients/ddm-admin-client/Cargo.toml
index f4aee0a961..8254007c7c 100644
--- a/clients/ddm-admin-client/Cargo.toml
+++ b/clients/ddm-admin-client/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "ddm-admin-client"
+name = "omicron-ddm-admin-client"
 version = "0.1.0"
 edition = "2021"
 license = "MPL-2.0"
@@ -12,10 +12,10 @@ serde.workspace = true
 slog.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
-
 omicron-common.workspace = true
 sled-hardware-types.workspace = true
 omicron-workspace-hack.workspace = true
+ddm-admin-client.workspace = true
 
 [build-dependencies]
 anyhow.workspace = true
diff --git a/clients/ddm-admin-client/build.rs b/clients/ddm-admin-client/build.rs
deleted file mode 100644
index c51ec05faa..0000000000
--- a/clients/ddm-admin-client/build.rs
+++ /dev/null
@@ -1,107 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// Copyright 2022 Oxide Computer Company
-
-use anyhow::bail;
-use anyhow::Context;
-use anyhow::Result;
-use omicron_zone_package::config::Config;
-use omicron_zone_package::package::PackageSource;
-use quote::quote;
-use std::env;
-use std::fs;
-use std::path::Path;
-
-fn main() -> Result<()> {
-    // Find the current maghemite repo commit from our package manifest.
-    let manifest = fs::read_to_string("../../package-manifest.toml")
-        .context("failed to read ../../package-manifest.toml")?;
-    println!("cargo:rerun-if-changed=../../package-manifest.toml");
-
-    let config: Config = toml::from_str(&manifest)
-        .context("failed to parse ../package-manifest.toml")?;
-
-    let ddm = config
-        .packages
-        .get("mg-ddm-gz")
-        .context("missing mg-ddm-gz package in ../package-manifest.toml")?;
-
-    let local_path = match &ddm.source {
-        PackageSource::Prebuilt { commit, .. } => {
-            // Report a relatively verbose error if we haven't downloaded the requisite
-            // openapi spec.
-            let local_path =
-                env::var("DDM_OPENAPI_PATH").unwrap_or_else(|_| {
-                    format!("../../out/downloads/ddm-admin-{commit}.json")
-                });
-            if !Path::new(&local_path).exists() {
-                bail!("{local_path} doesn't exist; rerun `tools/ci_download_maghemite_openapi` (after updating `tools/maghemite_ddm_openapi_version` if the maghemite commit in package-manifest.toml has changed)");
-            }
-            println!("cargo:rerun-if-changed={local_path}");
-            local_path
-        }
-
-        PackageSource::Manual => {
-            let local_path =
-                "../../out/downloads/ddm-admin-manual.json".to_string();
-            if !Path::new(&local_path).exists() {
-                bail!("{local_path} doesn't exist, please copy manually built ddm-admin.json there!");
-            }
-            println!("cargo:rerun-if-changed={local_path}");
-            local_path
-        }
-
-        _ => {
-            bail!(
-                "mg-ddm external package must have type `prebuilt` or `manual`"
-            )
-        }
-    };
-
-    let spec = {
-        let bytes = fs::read(&local_path)
-            .with_context(|| format!("failed to read {local_path}"))?;
-        serde_json::from_slice(&bytes).with_context(|| {
-            format!("failed to parse {local_path} as openapi spec")
-        })?
-    };
-
-    let code = progenitor::Generator::new(
-        progenitor::GenerationSettings::new()
-            .with_inner_type(quote!(slog::Logger))
-            .with_pre_hook(quote! {
-                |log: &slog::Logger, request: &reqwest::Request| {
-                    slog::debug!(log, "client request";
-                        "method" => %request.method(),
-                        "uri" => %request.url(),
-                        "body" => ?&request.body(),
-                    );
-                }
-            })
-            .with_post_hook(quote! {
-                |log: &slog::Logger, result: &Result<_, _>| {
-                    slog::debug!(log, "client response"; "result" => ?result);
-                }
-            }),
-    )
-    .generate_tokens(&spec)
-    .with_context(|| {
-        format!("failed to generate progenitor client from {local_path}")
-    })?;
-
-    let content = rustfmt_wrapper::rustfmt(code).with_context(|| {
-        format!("rustfmt failed on progenitor code from {local_path}")
-    })?;
-
-    let out_file =
-        Path::new(&env::var("OUT_DIR").expect("OUT_DIR env var not set"))
-            .join("ddm-admin-client.rs");
-
-    fs::write(&out_file, content).with_context(|| {
-        format!("failed to write client to {}", out_file.display())
-    })?;
-
-    Ok(())
-}
diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs
index b88ea51e00..c27532d7d7 100644
--- a/clients/ddm-admin-client/src/lib.rs
+++ b/clients/ddm-admin-client/src/lib.rs
@@ -9,19 +9,12 @@
 #![allow(clippy::match_single_binding)]
 #![allow(clippy::clone_on_copy)]
 
-#[allow(dead_code)]
-mod inner {
-    include!(concat!(env!("OUT_DIR"), "/ddm-admin-client.rs"));
-
-    impl Copy for types::Ipv6Prefix {}
-}
-
-pub use inner::types;
-pub use inner::Error;
+pub use ddm_admin_client::types;
+pub use ddm_admin_client::Error;
 
+use ddm_admin_client::Client as InnerClient;
+use ddm_admin_client::{Ipv6Prefix, TunnelOrigin};
 use either::Either;
-use inner::types::{Ipv6Prefix, TunnelOrigin};
-use inner::Client as InnerClient;
 use omicron_common::address::Ipv6Subnet;
 use omicron_common::address::SLED_PREFIX;
 use omicron_common::backoff::retry_notify;
diff --git a/clients/mg-admin-client/Cargo.toml b/clients/mg-admin-client/Cargo.toml
deleted file mode 100644
index e426fa2371..0000000000
--- a/clients/mg-admin-client/Cargo.toml
+++ /dev/null
@@ -1,25 +0,0 @@
-[package]
-name = "mg-admin-client"
-version = "0.1.0"
-edition = "2021"
-license = "MPL-2.0"
-
-[dependencies]
-either.workspace = true
-progenitor-client.workspace = true
-reqwest = { workspace = true, features = ["json", "stream", "rustls-tls"] }
-serde.workspace = true
-slog.workspace = true
-thiserror.workspace = true
-tokio.workspace = true
-omicron-common.workspace = true
-omicron-workspace-hack.workspace = true
-
-[build-dependencies]
-anyhow.workspace = true
-omicron-zone-package.workspace = true
-progenitor.workspace = true
-quote.workspace = true
-rustfmt-wrapper.workspace = true
-serde_json.workspace = true
-toml.workspace = true
diff --git a/clients/mg-admin-client/build.rs b/clients/mg-admin-client/build.rs
deleted file mode 100644
index d9886d0ece..0000000000
--- a/clients/mg-admin-client/build.rs
+++ /dev/null
@@ -1,103 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// Copyright 2022 Oxide Computer Company
-
-use anyhow::bail;
-use anyhow::Context;
-use anyhow::Result;
-use omicron_zone_package::config::Config;
-use omicron_zone_package::package::PackageSource;
-use quote::quote;
-use std::env;
-use std::fs;
-use std::path::Path;
-
-fn main() -> Result<()> {
-    // Find the current maghemite repo commit from our package manifest.
-    let manifest = fs::read_to_string("../../package-manifest.toml")
-        .context("failed to read ../../package-manifest.toml")?;
-    println!("cargo:rerun-if-changed=../../package-manifest.toml");
-
-    let config: Config = toml::from_str(&manifest)
-        .context("failed to parse ../../package-manifest.toml")?;
-    let mg = config
-        .packages
-        .get("mgd")
-        .context("missing mgd package in ../../package-manifest.toml")?;
-
-    let local_path = match &mg.source {
-        PackageSource::Prebuilt { commit, .. } => {
-            // Report a relatively verbose error if we haven't downloaded the requisite
-            // openapi spec.
-            let local_path = env::var("MG_OPENAPI_PATH").unwrap_or_else(|_| {
-                format!("../../out/downloads/mg-admin-{commit}.json")
-            });
-            if !Path::new(&local_path).exists() {
-                bail!("{local_path} doesn't exist; rerun `tools/ci_download_maghemite_openapi` (after updating `tools/maghemite_mg_openapi_version` if the maghemite commit in package-manifest.toml has changed)");
-            }
-            println!("cargo:rerun-if-changed={local_path}");
-            local_path
-        }
-
-        PackageSource::Manual => {
-            let local_path =
-                "../../out/downloads/mg-admin-manual.json".to_string();
-            if !Path::new(&local_path).exists() {
-                bail!("{local_path} doesn't exist, please copy manually built mg-admin.json there!");
-            }
-            println!("cargo:rerun-if-changed={local_path}");
-            local_path
-        }
-
-        _ => {
-            bail!("mgd external package must have type `prebuilt` or `manual`")
-        }
-    };
-
-    let spec = {
-        let bytes = fs::read(&local_path)
-            .with_context(|| format!("failed to read {local_path}"))?;
-        serde_json::from_slice(&bytes).with_context(|| {
-            format!("failed to parse {local_path} as openapi spec")
-        })?
-    };
-
-    let code = progenitor::Generator::new(
-        progenitor::GenerationSettings::new()
-            .with_inner_type(quote!(slog::Logger))
-            .with_pre_hook(quote! {
-                |log: &slog::Logger, request: &reqwest::Request| {
-                    slog::debug!(log, "client request";
-                        "method" => %request.method(),
-                        "uri" => %request.url(),
-                        "body" => ?&request.body(),
-                    );
-                }
-            })
-            .with_post_hook(quote! {
-                |log: &slog::Logger, result: &Result<_, _>| {
-                    slog::debug!(log, "client response"; "result" => ?result);
-                }
-            }),
-    )
-    .generate_tokens(&spec)
-    .with_context(|| {
-        format!("failed to generate progenitor client from {local_path}")
-    })?;
-
-    let content = rustfmt_wrapper::rustfmt(code).with_context(|| {
-        format!("rustfmt failed on progenitor code from {local_path}")
-    })?;
-
-    let out_file =
-        Path::new(&env::var("OUT_DIR").expect("OUT_DIR env var not set"))
-            .join("mg-admin-client.rs");
-
-    fs::write(&out_file, content).with_context(|| {
-        format!("failed to write client to {}", out_file.display())
-    })?;
-
-    Ok(())
-}
diff --git a/clients/mg-admin-client/src/lib.rs b/clients/mg-admin-client/src/lib.rs
deleted file mode 100644
index b246cddc26..0000000000
--- a/clients/mg-admin-client/src/lib.rs
+++ /dev/null
@@ -1,100 +0,0 @@
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// Copyright 2023 Oxide Computer Company
-
-#![allow(clippy::redundant_closure_call)]
-#![allow(clippy::needless_lifetimes)]
-#![allow(clippy::match_single_binding)]
-#![allow(clippy::clone_on_copy)]
-#![allow(rustdoc::broken_intra_doc_links)]
-#![allow(rustdoc::invalid_html_tags)]
-
-#[allow(dead_code)]
-mod inner {
-    include!(concat!(env!("OUT_DIR"), "/mg-admin-client.rs"));
-}
-
-pub use inner::types;
-use inner::types::Prefix4;
-pub use inner::Error;
-
-use inner::Client as InnerClient;
-use omicron_common::api::external::BgpPeerState;
-use slog::Logger;
-use std::hash::Hash;
-use std::net::Ipv6Addr;
-use std::net::SocketAddr;
-use thiserror::Error;
-
-// TODO-cleanup Is it okay to hardcode this port number here?
-const MGD_PORT: u16 = 4676;
-
-#[derive(Debug, Error)]
-pub enum MgError {
-    #[error("Failed to construct an HTTP client: {0}")]
-    HttpClient(#[from] reqwest::Error),
-
-    #[error("Failed making HTTP request to mgd: {0}")]
-    MgApi(#[from] Error<types::Error>),
-}
-
-impl From<inner::types::FsmStateKind> for BgpPeerState {
-    fn from(s: inner::types::FsmStateKind) -> BgpPeerState {
-        use inner::types::FsmStateKind;
-        match s {
-            FsmStateKind::Idle => BgpPeerState::Idle,
-            FsmStateKind::Connect => BgpPeerState::Connect,
-            FsmStateKind::Active => BgpPeerState::Active,
-            FsmStateKind::OpenSent => BgpPeerState::OpenSent,
-            FsmStateKind::OpenConfirm => BgpPeerState::OpenConfirm,
-            FsmStateKind::SessionSetup => BgpPeerState::SessionSetup,
-            FsmStateKind::Established => BgpPeerState::Established,
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct Client {
-    pub inner: InnerClient,
-    pub log: Logger,
-}
-
-impl Client {
-    /// Creates a new [`Client`] that points to localhost
-    pub fn localhost(log: &Logger) -> Result<Self, MgError> {
-        Self::new(log, SocketAddr::new(Ipv6Addr::LOCALHOST.into(), MGD_PORT))
-    }
-
-    pub fn new(log: &Logger, mgd_addr: SocketAddr) -> Result<Self, MgError> {
-        let dur = std::time::Duration::from_secs(60);
-        let log = log.new(slog::o!("MgAdminClient" => mgd_addr));
-
-        let inner = reqwest::ClientBuilder::new()
-            .connect_timeout(dur)
-            .timeout(dur)
-            .build()?;
-        let inner = InnerClient::new_with_client(
-            &format!("http://{mgd_addr}"),
-            inner,
-            log.clone(),
-        );
-        Ok(Self { inner, log })
-    }
-}
-
-impl Eq for Prefix4 {}
-
-impl PartialEq for Prefix4 {
-    fn eq(&self, other: &Self) -> bool {
-        self.value == other.value && self.length == other.length
-    }
-}
-
-impl Hash for Prefix4 {
-    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
-        self.value.hash(state);
-        self.length.hash(state);
-    }
-}
diff --git a/common/Cargo.toml b/common/Cargo.toml
index 3acd849b96..0485d3973b 100644
--- a/common/Cargo.toml
+++ b/common/Cargo.toml
@@ -23,6 +23,7 @@ hex.workspace = true
 http.workspace = true
 ipnetwork.workspace = true
 macaddr.workspace = true
+mg-admin-client.workspace = true
 omicron-uuid-kinds.workspace = true
 proptest = { workspace = true, optional = true }
 rand.workspace = true
diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs
index 1a355f4929..002f92ecc9 100644
--- a/common/src/api/external/mod.rs
+++ b/common/src/api/external/mod.rs
@@ -9,9 +9,6 @@
 
 mod error;
 pub mod http_pagination;
-use dropshot::HttpError;
-pub use error::*;
-
 pub use crate::api::internal::shared::SwitchLocation;
 use crate::update::ArtifactHash;
 use crate::update::ArtifactId;
@@ -20,7 +17,9 @@ use anyhow::Context;
 use api_identity::ObjectIdentity;
 use chrono::DateTime;
 use chrono::Utc;
+use dropshot::HttpError;
 pub use dropshot::PaginationOrder;
+pub use error::*;
 use futures::stream::BoxStream;
 use parse_display::Display;
 use parse_display::FromStr;
@@ -31,6 +30,7 @@ use semver;
 use serde::Deserialize;
 use serde::Serialize;
 use serde_with::{DeserializeFromStr, SerializeDisplay};
+use std::collections::HashMap;
 use std::convert::TryFrom;
 use std::fmt::Debug;
 use std::fmt::Display;
@@ -2718,6 +2718,21 @@ pub enum BgpPeerState {
     Established,
 }
 
+impl From<mg_admin_client::types::FsmStateKind> for BgpPeerState {
+    fn from(s: mg_admin_client::types::FsmStateKind) -> BgpPeerState {
+        use mg_admin_client::types::FsmStateKind;
+        match s {
+            FsmStateKind::Idle => BgpPeerState::Idle,
+            FsmStateKind::Connect => BgpPeerState::Connect,
+            FsmStateKind::Active => BgpPeerState::Active,
+            FsmStateKind::OpenSent => BgpPeerState::OpenSent,
+            FsmStateKind::OpenConfirm => BgpPeerState::OpenConfirm,
+            FsmStateKind::SessionSetup => BgpPeerState::SessionSetup,
+            FsmStateKind::Established => BgpPeerState::Established,
+        }
+    }
+}
+
 /// The current status of a BGP peer.
 #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)]
 pub struct BgpPeerStatus {
@@ -2740,6 +2755,56 @@ pub struct BgpPeerStatus {
     pub switch: SwitchLocation,
 }
 
+/// Opaque object representing BGP message history for a given BGP peer. The
+/// contents of this object are not yet stable.
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct BgpMessageHistory(mg_admin_client::MessageHistory);
+
+impl BgpMessageHistory {
+    pub fn new(arg: mg_admin_client::MessageHistory) -> Self {
+        Self(arg)
+    }
+}
+
+impl JsonSchema for BgpMessageHistory {
+    fn json_schema(
+        gen: &mut schemars::gen::SchemaGenerator,
+    ) -> schemars::schema::Schema {
+        let obj = schemars::schema::Schema::Object(
+            schemars::schema::SchemaObject::default(),
+        );
+        gen.definitions_mut().insert(Self::schema_name(), obj.clone());
+        obj
+    }
+
+    fn schema_name() -> String {
+        "BgpMessageHistory".to_owned()
+    }
+}
+
+/// BGP message history for a particular switch.
+#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]
+pub struct SwitchBgpHistory {
+    /// Switch this message history is associated with.
+    pub switch: SwitchLocation,
+
+    /// Message history indexed by peer address.
+    pub history: HashMap<String, BgpMessageHistory>,
+}
+
+/// BGP message history for rack switches.
+#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]
+pub struct AggregateBgpMessageHistory {
+    /// BGP history organized by switch.
+    switch_histories: Vec<SwitchBgpHistory>,
+}
+
+impl AggregateBgpMessageHistory {
+    pub fn new(switch_histories: Vec<SwitchBgpHistory>) -> Self {
+        Self { switch_histories }
+    }
+}
+
 /// A route imported from a BGP peer.
 #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)]
 pub struct BgpImportedRouteIpv4 {
diff --git a/end-to-end-tests/src/helpers/icmp.rs b/end-to-end-tests/src/helpers/icmp.rs
index 42617801d8..36b9bc5675 100644
--- a/end-to-end-tests/src/helpers/icmp.rs
+++ b/end-to-end-tests/src/helpers/icmp.rs
@@ -125,7 +125,7 @@ impl Pinger4 {
 
     fn show(self: Arc<Self>) {
         println!(
-            "{:15} {:7} {:7} {:7} {:7} {:7} {:7} {}",
+            "{:15} {:7} {:7} {:7} {:7} {:7} {:9} {}",
             "addr".dimmed(),
             "low".dimmed(),
             "avg".dimmed(),
@@ -140,7 +140,7 @@ impl Pinger4 {
             // print a status line for each target
             for (_id, t) in self.targets.lock().unwrap().iter() {
                 println!(
-                    "{:15} {:7} {:7} {:7} {:7} {:7} {:7} {:<7}",
+                    "{:15} {:7} {:7} {:7} {:7} {:7} {:9} {:<7}",
                     t.dest.to_string().cyan(),
                     format!("{:.3}", (t.low.as_micros() as f32 / 1000.0)),
                     if t.rx_count == 0 {
diff --git a/installinator/Cargo.toml b/installinator/Cargo.toml
index 1f92913aa1..abfcc5e892 100644
--- a/installinator/Cargo.toml
+++ b/installinator/Cargo.toml
@@ -12,7 +12,6 @@ bytes.workspace = true
 camino.workspace = true
 cancel-safe-futures.workspace = true
 clap.workspace = true
-ddm-admin-client.workspace = true
 display-error-chain.workspace = true
 futures.workspace = true
 hex.workspace = true
@@ -24,6 +23,7 @@ ipcc.workspace = true
 itertools.workspace = true
 libc.workspace = true
 omicron-common.workspace = true
+omicron-ddm-admin-client.workspace = true
 reqwest.workspace = true
 sha2.workspace = true
 sled-hardware.workspace = true
diff --git a/installinator/src/bootstrap.rs b/installinator/src/bootstrap.rs
index 59541df28e..9be1c12b86 100644
--- a/installinator/src/bootstrap.rs
+++ b/installinator/src/bootstrap.rs
@@ -9,12 +9,12 @@
 use anyhow::ensure;
 use anyhow::Context;
 use anyhow::Result;
-use ddm_admin_client::Client as DdmAdminClient;
 use illumos_utils::addrobj::AddrObject;
 use illumos_utils::dladm;
 use illumos_utils::dladm::Dladm;
 use illumos_utils::zone::Zones;
 use omicron_common::address::Ipv6Subnet;
+use omicron_ddm_admin_client::Client as DdmAdminClient;
 use sled_hardware::underlay;
 use sled_hardware_types::underlay::BootstrapInterface;
 use slog::info;
diff --git a/installinator/src/peers.rs b/installinator/src/peers.rs
index 33ba0de0e5..644507da4b 100644
--- a/installinator/src/peers.rs
+++ b/installinator/src/peers.rs
@@ -14,7 +14,6 @@ use anyhow::{bail, Result};
 use async_trait::async_trait;
 use buf_list::BufList;
 use bytes::Bytes;
-use ddm_admin_client::Client as DdmAdminClient;
 use display_error_chain::DisplayErrorChain;
 use futures::{Stream, StreamExt};
 use installinator_artifact_client::ClientError;
@@ -24,6 +23,7 @@ use installinator_common::{
 use itertools::Itertools;
 use omicron_common::address::BOOTSTRAP_ARTIFACT_PORT;
 use omicron_common::update::ArtifactHashId;
+use omicron_ddm_admin_client::Client as DdmAdminClient;
 use reqwest::StatusCode;
 use sled_hardware_types::underlay::BootstrapInterface;
 use tokio::{sync::mpsc, time::Instant};
diff --git a/nexus/src/app/background/bfd.rs b/nexus/src/app/background/bfd.rs
index ca23fb7f65..39b3c8f661 100644
--- a/nexus/src/app/background/bfd.rs
+++ b/nexus/src/app/background/bfd.rs
@@ -140,11 +140,11 @@ impl BackgroundTask for BfdManager {
             let mgd_clients = build_mgd_clients(mappings, log);
 
             for (location, c) in &mgd_clients {
-                let client_current = match c.inner.get_bfd_peers().await {
+                let client_current = match c.get_bfd_peers().await {
                     Ok(x) => x.into_inner(),
                     Err(e) => {
                         error!(&log, "failed to get bfd sessions from mgd: {}",
-                            c.inner.baseurl();
+                            c.baseurl();
                             "error" => e.to_string()
                         );
                         continue;
@@ -194,7 +194,6 @@ impl BackgroundTask for BfdManager {
                     }
                 };
                 if let Err(e) = mg
-                    .inner
                     .add_bfd_peer(&BfdPeerConfig {
                         peer: x.remote,
                         detection_threshold: x.detection_threshold,
@@ -224,7 +223,7 @@ impl BackgroundTask for BfdManager {
                         continue;
                     }
                 };
-                if let Err(e) = mg.inner.remove_bfd_peer(&x.remote).await {
+                if let Err(e) = mg.remove_bfd_peer(&x.remote).await {
                     error!(&log, "failed to remove bfd peer from switch daemon";
                         "error" => e.to_string(),
                         "switch" => x.switch.to_string(),
diff --git a/nexus/src/app/background/networking.rs b/nexus/src/app/background/networking.rs
index 95e0cd32b3..95005d07cd 100644
--- a/nexus/src/app/background/networking.rs
+++ b/nexus/src/app/background/networking.rs
@@ -21,20 +21,10 @@ pub(crate) fn build_mgd_clients(
         let port = MGD_PORT;
         let socketaddr =
             std::net::SocketAddr::V6(SocketAddrV6::new(*addr, port, 0, 0));
-        let client =
-            match mg_admin_client::Client::new(&log.clone(), socketaddr) {
-                Ok(client) => client,
-                Err(e) => {
-                    error!(
-                        log,
-                        "error building mgd client";
-                        "location" => %location,
-                        "addr" => %addr,
-                        "error" => %e,
-                    );
-                    continue;
-                }
-            };
+        let client = mg_admin_client::Client::new(
+            format!("http://{}", socketaddr).as_str(),
+            log.clone(),
+        );
         clients.push((*location, client));
     }
     clients.into_iter().collect::<HashMap<_, _>>()
diff --git a/nexus/src/app/background/sync_switch_configuration.rs b/nexus/src/app/background/sync_switch_configuration.rs
index 4bc75bad10..e21f82b007 100644
--- a/nexus/src/app/background/sync_switch_configuration.rs
+++ b/nexus/src/app/background/sync_switch_configuration.rs
@@ -28,8 +28,9 @@ use futures::future::BoxFuture;
 use futures::FutureExt;
 use mg_admin_client::types::{
     AddStaticRoute4Request, ApplyRequest, BgpPeerConfig,
-    DeleteStaticRoute4Request, Prefix4, StaticRoute4, StaticRoute4List,
+    DeleteStaticRoute4Request, StaticRoute4, StaticRoute4List,
 };
+use mg_admin_client::Prefix4;
 use nexus_db_queries::{
     context::OpContext,
     db::{datastore::SwitchPortSettingsCombinedResult, DataStore},
@@ -605,7 +606,7 @@ impl BackgroundTask for SwitchPortSettingsManager {
                             "switch_location" => ?location,
                             "config" => ?config,
                         );
-                        if let Err(e) = client.inner.bgp_apply(config).await {
+                        if let Err(e) = client.bgp_apply(config).await {
                             error!(log, "error while applying bgp configuration"; "error" => ?e);
                         }
                     }
@@ -1083,7 +1084,7 @@ fn static_routes_to_del(
                 .difference(routes_wanted)
                 .map(|(nexthop, prefix)| StaticRoute4 {
                     nexthop: *nexthop,
-                    prefix: prefix.clone(),
+                    prefix: *prefix,
                 })
                 .collect::<Vec<StaticRoute4>>();
 
@@ -1099,7 +1100,7 @@ fn static_routes_to_del(
                 .iter()
                 .map(|(nexthop, prefix)| StaticRoute4 {
                     nexthop: *nexthop,
-                    prefix: prefix.clone(),
+                    prefix: *prefix,
                 })
                 .collect::<Vec<StaticRoute4>>();
 
@@ -1146,7 +1147,7 @@ fn static_routes_to_add(
             .difference(routes_on_switch)
             .map(|(nexthop, prefix)| StaticRoute4 {
                 nexthop: *nexthop,
-                prefix: prefix.clone(),
+                prefix: *prefix,
             })
             .collect::<Vec<StaticRoute4>>();
 
@@ -1325,12 +1326,10 @@ async fn static_routes_on_switch<'a>(
 
     for (location, client) in mgd_clients {
         let static_routes: HashSet<(Ipv4Addr, Prefix4)> =
-            match client.inner.static_list_v4_routes().await {
-                Ok(routes) => routes
-                    .list
-                    .iter()
-                    .map(|r| (r.nexthop, r.prefix.clone()))
-                    .collect(),
+            match client.static_list_v4_routes().await {
+                Ok(routes) => {
+                    routes.list.iter().map(|r| (r.nexthop, r.prefix)).collect()
+                }
                 Err(_) => {
                     error!(
                         &log,
@@ -1369,7 +1368,7 @@ async fn delete_static_routes(
             "switch_location" => ?switch_location,
             "request" => ?request,
         );
-        if let Err(e) = client.inner.static_remove_v4_route(&request).await {
+        if let Err(e) = client.static_remove_v4_route(&request).await {
             error!(
                 &log,
                 "failed to delete routes from mgd";
@@ -1405,7 +1404,7 @@ async fn add_static_routes<'a>(
             "switch_location" => ?switch_location,
             "request" => ?request,
         );
-        if let Err(e) = client.inner.static_add_v4_route(&request).await {
+        if let Err(e) = client.static_add_v4_route(&request).await {
             error!(
                 &log,
                 "failed to add routes to mgd";
diff --git a/nexus/src/app/bfd.rs b/nexus/src/app/bfd.rs
index 22b9fc82ef..d3d0a731d0 100644
--- a/nexus/src/app/bfd.rs
+++ b/nexus/src/app/bfd.rs
@@ -69,7 +69,6 @@ impl super::Nexus {
         for s in &[SwitchLocation::Switch0, SwitchLocation::Switch1] {
             let mg_client = self.mg_client_for_switch_location(*s).await?;
             let status = mg_client
-                .inner
                 .get_bfd_peers()
                 .await
                 .map_err(|e| {
diff --git a/nexus/src/app/bgp.rs b/nexus/src/app/bgp.rs
index 9e609712e2..4a85306d56 100644
--- a/nexus/src/app/bgp.rs
+++ b/nexus/src/app/bgp.rs
@@ -4,12 +4,14 @@
 
 use crate::app::authz;
 use crate::external_api::params;
+use mg_admin_client::types::MessageHistoryRequest;
 use nexus_db_model::{BgpAnnounceSet, BgpAnnouncement, BgpConfig};
 use nexus_db_queries::context::OpContext;
 use omicron_common::api::external::http_pagination::PaginatedBy;
 use omicron_common::api::external::{
-    self, BgpImportedRouteIpv4, BgpPeerStatus, CreateResult, DeleteResult,
-    Ipv4Net, ListResultVec, LookupResult, NameOrId,
+    self, BgpImportedRouteIpv4, BgpMessageHistory, BgpPeerStatus, CreateResult,
+    DeleteResult, Ipv4Net, ListResultVec, LookupResult, NameOrId,
+    SwitchBgpHistory,
 };
 
 impl super::Nexus {
@@ -93,7 +95,7 @@ impl super::Nexus {
                 "failed to get mg clients: {e}"
             ))
         })? {
-            let router_info = match client.inner.get_routers().await {
+            let router_info = match client.get_routers().await {
                 Ok(result) => result.into_inner(),
                 Err(e) => {
                     error!(
@@ -123,6 +125,45 @@ impl super::Nexus {
         Ok(result)
     }
 
+    pub async fn bgp_message_history(
+        &self,
+        opctx: &OpContext,
+        sel: &params::BgpRouteSelector,
+    ) -> ListResultVec<SwitchBgpHistory> {
+        opctx.authorize(authz::Action::Read, &authz::FLEET).await?;
+
+        let mut result = Vec::new();
+        for (switch, client) in &self.mg_clients().await.map_err(|e| {
+            external::Error::internal_error(&format!(
+                "failed to get mg clients: {e}"
+            ))
+        })? {
+            let history = match client
+                .message_history(&MessageHistoryRequest { asn: sel.asn })
+                .await
+            {
+                Ok(result) => result.into_inner().by_peer.clone(),
+                Err(e) => {
+                    error!(
+                        self.log,
+                        "failed to get bgp history from {switch}: {e}"
+                    );
+                    continue;
+                }
+            };
+
+            result.push(SwitchBgpHistory {
+                switch: *switch,
+                history: history
+                    .into_iter()
+                    .map(|(k, v)| (k, BgpMessageHistory::new(v)))
+                    .collect(),
+            });
+        }
+
+        Ok(result)
+    }
+
     pub async fn bgp_imported_routes_ipv4(
         &self,
         opctx: &OpContext,
@@ -136,7 +177,6 @@ impl super::Nexus {
             ))
         })? {
             let imported: Vec<BgpImportedRouteIpv4> = match client
-                .inner
                 .get_imported4(&mg_admin_client::types::GetImported4Request {
                     asn: sel.asn,
                 })
diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs
index 9d94e08a5d..fdae303489 100644
--- a/nexus/src/app/mod.rs
+++ b/nexus/src/app/mod.rs
@@ -253,8 +253,10 @@ impl Nexus {
             dpd_clients.insert(*location, Arc::new(dpd_client));
         }
         for (location, config) in &config.pkg.mgd {
-            let mg_client = mg_admin_client::Client::new(&log, config.address)
-                .map_err(|e| format!("mg admin client: {e}"))?;
+            let mg_client = mg_admin_client::Client::new(
+                &format!("http://{}", config.address),
+                log.clone(),
+            );
             mg_clients.insert(*location, Arc::new(mg_client));
         }
         if config.pkg.dendrite.is_empty() {
@@ -310,10 +312,15 @@ impl Nexus {
                         for (location, addr) in &mappings {
                             let port = MGD_PORT;
                             let mgd_client = mg_admin_client::Client::new(
-                                &log,
-                                std::net::SocketAddr::new((*addr).into(), port),
-                            )
-                            .map_err(|e| format!("mg admin client: {e}"))?;
+                                &format!(
+                                    "http://{}",
+                                    &std::net::SocketAddr::new(
+                                        (*addr).into(),
+                                        port,
+                                    )
+                                ),
+                                log.clone(),
+                            );
                             mg_clients.insert(*location, Arc::new(mgd_client));
                         }
                         break;
@@ -903,22 +910,10 @@ impl Nexus {
             let port = MGD_PORT;
             let socketaddr =
                 std::net::SocketAddr::V6(SocketAddrV6::new(*addr, port, 0, 0));
-            let client = match mg_admin_client::Client::new(
-                &self.log.clone(),
-                socketaddr,
-            ) {
-                Ok(client) => client,
-                Err(e) => {
-                    error!(
-                        self.log,
-                        "error building mgd client";
-                        "location" => %location,
-                        "addr" => %addr,
-                        "error" => %e,
-                    );
-                    continue;
-                }
-            };
+            let client = mg_admin_client::Client::new(
+                format!("http://{}", socketaddr).as_str(),
+                self.log.clone(),
+            );
             clients.push((*location, client));
         }
         Ok(clients.into_iter().collect::<HashMap<_, _>>())
diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs
index 9c8d9cef50..94cd35519b 100644
--- a/nexus/src/external_api/http_entrypoints.rs
+++ b/nexus/src/external_api/http_entrypoints.rs
@@ -42,7 +42,6 @@ use nexus_db_queries::db::lookup::ImageParentLookup;
 use nexus_db_queries::db::model::Name;
 use nexus_db_queries::{authz, db::datastore::ProbeInfo};
 use nexus_types::external_api::shared::BfdStatus;
-use omicron_common::api::external::http_pagination::data_page_params_for;
 use omicron_common::api::external::http_pagination::marker_for_name;
 use omicron_common::api::external::http_pagination::marker_for_name_or_id;
 use omicron_common::api::external::http_pagination::name_or_id_pagination;
@@ -80,6 +79,9 @@ use omicron_common::api::external::TufRepoGetResponse;
 use omicron_common::api::external::TufRepoInsertResponse;
 use omicron_common::api::external::VpcFirewallRuleUpdateParams;
 use omicron_common::api::external::VpcFirewallRules;
+use omicron_common::api::external::{
+    http_pagination::data_page_params_for, AggregateBgpMessageHistory,
+};
 use omicron_common::bail_unless;
 use parse_display::Display;
 use propolis_client::support::tungstenite::protocol::frame::coding::CloseCode;
@@ -278,6 +280,7 @@ pub(crate) fn external_api() -> NexusApiDescription {
         api.register(networking_bgp_announce_set_create)?;
         api.register(networking_bgp_announce_set_list)?;
         api.register(networking_bgp_announce_set_delete)?;
+        api.register(networking_bgp_message_history)?;
 
         api.register(networking_bfd_enable)?;
         api.register(networking_bfd_disable)?;
@@ -3536,6 +3539,27 @@ async fn networking_bgp_status(
     apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
 }
 
+/// Get BGP router message history
+#[endpoint {
+    method = GET,
+    path = "/v1/system/networking/bgp-message-history",
+    tags = ["system/networking"],
+}]
+async fn networking_bgp_message_history(
+    rqctx: RequestContext<Arc<ServerContext>>,
+    query_params: Query<params::BgpRouteSelector>,
+) -> Result<HttpResponseOk<AggregateBgpMessageHistory>, HttpError> {
+    let apictx = rqctx.context();
+    let opctx = crate::context::op_context_for_external_api(&rqctx).await?;
+    let handler = async {
+        let nexus = &apictx.nexus;
+        let sel = query_params.into_inner();
+        let result = nexus.bgp_message_history(&opctx, &sel).await?;
+        Ok(HttpResponseOk(AggregateBgpMessageHistory::new(result)))
+    };
+    apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await
+}
+
 //TODO pagination? the normal by-name/by-id stuff does not work here
 /// Get imported IPv4 BGP routes
 #[endpoint {
diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs
index 7d2ff2596a..254bdabe8b 100644
--- a/nexus/tests/integration_tests/endpoints.rs
+++ b/nexus/tests/integration_tests/endpoints.rs
@@ -576,6 +576,8 @@ pub const DEMO_BGP_STATUS_URL: &'static str =
     "/v1/system/networking/bgp-status";
 pub const DEMO_BGP_ROUTES_IPV4_URL: &'static str =
     "/v1/system/networking/bgp-routes-ipv4?asn=47";
+pub const DEMO_BGP_MESSAGE_HISTORY_URL: &'static str =
+    "/v1/system/networking/bgp-message-history?asn=47";
 
 pub const DEMO_BFD_STATUS_URL: &'static str =
     "/v1/system/networking/bfd-status";
@@ -2247,6 +2249,15 @@ pub static VERIFY_ENDPOINTS: Lazy<Vec<VerifyEndpoint>> = Lazy::new(|| {
             ],
         },
 
+        VerifyEndpoint {
+            url: &DEMO_BGP_MESSAGE_HISTORY_URL,
+            visibility: Visibility::Public,
+            unprivileged_access: UnprivilegedAccess::None,
+            allowed_methods: vec![
+                AllowedMethod::GetNonexistent,
+            ],
+        },
+
         VerifyEndpoint {
             url: &DEMO_BFD_STATUS_URL,
             visibility: Visibility::Public,
diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt
index ba79d75f22..64413f396e 100644
--- a/nexus/tests/output/nexus_tags.txt
+++ b/nexus/tests/output/nexus_tags.txt
@@ -180,6 +180,7 @@ networking_bgp_config_create             POST     /v1/system/networking/bgp
 networking_bgp_config_delete             DELETE   /v1/system/networking/bgp
 networking_bgp_config_list               GET      /v1/system/networking/bgp
 networking_bgp_imported_routes_ipv4      GET      /v1/system/networking/bgp-routes-ipv4
+networking_bgp_message_history           GET      /v1/system/networking/bgp-message-history
 networking_bgp_status                    GET      /v1/system/networking/bgp-status
 networking_loopback_address_create       POST     /v1/system/networking/loopback-address
 networking_loopback_address_delete       DELETE   /v1/system/networking/loopback-address/{rack_id}/{switch_location}/{address}/{subnet_mask}
diff --git a/openapi/nexus.json b/openapi/nexus.json
index 7516a896a7..7d8e1909d2 100644
--- a/openapi/nexus.json
+++ b/openapi/nexus.json
@@ -6526,6 +6526,46 @@
         }
       }
     },
+    "/v1/system/networking/bgp-message-history": {
+      "get": {
+        "tags": [
+          "system/networking"
+        ],
+        "summary": "Get BGP router message history",
+        "operationId": "networking_bgp_message_history",
+        "parameters": [
+          {
+            "in": "query",
+            "name": "asn",
+            "description": "The ASN to filter on. Required.",
+            "required": true,
+            "schema": {
+              "type": "integer",
+              "format": "uint32",
+              "minimum": 0
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/AggregateBgpMessageHistory"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/v1/system/networking/bgp-routes-ipv4": {
       "get": {
         "tags": [
@@ -8930,6 +8970,22 @@
           "items"
         ]
       },
+      "AggregateBgpMessageHistory": {
+        "description": "BGP message history for rack switches.",
+        "type": "object",
+        "properties": {
+          "switch_histories": {
+            "description": "BGP history organized by switch.",
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/SwitchBgpHistory"
+            }
+          }
+        },
+        "required": [
+          "switch_histories"
+        ]
+      },
       "Baseboard": {
         "description": "Properties that uniquely identify an Oxide hardware component",
         "type": "object",
@@ -9373,6 +9429,7 @@
           "switch"
         ]
       },
+      "BgpMessageHistory": {},
       "BgpPeer": {
         "description": "A BGP peer configuration for an interface. Includes the set of announcements that will be advertised to the peer identified by `addr`. The `bgp_config` parameter is a reference to global BGP parameters. The `interface_name` indicates what interface the peer should be contacted on.",
         "type": "object",
@@ -16019,6 +16076,31 @@
           "time_modified"
         ]
       },
+      "SwitchBgpHistory": {
+        "description": "BGP message history for a particular switch.",
+        "type": "object",
+        "properties": {
+          "history": {
+            "description": "Message history indexed by peer address.",
+            "type": "object",
+            "additionalProperties": {
+              "$ref": "#/components/schemas/BgpMessageHistory"
+            }
+          },
+          "switch": {
+            "description": "Switch this message history is associated with.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/SwitchLocation"
+              }
+            ]
+          }
+        },
+        "required": [
+          "history",
+          "switch"
+        ]
+      },
       "SwitchInterfaceConfig": {
         "description": "A switch port interface configuration for a port settings object.",
         "type": "object",
diff --git a/package-manifest.toml b/package-manifest.toml
index 83c12b9174..e88f135985 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -535,10 +535,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "ee0896f5cf9ba0799f155dd8c95f6e1dabaf80ba"
+source.commit = "1d8b818b0fd000bc6d2739575c8868b0781dfe41"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/maghemite.sha256.txt
-source.sha256 = "f1407cb9aac188d6493d2b0f948c75aad2c36668ddf4ae2a1ed80e9dd395b35d"
+source.sha256 = "a3914f991ab1441486d42097d135df15c09604c02a8104810e634b05f2e3f525"
 output.type = "tarball"
 
 [package.mg-ddm]
@@ -551,10 +551,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "ee0896f5cf9ba0799f155dd8c95f6e1dabaf80ba"
+source.commit = "1d8b818b0fd000bc6d2739575c8868b0781dfe41"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mg-ddm.sha256.txt
-source.sha256 = "fae53cb39536dc92d97cb9610de65b0acbce285e685d7167b719ea6311844fec"
+source.sha256 = "e973385b276c7910a76a58151723f30c39e2f8a302a506992b0543a7e95616fb"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -566,10 +566,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "ee0896f5cf9ba0799f155dd8c95f6e1dabaf80ba"
+source.commit = "1d8b818b0fd000bc6d2739575c8868b0781dfe41"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mg-ddm.sha256.txt
-source.sha256 = "151aeb26414989cad571b3886786efbeeafd91c41a93a747c784cdc654d5876d"
+source.sha256 = "88065a5d5051c4e684ea15bd182c1d734d649df2336b7612108e2aee330b81a6"
 output.type = "zone"
 output.intermediate_only = true
 
diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml
index 52c6e61efa..d210b9b86b 100644
--- a/sled-agent/Cargo.toml
+++ b/sled-agent/Cargo.toml
@@ -20,7 +20,6 @@ chrono.workspace = true
 clap.workspace = true
 # Only used by the simulated sled agent.
 crucible-agent-client.workspace = true
-ddm-admin-client.workspace = true
 derive_more.workspace = true
 dns-server.workspace = true
 dns-service-client.workspace = true
@@ -48,6 +47,7 @@ nexus-client.workspace = true
 nexus-config.workspace = true
 nexus-types.workspace = true
 omicron-common.workspace = true
+omicron-ddm-admin-client.workspace = true
 once_cell.workspace = true
 oximeter.workspace = true
 oximeter-instruments.workspace = true
diff --git a/sled-agent/src/bootstrap/bootstore_setup.rs b/sled-agent/src/bootstrap/bootstore_setup.rs
index 120c7df000..e5079b978e 100644
--- a/sled-agent/src/bootstrap/bootstore_setup.rs
+++ b/sled-agent/src/bootstrap/bootstore_setup.rs
@@ -11,7 +11,7 @@ use super::config::BOOTSTORE_PORT;
 use super::server::StartError;
 use bootstore::schemes::v0 as bootstore;
 use camino::Utf8PathBuf;
-use ddm_admin_client::Client as DdmAdminClient;
+use omicron_ddm_admin_client::Client as DdmAdminClient;
 use sled_hardware_types::underlay::BootstrapInterface;
 use sled_hardware_types::Baseboard;
 use sled_storage::dataset::CLUSTER_DATASET;
diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs
index c70758f775..0cc0f22f5d 100644
--- a/sled-agent/src/bootstrap/early_networking.rs
+++ b/sled-agent/src/bootstrap/early_networking.rs
@@ -6,7 +6,6 @@
 
 use anyhow::{anyhow, Context};
 use bootstore::schemes::v0 as bootstore;
-use ddm_admin_client::DdmError;
 use dpd_client::types::{
     LinkCreate, LinkId, LinkSettings, PortId, PortSettings,
 };
@@ -17,10 +16,10 @@ use internal_dns::resolver::{ResolveError, Resolver as DnsResolver};
 use internal_dns::ServiceName;
 use ipnetwork::Ipv6Network;
 use mg_admin_client::types::{
-    AddStaticRoute4Request, ApplyRequest, BgpPeerConfig, Prefix4, StaticRoute4,
+    AddStaticRoute4Request, ApplyRequest, BgpPeerConfig, StaticRoute4,
     StaticRoute4List,
 };
-use mg_admin_client::Client as MgdClient;
+use mg_admin_client::{Client as MgdClient, Prefix4};
 use omicron_common::address::DENDRITE_PORT;
 use omicron_common::address::{MGD_PORT, MGS_PORT};
 use omicron_common::api::internal::shared::{
@@ -32,6 +31,7 @@ use omicron_common::backoff::{
     ExponentialBackoffBuilder,
 };
 use omicron_common::OMICRON_DPD_TAG;
+use omicron_ddm_admin_client::DdmError;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use slog::Logger;
@@ -446,14 +446,12 @@ impl<'a> EarlyNetworkSetup<'a> {
         }
 
         let mgd = MgdClient::new(
-            &self.log,
-            SocketAddrV6::new(switch_zone_underlay_ip, MGD_PORT, 0, 0).into(),
-        )
-        .map_err(|e| {
-            EarlyNetworkSetupError::MgdError(format!(
-                "initialize mgd client: {e}"
-            ))
-        })?;
+            &format!(
+                "http://{}",
+                &SocketAddrV6::new(switch_zone_underlay_ip, MGD_PORT, 0, 0)
+            ),
+            self.log.clone(),
+        );
 
         let mut config: Option<BgpConfig> = None;
         let mut bgp_peer_configs = HashMap::<String, Vec<BgpPeerConfig>>::new();
@@ -509,25 +507,21 @@ impl<'a> EarlyNetworkSetup<'a> {
 
         if !bgp_peer_configs.is_empty() {
             if let Some(config) = &config {
-                mgd.inner
-                    .bgp_apply(&ApplyRequest {
-                        asn: config.asn,
-                        peers: bgp_peer_configs,
-                        originate: config
-                            .originate
-                            .iter()
-                            .map(|x| Prefix4 {
-                                length: x.prefix(),
-                                value: x.ip(),
-                            })
-                            .collect(),
-                    })
-                    .await
-                    .map_err(|e| {
-                        EarlyNetworkSetupError::BgpConfigurationError(format!(
-                            "BGP peer configuration failed: {e}",
-                        ))
-                    })?;
+                mgd.bgp_apply(&ApplyRequest {
+                    asn: config.asn,
+                    peers: bgp_peer_configs,
+                    originate: config
+                        .originate
+                        .iter()
+                        .map(|x| Prefix4 { length: x.prefix(), value: x.ip() })
+                        .collect(),
+                })
+                .await
+                .map_err(|e| {
+                    EarlyNetworkSetupError::BgpConfigurationError(format!(
+                        "BGP peer configuration failed: {e}",
+                    ))
+                })?;
             }
         }
 
@@ -551,7 +545,7 @@ impl<'a> EarlyNetworkSetup<'a> {
                 rq.routes.list.push(sr);
             }
         }
-        mgd.inner.static_add_v4_route(&rq).await.map_err(|e| {
+        mgd.static_add_v4_route(&rq).await.map_err(|e| {
             EarlyNetworkSetupError::BgpConfigurationError(format!(
                 "static routing configuration failed: {e}",
             ))
diff --git a/sled-agent/src/bootstrap/pre_server.rs b/sled-agent/src/bootstrap/pre_server.rs
index 6769af8ca2..394cde7ac2 100644
--- a/sled-agent/src/bootstrap/pre_server.rs
+++ b/sled-agent/src/bootstrap/pre_server.rs
@@ -23,7 +23,6 @@ use crate::sled_agent::SledAgent;
 use crate::storage_monitor::UnderlayAccess;
 use camino::Utf8PathBuf;
 use cancel_safe_futures::TryStreamExt;
-use ddm_admin_client::Client as DdmAdminClient;
 use futures::stream;
 use futures::StreamExt;
 use illumos_utils::addrobj::AddrObject;
@@ -35,6 +34,7 @@ use illumos_utils::zone;
 use illumos_utils::zone::Zones;
 use omicron_common::address::Ipv6Subnet;
 use omicron_common::FileKv;
+use omicron_ddm_admin_client::Client as DdmAdminClient;
 use sled_hardware::underlay;
 use sled_hardware::DendriteAsic;
 use sled_hardware::SledMode;
diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs
index 47a8019ac5..d1f3fa0646 100644
--- a/sled-agent/src/bootstrap/server.rs
+++ b/sled-agent/src/bootstrap/server.rs
@@ -30,8 +30,6 @@ use crate::storage_monitor::UnderlayAccess;
 use bootstore::schemes::v0 as bootstore;
 use camino::Utf8PathBuf;
 use cancel_safe_futures::TryStreamExt;
-use ddm_admin_client::Client as DdmAdminClient;
-use ddm_admin_client::DdmError;
 use dropshot::HttpServer;
 use futures::StreamExt;
 use illumos_utils::dladm;
@@ -40,6 +38,8 @@ use illumos_utils::zone;
 use illumos_utils::zone::Zones;
 use omicron_common::ledger;
 use omicron_common::ledger::Ledger;
+use omicron_ddm_admin_client::Client as DdmAdminClient;
+use omicron_ddm_admin_client::DdmError;
 use sled_hardware::underlay;
 use sled_storage::dataset::CONFIG_DATASET;
 use sled_storage::manager::StorageHandle;
diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs
index 6b189f423c..0fae365242 100644
--- a/sled-agent/src/rack_setup/service.rs
+++ b/sled-agent/src/rack_setup/service.rs
@@ -86,7 +86,6 @@ use anyhow::{bail, Context};
 use bootstore::schemes::v0 as bootstore;
 use camino::Utf8PathBuf;
 use chrono::Utc;
-use ddm_admin_client::{Client as DdmAdminClient, DdmError};
 use internal_dns::resolver::{DnsError, Resolver as DnsResolver};
 use internal_dns::ServiceName;
 use nexus_client::{
@@ -100,6 +99,7 @@ use omicron_common::backoff::{
     retry_notify, retry_policy_internal_service_aggressive, BackoffError,
 };
 use omicron_common::ledger::{self, Ledger, Ledgerable};
+use omicron_ddm_admin_client::{Client as DdmAdminClient, DdmError};
 use serde::{Deserialize, Serialize};
 use sled_agent_client::{
     types as SledAgentTypes, Client as SledAgentClient, Error as SledAgentError,
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
index e0a4693877..55ebdccb5c 100644
--- a/sled-agent/src/services.rs
+++ b/sled-agent/src/services.rs
@@ -41,7 +41,6 @@ use crate::zone_bundle::BundleError;
 use crate::zone_bundle::ZoneBundler;
 use anyhow::anyhow;
 use camino::{Utf8Path, Utf8PathBuf};
-use ddm_admin_client::{Client as DdmAdminClient, DdmError};
 use dpd_client::{types as DpdTypes, Client as DpdClient, Error as DpdError};
 use dropshot::HandlerTaskMode;
 use illumos_utils::addrobj::AddrObject;
@@ -87,6 +86,7 @@ use omicron_common::backoff::{
     retry_notify, retry_policy_internal_service_aggressive, BackoffError,
 };
 use omicron_common::ledger::{self, Ledger, Ledgerable};
+use omicron_ddm_admin_client::{Client as DdmAdminClient, DdmError};
 use once_cell::sync::OnceCell;
 use rand::prelude::SliceRandom;
 use sled_hardware::is_gimlet;
diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs
index 713b10428f..aa051e6e5b 100644
--- a/sled-agent/src/sled_agent.rs
+++ b/sled-agent/src/sled_agent.rs
@@ -34,7 +34,6 @@ use crate::zone_bundle;
 use crate::zone_bundle::BundleError;
 use bootstore::schemes::v0 as bootstore;
 use camino::Utf8PathBuf;
-use ddm_admin_client::Client as DdmAdminClient;
 use derive_more::From;
 use dropshot::HttpError;
 use futures::stream::FuturesUnordered;
@@ -65,6 +64,7 @@ use omicron_common::backoff::{
     retry_notify, retry_policy_internal_service,
     retry_policy_internal_service_aggressive, BackoffError,
 };
+use omicron_ddm_admin_client::Client as DdmAdminClient;
 use oximeter::types::ProducerRegistry;
 use sled_hardware::{underlay, HardwareManager};
 use sled_hardware_types::underlay::BootstrapInterface;
@@ -1174,7 +1174,7 @@ pub enum AddSledError {
         err: bootstrap_agent_client::Error,
     },
     #[error("Failed to connect to DDM")]
-    DdmAdminClient(#[source] ddm_admin_client::DdmError),
+    DdmAdminClient(#[source] omicron_ddm_admin_client::DdmError),
     #[error("Failed to learn bootstrap ip for {0:?}")]
     NotFound(BaseboardId),
     #[error("Failed to initialize {sled_id}: {err}")]
diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version
index 14ff749506..73cbfefad8 100644
--- a/tools/maghemite_ddm_openapi_version
+++ b/tools/maghemite_ddm_openapi_version
@@ -1,2 +1,2 @@
-COMMIT="ee0896f5cf9ba0799f155dd8c95f6e1dabaf80ba"
+COMMIT="a19e88979ab8ab9a0136854182c410c29047bfcb"
 SHA2="0b0dbc2f8bbc5d2d9be92d64c4865f8f9335355aae62f7de9f67f81dfb3f1803"
diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version
index a2becb4efc..f46f293368 100644
--- a/tools/maghemite_mg_openapi_version
+++ b/tools/maghemite_mg_openapi_version
@@ -1,2 +1,2 @@
-COMMIT="ee0896f5cf9ba0799f155dd8c95f6e1dabaf80ba"
-SHA2="0ac038bbaa54d0ae0ac5ccaeff48f03070618372cca26c9d09b716b909bf9355"
+COMMIT="a19e88979ab8ab9a0136854182c410c29047bfcb"
+SHA2="768b38ad9b01299be66a899241158123fc90ea240708a517c8acfa52b4b2eb8e"
diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums
index 2f58a88808..face9b9b1b 100644
--- a/tools/maghemite_mgd_checksums
+++ b/tools/maghemite_mgd_checksums
@@ -1,2 +1,2 @@
-CIDL_SHA256="151aeb26414989cad571b3886786efbeeafd91c41a93a747c784cdc654d5876d"
-MGD_LINUX_SHA256="943b0a52d279bde55a419e2cdb24873acc32703bc97bd599376117ee0edc1511"
\ No newline at end of file
+CIDL_SHA256="813fe3d408f538a9d7121a616fce5f87e6c42cf2800e20c1a2759c24c9e8b77b"
+MGD_LINUX_SHA256="03bf4e9fbe7d46be5dbcf83f9cd67520383f843767939e06d0db93ce4e584111"
\ No newline at end of file
diff --git a/tools/update_maghemite.sh b/tools/update_maghemite.sh
index 583a75ab22..5c146820a3 100755
--- a/tools/update_maghemite.sh
+++ b/tools/update_maghemite.sh
@@ -19,50 +19,9 @@ PACKAGES=(
   "mg-ddm"
   "mgd"
 )
-
 REPO="oxidecomputer/maghemite"
-
 . "$SOURCE_DIR/update_helpers.sh"
 
-function update_openapi {
-    TARGET_COMMIT="$1"
-    DRY_RUN="$2"
-    DAEMON="$3"
-    SHA=$(get_sha "$REPO" "$TARGET_COMMIT" "${DAEMON}-admin.json" "openapi")
-    OUTPUT=$(printf "COMMIT=\"%s\"\nSHA2=\"%s\"\n" "$TARGET_COMMIT" "$SHA")
-
-    if [ -n "$DRY_RUN" ]; then
-        OPENAPI_PATH="/dev/null"
-    else
-        OPENAPI_PATH="$SOURCE_DIR/maghemite_${DAEMON}_openapi_version"
-    fi
-    echo "Updating Maghemite OpenAPI from: $TARGET_COMMIT"
-    set -x
-    echo "$OUTPUT" > "$OPENAPI_PATH"
-    set +x
-}
-
-function update_mgd {
-    TARGET_COMMIT="$1"
-    DRY_RUN="$2"
-    DAEMON="$3"
-    SHA=$(get_sha "$REPO" "$TARGET_COMMIT" "mgd" "image")
-    OUTPUT=$(printf "CIDL_SHA256=\"%s\"\n" "$SHA")
-
-    SHA_LINUX=$(get_sha "$REPO" "$TARGET_COMMIT" "mgd" "linux")
-    OUTPUT_LINUX=$(printf "MGD_LINUX_SHA256=\"%s\"\n" "$SHA_LINUX")
-
-    if [ -n "$DRY_RUN" ]; then
-        MGD_PATH="/dev/null"
-    else
-        MGD_PATH="$SOURCE_DIR/maghemite_mgd_checksums"
-    fi
-    echo "Updating Maghemite mgd from: $TARGET_COMMIT"
-    set -x
-    printf "$OUTPUT\n$OUTPUT_LINUX" > $MGD_PATH
-    set +x
-}
-
 function main {
     TARGET_COMMIT=""
     DRY_RUN=""
@@ -83,10 +42,6 @@ function main {
     TARGET_COMMIT=$(get_latest_commit_from_gh "$REPO" "$TARGET_COMMIT")
     install_toml2json
     do_update_packages "$TARGET_COMMIT" "$DRY_RUN" "$REPO" "${PACKAGES[@]}"
-    update_openapi "$TARGET_COMMIT" "$DRY_RUN" ddm
-    update_openapi "$TARGET_COMMIT" "$DRY_RUN" mg
-    update_mgd "$TARGET_COMMIT" "$DRY_RUN"
-    do_update_packages "$TARGET_COMMIT" "$DRY_RUN" "$REPO" "${PACKAGES[@]}"
 }
 
 main "$@"
diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml
index 4b6e6d6f5f..ef9e0c2378 100644
--- a/wicketd/Cargo.toml
+++ b/wicketd/Cargo.toml
@@ -46,7 +46,7 @@ toml.workspace = true
 uuid.workspace = true
 
 bootstrap-agent-client.workspace = true
-ddm-admin-client.workspace = true
+omicron-ddm-admin-client.workspace = true
 gateway-client.workspace = true
 installinator-artifactd.workspace = true
 installinator-common.workspace = true
diff --git a/wicketd/src/bootstrap_addrs.rs b/wicketd/src/bootstrap_addrs.rs
index f7256ae6ac..dd13fe0f44 100644
--- a/wicketd/src/bootstrap_addrs.rs
+++ b/wicketd/src/bootstrap_addrs.rs
@@ -2,8 +2,8 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use ddm_admin_client::Client as DdmAdminClient;
 use futures::stream::FuturesUnordered;
+use omicron_ddm_admin_client::Client as DdmAdminClient;
 use sled_hardware_types::underlay::BootstrapInterface;
 use sled_hardware_types::Baseboard;
 use slog::warn;
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index 5efbb6c1f1..c4fd52aef7 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -28,8 +28,8 @@ byteorder = { version = "1.5.0" }
 bytes = { version = "1.5.0", features = ["serde"] }
 chrono = { version = "0.4.34", features = ["serde"] }
 cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] }
-clap = { version = "4.5.1", features = ["cargo", "derive", "env", "wrap_help"] }
-clap_builder = { version = "4.5.1", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
+clap = { version = "4.5.1", features = ["cargo", "derive", "env", "unstable-styles", "wrap_help"] }
+clap_builder = { version = "4.5.1", default-features = false, features = ["cargo", "env", "std", "suggestions", "unstable-styles", "usage", "wrap_help"] }
 console = { version = "0.15.8" }
 const-oid = { version = "0.9.6", default-features = false, features = ["db", "std"] }
 crossbeam-epoch = { version = "0.9.18" }
@@ -65,7 +65,7 @@ itertools = { version = "0.10.5" }
 lalrpop-util = { version = "0.19.12" }
 lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] }
 libc = { version = "0.2.153", features = ["extra_traits"] }
-log = { version = "0.4.20", default-features = false, features = ["std"] }
+log = { version = "0.4.21", default-features = false, features = ["std"] }
 managed = { version = "0.8.0", default-features = false, features = ["alloc", "map"] }
 memchr = { version = "2.7.1" }
 nom = { version = "7.1.3" }
@@ -87,7 +87,7 @@ regex-automata = { version = "0.4.5", default-features = false, features = ["dfa
 regex-syntax = { version = "0.8.2" }
 reqwest = { version = "0.11.24", features = ["blocking", "cookies", "json", "rustls-tls", "stream"] }
 ring = { version = "0.17.8", features = ["std"] }
-schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid1"] }
+schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid", "uuid1"] }
 semver = { version = "1.0.22", features = ["serde"] }
 serde = { version = "1.0.197", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1.0.114", features = ["raw_value", "unbounded_depth"] }
@@ -98,14 +98,16 @@ spin = { version = "0.9.8" }
 string_cache = { version = "0.8.7" }
 subtle = { version = "2.5.0" }
 syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] }
-syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.51", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
+syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.52", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
 time = { version = "0.3.34", features = ["formatting", "local-offset", "macros", "parsing"] }
 tokio = { version = "1.36.0", features = ["full", "test-util"] }
 tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] }
 tokio-stream = { version = "0.1.14", features = ["net"] }
 tokio-util = { version = "0.7.10", features = ["codec", "io-util"] }
 toml = { version = "0.7.8" }
+toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] }
 toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.6", features = ["serde"] }
+toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] }
 tracing = { version = "0.1.40", features = ["log"] }
 trust-dns-proto = { version = "0.22.0" }
 unicode-bidi = { version = "0.3.15" }
@@ -133,8 +135,8 @@ byteorder = { version = "1.5.0" }
 bytes = { version = "1.5.0", features = ["serde"] }
 chrono = { version = "0.4.34", features = ["serde"] }
 cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] }
-clap = { version = "4.5.1", features = ["cargo", "derive", "env", "wrap_help"] }
-clap_builder = { version = "4.5.1", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
+clap = { version = "4.5.1", features = ["cargo", "derive", "env", "unstable-styles", "wrap_help"] }
+clap_builder = { version = "4.5.1", default-features = false, features = ["cargo", "env", "std", "suggestions", "unstable-styles", "usage", "wrap_help"] }
 console = { version = "0.15.8" }
 const-oid = { version = "0.9.6", default-features = false, features = ["db", "std"] }
 crossbeam-epoch = { version = "0.9.18" }
@@ -170,7 +172,7 @@ itertools = { version = "0.10.5" }
 lalrpop-util = { version = "0.19.12" }
 lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] }
 libc = { version = "0.2.153", features = ["extra_traits"] }
-log = { version = "0.4.20", default-features = false, features = ["std"] }
+log = { version = "0.4.21", default-features = false, features = ["std"] }
 managed = { version = "0.8.0", default-features = false, features = ["alloc", "map"] }
 memchr = { version = "2.7.1" }
 nom = { version = "7.1.3" }
@@ -192,7 +194,7 @@ regex-automata = { version = "0.4.5", default-features = false, features = ["dfa
 regex-syntax = { version = "0.8.2" }
 reqwest = { version = "0.11.24", features = ["blocking", "cookies", "json", "rustls-tls", "stream"] }
 ring = { version = "0.17.8", features = ["std"] }
-schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid1"] }
+schemars = { version = "0.8.16", features = ["bytes", "chrono", "uuid", "uuid1"] }
 semver = { version = "1.0.22", features = ["serde"] }
 serde = { version = "1.0.197", features = ["alloc", "derive", "rc"] }
 serde_json = { version = "1.0.114", features = ["raw_value", "unbounded_depth"] }
@@ -203,7 +205,7 @@ spin = { version = "0.9.8" }
 string_cache = { version = "0.8.7" }
 subtle = { version = "2.5.0" }
 syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] }
-syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.51", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
+syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.52", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
 time = { version = "0.3.34", features = ["formatting", "local-offset", "macros", "parsing"] }
 time-macros = { version = "0.2.17", default-features = false, features = ["formatting", "parsing"] }
 tokio = { version = "1.36.0", features = ["full", "test-util"] }
@@ -211,7 +213,9 @@ tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serd
 tokio-stream = { version = "0.1.14", features = ["net"] }
 tokio-util = { version = "0.7.10", features = ["codec", "io-util"] }
 toml = { version = "0.7.8" }
+toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] }
 toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.6", features = ["serde"] }
+toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] }
 tracing = { version = "0.1.40", features = ["log"] }
 trust-dns-proto = { version = "0.22.0" }
 unicode-bidi = { version = "0.3.15" }
@@ -268,8 +272,6 @@ dof = { version = "0.3.0", default-features = false, features = ["des"] }
 mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
-toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] }
-toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] }
 
 [target.x86_64-unknown-illumos.build-dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.2", default-features = false, features = ["std"] }
@@ -277,7 +279,5 @@ dof = { version = "0.3.0", default-features = false, features = ["des"] }
 mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
-toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] }
-toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] }
 
 ### END HAKARI SECTION

From f8c8bb1e290810ff2e8754486bf034e5fb111660 Mon Sep 17 00:00:00 2001
From: liffy <629075+lifning@users.noreply.github.com>
Date: Mon, 18 Mar 2024 22:37:21 -0700
Subject: [PATCH 129/157] sled-agent: Don't block during instance creation
 request from nexus (#4691)

Alleviating request timeouts occurring when propolis zone installation
takes too long - #3927 - by making the zone installation not happen
during a request handler, and instead having sled-agent report the
result of instance creation back to nexus with an internal cpapi call.

In this world where instance create calls to nexus now no longer block
on sled-agent's propolis zone installation finishing, care must be taken
in e.g. integration tests not to assume you can, say, connect to the
instance as soon as the instance create call returns.

Accordingly, integration tests have been changed to poll until the
instance is ready, and additionally some unit tests have been added
for instance creation in sled-agent's Instance and InstanceManager.
---
 end-to-end-tests/src/instance_launch.rs    |  70 ++-
 illumos-utils/src/running_zone.rs          |  48 +-
 nexus/src/app/instance.rs                  |   3 +
 nexus/tests/integration_tests/instances.rs |  21 +-
 sled-agent/src/fakes/nexus.rs              |  34 +-
 sled-agent/src/instance.rs                 | 686 ++++++++++++++++++++-
 sled-agent/src/instance_manager.rs         |  23 +-
 sled-agent/src/nexus.rs                    |  10 +
 sled-agent/src/params.rs                   |   2 +-
 sled-agent/src/sim/sled_agent.rs           |  26 +-
 sled-agent/src/vmm_reservoir.rs            |  13 +
 11 files changed, 912 insertions(+), 24 deletions(-)

diff --git a/end-to-end-tests/src/instance_launch.rs b/end-to-end-tests/src/instance_launch.rs
index 1aae46fe98..377fef4c0b 100644
--- a/end-to-end-tests/src/instance_launch.rs
+++ b/end-to-end-tests/src/instance_launch.rs
@@ -7,7 +7,7 @@ use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError};
 use oxide_client::types::{
     ByteCount, DiskCreate, DiskSource, ExternalIp, ExternalIpCreate,
     InstanceCpuCount, InstanceCreate, InstanceDiskAttachment,
-    InstanceNetworkInterfaceAttachment, SshKeyCreate,
+    InstanceNetworkInterfaceAttachment, InstanceState, SshKeyCreate,
 };
 use oxide_client::{ClientDisksExt, ClientInstancesExt, ClientSessionExt};
 use russh::{ChannelMsg, Disconnect};
@@ -15,7 +15,6 @@ use russh_keys::key::{KeyPair, PublicKey};
 use russh_keys::PublicKeyBase64;
 use std::sync::Arc;
 use std::time::Duration;
-use tokio::time::sleep;
 
 #[tokio::test]
 async fn instance_launch() -> Result<()> {
@@ -106,6 +105,19 @@ async fn instance_launch() -> Result<()> {
             type Error =
                 CondCheckError<oxide_client::Error<oxide_client::types::Error>>;
 
+            let instance_state = ctx
+                .client
+                .instance_view()
+                .project(ctx.project_name.clone())
+                .instance(instance.name.clone())
+                .send()
+                .await?
+                .run_state;
+
+            if instance_state == InstanceState::Starting {
+                return Err(Error::NotYet);
+            }
+
             let data = String::from_utf8_lossy(
                 &ctx.client
                     .instance_serial_console()
@@ -188,19 +200,49 @@ async fn instance_launch() -> Result<()> {
 
     // check that we saw it on the console
     eprintln!("waiting for serial console");
-    sleep(Duration::from_secs(5)).await;
-    let data = String::from_utf8_lossy(
-        &ctx.client
-            .instance_serial_console()
-            .project(ctx.project_name.clone())
-            .instance(instance.name.clone())
-            .most_recent(1024 * 1024)
-            .max_bytes(1024 * 1024)
-            .send()
-            .await?
-            .data,
+
+    let data = wait_for_condition(
+        || async {
+            type Error =
+                CondCheckError<oxide_client::Error<oxide_client::types::Error>>;
+
+            let instance_state = ctx
+                .client
+                .instance_view()
+                .project(ctx.project_name.clone())
+                .instance(instance.name.clone())
+                .send()
+                .await?
+                .run_state;
+
+            if instance_state == InstanceState::Starting {
+                return Err(Error::NotYet);
+            }
+
+            let data = String::from_utf8_lossy(
+                &ctx.client
+                    .instance_serial_console()
+                    .project(ctx.project_name.clone())
+                    .instance(instance.name.clone())
+                    .most_recent(1024 * 1024)
+                    .max_bytes(1024 * 1024)
+                    .send()
+                    .await
+                    .map_err(|_e| Error::NotYet)?
+                    .data,
+            )
+            .into_owned();
+            if data.contains("-----END SSH HOST KEY KEYS-----") {
+                Ok(data)
+            } else {
+                Err(Error::NotYet)
+            }
+        },
+        &Duration::from_secs(5),
+        &Duration::from_secs(300),
     )
-    .into_owned();
+    .await?;
+
     ensure!(
         data.contains("Hello, Oxide!"),
         "string not seen on console\n{}",
diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs
index 0dd8f85e4e..c529a1b6d4 100644
--- a/illumos-utils/src/running_zone.rs
+++ b/illumos-utils/src/running_zone.rs
@@ -1193,29 +1193,55 @@ impl ZoneBuilderFactory {
 /// Created by [ZoneBuilderFactory].
 #[derive(Default)]
 pub struct ZoneBuilder<'a> {
+    /// Logger to which status messages are written during zone installation.
     log: Option<Logger>,
+    /// Allocates the NIC used for control plane communication.
     underlay_vnic_allocator: Option<&'a VnicAllocator<Etherstub>>,
+    /// Filesystem path at which the installed zone will reside.
     zone_root_path: Option<&'a Utf8Path>,
+    /// The directories that will be searched for the image tarball for the
+    /// provided zone type ([`Self::with_zone_type`]).
     zone_image_paths: Option<&'a [Utf8PathBuf]>,
+    /// The name of the type of zone being created (e.g. "propolis-server")
     zone_type: Option<&'a str>,
-    unique_name: Option<Uuid>, // actually optional
+    /// Unique ID of the instance of the zone being created. (optional)
+    // *actually* optional (in contrast to other fields that are `Option` for
+    // builder purposes - that is, skipping this field in the builder will
+    // still result in an `Ok(InstalledZone)` from `.install()`, rather than
+    // an `Err(InstallZoneError::IncompleteBuilder)`.
+    unique_name: Option<Uuid>,
+    /// ZFS datasets to be accessed from within the zone.
     datasets: Option<&'a [zone::Dataset]>,
+    /// Filesystems to mount within the zone.
     filesystems: Option<&'a [zone::Fs]>,
+    /// Additional network device names to add to the zone.
     data_links: Option<&'a [String]>,
+    /// Device nodes to pass through to the zone.
     devices: Option<&'a [zone::Device]>,
+    /// OPTE devices for the guest network interfaces.
     opte_ports: Option<Vec<(Port, PortTicket)>>,
-    bootstrap_vnic: Option<Link>, // actually optional
+    /// NIC to use for creating a bootstrap address on the switch zone.
+    // actually optional (as above)
+    bootstrap_vnic: Option<Link>,
+    /// Physical NICs possibly provisioned to the zone.
     links: Option<Vec<Link>>,
+    /// The maximum set of privileges any process in this zone can obtain.
     limit_priv: Option<Vec<String>>,
+    /// For unit tests only: if `Some`, then no actual zones will be installed
+    /// by this builder, and minimal facsimiles of them will be placed in
+    /// temporary directories according to the contents of the provided
+    /// `FakeZoneBuilderConfig`.
     fake_cfg: Option<FakeZoneBuilderConfig>,
 }
 
 impl<'a> ZoneBuilder<'a> {
+    /// Logger to which status messages are written during zone installation.
     pub fn with_log(mut self, log: Logger) -> Self {
         self.log = Some(log);
         self
     }
 
+    /// Allocates the NIC used for control plane communication.
     pub fn with_underlay_vnic_allocator(
         mut self,
         vnic_allocator: &'a VnicAllocator<Etherstub>,
@@ -1224,11 +1250,14 @@ impl<'a> ZoneBuilder<'a> {
         self
     }
 
+    /// Filesystem path at which the installed zone will reside.
     pub fn with_zone_root_path(mut self, root_path: &'a Utf8Path) -> Self {
         self.zone_root_path = Some(root_path);
         self
     }
 
+    /// The directories that will be searched for the image tarball for the
+    /// provided zone type ([`Self::with_zone_type`]).
     pub fn with_zone_image_paths(
         mut self,
         image_paths: &'a [Utf8PathBuf],
@@ -1237,56 +1266,68 @@ impl<'a> ZoneBuilder<'a> {
         self
     }
 
+    /// The name of the type of zone being created (e.g. "propolis-server")
     pub fn with_zone_type(mut self, zone_type: &'a str) -> Self {
         self.zone_type = Some(zone_type);
         self
     }
 
+    /// Unique ID of the instance of the zone being created. (optional)
     pub fn with_unique_name(mut self, uuid: Uuid) -> Self {
         self.unique_name = Some(uuid);
         self
     }
 
+    /// ZFS datasets to be accessed from within the zone.
     pub fn with_datasets(mut self, datasets: &'a [zone::Dataset]) -> Self {
         self.datasets = Some(datasets);
         self
     }
 
+    /// Filesystems to mount within the zone.
     pub fn with_filesystems(mut self, filesystems: &'a [zone::Fs]) -> Self {
         self.filesystems = Some(filesystems);
         self
     }
 
+    /// Additional network device names to add to the zone.
     pub fn with_data_links(mut self, links: &'a [String]) -> Self {
         self.data_links = Some(links);
         self
     }
 
+    /// Device nodes to pass through to the zone.
     pub fn with_devices(mut self, devices: &'a [zone::Device]) -> Self {
         self.devices = Some(devices);
         self
     }
 
+    /// OPTE devices for the guest network interfaces.
     pub fn with_opte_ports(mut self, ports: Vec<(Port, PortTicket)>) -> Self {
         self.opte_ports = Some(ports);
         self
     }
 
+    /// NIC to use for creating a bootstrap address on the switch zone.
+    /// (optional)
     pub fn with_bootstrap_vnic(mut self, vnic: Link) -> Self {
         self.bootstrap_vnic = Some(vnic);
         self
     }
 
+    /// Physical NICs possibly provisioned to the zone.
     pub fn with_links(mut self, links: Vec<Link>) -> Self {
         self.links = Some(links);
         self
     }
 
+    /// The maximum set of privileges any process in this zone can obtain.
     pub fn with_limit_priv(mut self, limit_priv: Vec<String>) -> Self {
         self.limit_priv = Some(limit_priv);
         self
     }
 
+    // (used in unit tests)
     fn fake_install(self) -> Result<InstalledZone, InstallZoneError> {
         let zone = self
             .zone_type
@@ -1324,6 +1365,9 @@ impl<'a> ZoneBuilder<'a> {
         .ok_or(InstallZoneError::IncompleteBuilder)
     }
 
+    /// Create the zone with the provided parameters.
+    /// Returns `Err(InstallZoneError::IncompleteBuilder)` if a necessary
+    /// parameter was not provided.
     pub async fn install(self) -> Result<InstalledZone, InstallZoneError> {
         if self.fake_cfg.is_some() {
             return self.fake_install();
diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs
index 2d09078e18..2300bd56f2 100644
--- a/nexus/src/app/instance.rs
+++ b/nexus/src/app/instance.rs
@@ -986,6 +986,9 @@ impl super::Nexus {
                 //
                 // If the operation failed, kick the sled agent error back up to
                 // the caller to let it decide how to handle it.
+                //
+                // When creating the zone for the first time, we just get
+                // Ok(None) here, which is a no-op in write_returned_instance_state.
                 match instance_put_result {
                     Ok(state) => self
                         .write_returned_instance_state(&instance_id, state)
diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs
index 5d8bcc2136..128ed8967b 100644
--- a/nexus/tests/integration_tests/instances.rs
+++ b/nexus/tests/integration_tests/instances.rs
@@ -68,6 +68,7 @@ use sled_agent_client::TestInterfaces as _;
 use std::convert::TryFrom;
 use std::net::Ipv4Addr;
 use std::sync::Arc;
+use std::time::Duration;
 use uuid::Uuid;
 
 use dropshot::test_util::ClientTestContext;
@@ -81,6 +82,8 @@ use nexus_test_utils::resource_helpers::{
 use nexus_test_utils_macros::nexus_test;
 use nexus_types::external_api::shared::SiloRole;
 use omicron_sled_agent::sim;
+use omicron_test_utils::dev::poll;
+use omicron_test_utils::dev::poll::CondCheckError;
 
 type ControlPlaneTestContext =
     nexus_test_utils::ControlPlaneTestContext<omicron_nexus::Server>;
@@ -3795,10 +3798,22 @@ async fn test_instance_serial(cptestctx: &ControlPlaneTestContext) {
 
     // Create an instance and poke it to ensure it's running.
     let instance = create_instance(client, PROJECT_NAME, instance_name).await;
-    instance_simulate(nexus, &instance.identity.id).await;
-    let instance_next = instance_get(&client, &instance_url).await;
+    let instance_next = poll::wait_for_condition(
+        || async {
+            instance_simulate(nexus, &instance.identity.id).await;
+            let instance_next = instance_get(&client, &instance_url).await;
+            if instance_next.runtime.run_state == InstanceState::Running {
+                Ok(instance_next)
+            } else {
+                Err(CondCheckError::<()>::NotYet)
+            }
+        },
+        &Duration::from_secs(5),
+        &Duration::from_secs(60),
+    )
+    .await
+    .unwrap();
     identity_eq(&instance.identity, &instance_next.identity);
-    assert_eq!(instance_next.runtime.run_state, InstanceState::Running);
     assert!(
         instance_next.runtime.time_run_state_updated
             > instance.runtime.time_run_state_updated
diff --git a/sled-agent/src/fakes/nexus.rs b/sled-agent/src/fakes/nexus.rs
index 5920d40f6c..de37b77bcd 100644
--- a/sled-agent/src/fakes/nexus.rs
+++ b/sled-agent/src/fakes/nexus.rs
@@ -15,7 +15,9 @@ use hyper::Body;
 use internal_dns::ServiceName;
 use nexus_client::types::SledAgentInfo;
 use omicron_common::api::external::Error;
-use omicron_common::api::internal::nexus::UpdateArtifactId;
+use omicron_common::api::internal::nexus::{
+    SledInstanceState, UpdateArtifactId,
+};
 use schemars::JsonSchema;
 use serde::Deserialize;
 use uuid::Uuid;
@@ -44,6 +46,14 @@ pub trait FakeNexusServer: Send + Sync {
     ) -> Result<(), Error> {
         Err(Error::internal_error("Not implemented"))
     }
+
+    fn cpapi_instances_put(
+        &self,
+        _instance_id: Uuid,
+        _new_runtime_state: SledInstanceState,
+    ) -> Result<(), Error> {
+        Err(Error::internal_error("Not implemented"))
+    }
 }
 
 /// Describes the server context type.
@@ -107,11 +117,33 @@ async fn sled_agent_put(
     Ok(HttpResponseUpdatedNoContent())
 }
 
+#[derive(Deserialize, JsonSchema)]
+struct InstancePathParam {
+    instance_id: Uuid,
+}
+#[endpoint {
+    method = PUT,
+    path = "/instances/{instance_id}",
+}]
+async fn cpapi_instances_put(
+    request_context: RequestContext<ServerContext>,
+    path_params: Path<InstancePathParam>,
+    new_runtime_state: TypedBody<SledInstanceState>,
+) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+    let context = request_context.context();
+    context.cpapi_instances_put(
+        path_params.into_inner().instance_id,
+        new_runtime_state.into_inner(),
+    )?;
+    Ok(HttpResponseUpdatedNoContent())
+}
+
 fn api() -> ApiDescription<ServerContext> {
     let mut api = ApiDescription::new();
     api.register(cpapi_artifact_download).unwrap();
     api.register(sled_agent_get).unwrap();
     api.register(sled_agent_put).unwrap();
+    api.register(cpapi_instances_put).unwrap();
     api
 }
 
diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs
index 6b69de8bb5..7b76466964 100644
--- a/sled-agent/src/instance.rs
+++ b/sled-agent/src/instance.rs
@@ -30,7 +30,6 @@ use illumos_utils::link::VnicAllocator;
 use illumos_utils::opte::{DhcpCfg, PortManager};
 use illumos_utils::running_zone::{RunningZone, ZoneBuilderFactory};
 use illumos_utils::svc::wait_for_service;
-use illumos_utils::zone::Zones;
 use illumos_utils::zone::PROPOLIS_ZONE_PREFIX;
 use omicron_common::address::NEXUS_INTERNAL_PORT;
 use omicron_common::api::internal::nexus::{
@@ -52,6 +51,11 @@ use std::sync::Arc;
 use tokio::sync::{mpsc, oneshot};
 use uuid::Uuid;
 
+#[cfg(test)]
+use illumos_utils::zone::MockZones as Zones;
+#[cfg(not(test))]
+use illumos_utils::zone::Zones;
+
 // The depth of the request queue for the instance.
 const QUEUE_SIZE: usize = 32;
 
@@ -1515,3 +1519,683 @@ impl InstanceRunner {
         out
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::fakes::nexus::{FakeNexusServer, ServerContext};
+    use crate::nexus::NexusClient;
+    use crate::vmm_reservoir::VmmReservoirManagerHandle;
+    use crate::zone_bundle::CleanupContext;
+    use camino_tempfile::Utf8TempDir;
+    use dns_server::dns_server::ServerHandle as DnsServerHandle;
+    use dropshot::test_util::LogContext;
+    use dropshot::{HandlerTaskMode, HttpServer};
+    use illumos_utils::dladm::MockDladm;
+    use illumos_utils::dladm::__mock_MockDladm::__create_vnic::Context as MockDladmCreateVnicContext;
+    use illumos_utils::dladm::__mock_MockDladm::__delete_vnic::Context as MockDladmDeleteVnicContext;
+    use illumos_utils::opte::params::DhcpConfig;
+    use illumos_utils::svc::__wait_for_service::Context as MockWaitForServiceContext;
+    use illumos_utils::zone::MockZones;
+    use illumos_utils::zone::__mock_MockZones::__boot::Context as MockZonesBootContext;
+    use illumos_utils::zone::__mock_MockZones::__id::Context as MockZonesIdContext;
+    use illumos_utils::zpool::ZpoolName;
+    use internal_dns::resolver::Resolver;
+    use internal_dns::ServiceName;
+    use omicron_common::api::external::{
+        ByteCount, Generation, Hostname, InstanceCpuCount, InstanceState,
+    };
+    use omicron_common::api::internal::nexus::InstanceProperties;
+    use sled_storage::disk::{RawDisk, SyntheticDisk};
+    use sled_storage::manager::FakeStorageManager;
+    use std::net::Ipv6Addr;
+    use std::str::FromStr;
+    use tokio::sync::watch::Receiver;
+    use tokio::time::timeout;
+
+    const TIMEOUT_DURATION: tokio::time::Duration =
+        tokio::time::Duration::from_secs(30);
+
+    #[derive(Default, Clone)]
+    enum ReceivedInstanceState {
+        #[default]
+        None,
+        InstancePut(SledInstanceState),
+    }
+
+    struct NexusServer {
+        observed_runtime_state:
+            tokio::sync::watch::Sender<ReceivedInstanceState>,
+    }
+    impl FakeNexusServer for NexusServer {
+        fn cpapi_instances_put(
+            &self,
+            _instance_id: Uuid,
+            new_runtime_state: SledInstanceState,
+        ) -> Result<(), omicron_common::api::external::Error> {
+            self.observed_runtime_state
+                .send(ReceivedInstanceState::InstancePut(new_runtime_state))
+                .map_err(|_| {
+                    omicron_common::api::external::Error::internal_error(
+                        "couldn't send SledInstanceState to test driver",
+                    )
+                })
+        }
+    }
+
+    struct FakeNexusParts {
+        nexus_client: NexusClient,
+        nexus_server: HttpServer<ServerContext>,
+        state_rx: Receiver<ReceivedInstanceState>,
+    }
+
+    impl FakeNexusParts {
+        fn new(logctx: &LogContext) -> Self {
+            let (state_tx, state_rx) =
+                tokio::sync::watch::channel(ReceivedInstanceState::None);
+
+            let nexus_server = crate::fakes::nexus::start_test_server(
+                logctx.log.new(o!("component" => "FakeNexusServer")),
+                Box::new(NexusServer { observed_runtime_state: state_tx }),
+            );
+            let nexus_client = NexusClient::new(
+                &format!("http://{}", nexus_server.local_addr()),
+                logctx.log.new(o!("component" => "NexusClient")),
+            );
+
+            Self { nexus_client, nexus_server, state_rx }
+        }
+    }
+
+    fn mock_vnic_contexts(
+    ) -> (MockDladmCreateVnicContext, MockDladmDeleteVnicContext) {
+        let create_vnic_ctx = MockDladm::create_vnic_context();
+        let delete_vnic_ctx = MockDladm::delete_vnic_context();
+        create_vnic_ctx.expect().return_once(
+            |physical_link: &Etherstub, _, _, _, _| {
+                assert_eq!(&physical_link.0, "mystub");
+                Ok(())
+            },
+        );
+        delete_vnic_ctx.expect().returning(|_| Ok(()));
+        (create_vnic_ctx, delete_vnic_ctx)
+    }
+
+    // InstanceManager::ensure_state calls Instance::put_state(Running),
+    //  which calls Instance::propolis_ensure,
+    //   which spawns Instance::monitor_state_task,
+    //    which calls cpapi_instances_put
+    //   and calls Instance::setup_propolis_inner,
+    //    which creates the zone (which isn't real in these tests, of course)
+    fn mock_zone_contexts(
+    ) -> (MockZonesBootContext, MockWaitForServiceContext, MockZonesIdContext)
+    {
+        let boot_ctx = MockZones::boot_context();
+        boot_ctx.expect().return_once(|_| Ok(()));
+        let wait_ctx = illumos_utils::svc::wait_for_service_context();
+        wait_ctx.expect().times(..).returning(|_, _, _| Ok(()));
+        let zone_id_ctx = MockZones::id_context();
+        zone_id_ctx.expect().times(..).returning(|_| Ok(Some(1)));
+        (boot_ctx, wait_ctx, zone_id_ctx)
+    }
+
+    async fn dns_server(
+        logctx: &LogContext,
+        nexus_server: &HttpServer<ServerContext>,
+    ) -> (DnsServerHandle, Arc<Resolver>, Utf8TempDir) {
+        let storage_path =
+            Utf8TempDir::new().expect("Failed to create temporary directory");
+        let config_store = dns_server::storage::Config {
+            keep_old_generations: 3,
+            storage_path: storage_path.path().to_owned(),
+        };
+
+        let (dns_server, dns_dropshot) = dns_server::start_servers(
+            logctx.log.new(o!("component" => "DnsServer")),
+            dns_server::storage::Store::new(
+                logctx.log.new(o!("component" => "DnsStore")),
+                &config_store,
+            )
+            .unwrap(),
+            &dns_server::dns_server::Config {
+                bind_address: "[::1]:0".parse().unwrap(),
+            },
+            &dropshot::ConfigDropshot {
+                bind_address: "[::1]:0".parse().unwrap(),
+                request_body_max_bytes: 8 * 1024,
+                default_handler_task_mode: HandlerTaskMode::Detached,
+            },
+        )
+        .await
+        .expect("starting DNS server");
+
+        let dns_dropshot_client = dns_service_client::Client::new(
+            &format!("http://{}", dns_dropshot.local_addr()),
+            logctx.log.new(o!("component" => "DnsDropshotClient")),
+        );
+        let mut dns_config = internal_dns::DnsConfigBuilder::new();
+        let IpAddr::V6(nexus_ip_addr) = nexus_server.local_addr().ip() else {
+            panic!("IPv6 address required for nexus_server")
+        };
+        let zone = dns_config.host_zone(Uuid::new_v4(), nexus_ip_addr).unwrap();
+        dns_config
+            .service_backend_zone(
+                ServiceName::Nexus,
+                &zone,
+                nexus_server.local_addr().port(),
+            )
+            .unwrap();
+        let dns_config = dns_config.build();
+        dns_dropshot_client.dns_config_put(&dns_config).await.unwrap();
+
+        let resolver = Arc::new(
+            Resolver::new_from_addrs(
+                logctx.log.new(o!("component" => "Resolver")),
+                &[dns_server.local_address()],
+            )
+            .unwrap(),
+        );
+        (dns_server, resolver, storage_path)
+    }
+
+    // note the "mock" here is different from the vnic/zone contexts above.
+    // this is actually running code for a dropshot server from propolis.
+    // (might we want a locally-defined fake whose behavior we can control
+    // more directly from the test driver?)
+    // TODO: factor out, this is also in sled-agent-sim.
+    fn propolis_mock_server(
+        log: &Logger,
+    ) -> (HttpServer<Arc<propolis_mock_server::Context>>, PropolisClient) {
+        let propolis_bind_address =
+            SocketAddr::new(Ipv6Addr::LOCALHOST.into(), 0); // allocate port
+        let dropshot_config = dropshot::ConfigDropshot {
+            bind_address: propolis_bind_address,
+            ..Default::default()
+        };
+        let propolis_log = log.new(o!("component" => "propolis-server-mock"));
+        let private =
+            Arc::new(propolis_mock_server::Context::new(propolis_log));
+        info!(log, "Starting mock propolis-server...");
+        let dropshot_log = log.new(o!("component" => "dropshot"));
+        let mock_api = propolis_mock_server::api();
+
+        let srv = dropshot::HttpServerStarter::new(
+            &dropshot_config,
+            mock_api,
+            private,
+            &dropshot_log,
+        )
+        .expect("couldn't create mock propolis-server")
+        .start();
+
+        let client = propolis_client::Client::new(&format!(
+            "http://{}",
+            srv.local_addr()
+        ));
+
+        (srv, client)
+    }
+
+    // make a FakeStorageManager with a "U2" upserted
+    async fn fake_storage_manager_with_u2() -> StorageHandle {
+        let (storage_manager, storage_handle) = FakeStorageManager::new();
+        tokio::spawn(storage_manager.run());
+        let external_zpool_name = ZpoolName::new_external(Uuid::new_v4());
+        let external_disk: RawDisk =
+            SyntheticDisk::new(external_zpool_name, 0).into();
+        storage_handle.upsert_disk(external_disk).await;
+        storage_handle
+    }
+
+    async fn instance_struct(
+        logctx: &LogContext,
+        propolis_addr: SocketAddr,
+        nexus_client_with_resolver: NexusClientWithResolver,
+        storage_handle: StorageHandle,
+        temp_dir: &String,
+    ) -> Instance {
+        let id = Uuid::new_v4();
+        let propolis_id = Uuid::new_v4();
+
+        let ticket = InstanceTicket::new_without_manager_for_test(id);
+
+        let initial_state =
+            fake_instance_initial_state(propolis_id, propolis_addr);
+
+        let services = fake_instance_manager_services(
+            logctx,
+            storage_handle,
+            nexus_client_with_resolver,
+            temp_dir,
+        );
+
+        let metadata = InstanceMetadata {
+            silo_id: Uuid::new_v4(),
+            project_id: Uuid::new_v4(),
+        };
+
+        Instance::new(
+            logctx.log.new(o!("component" => "Instance")),
+            id,
+            propolis_id,
+            ticket,
+            initial_state,
+            services,
+            metadata,
+        )
+        .unwrap()
+    }
+
+    fn fake_instance_initial_state(
+        propolis_id: Uuid,
+        propolis_addr: SocketAddr,
+    ) -> InstanceInitialState {
+        let hardware = InstanceHardware {
+            properties: InstanceProperties {
+                ncpus: InstanceCpuCount(1),
+                memory: ByteCount::from_gibibytes_u32(1),
+                hostname: Hostname::from_str("bert").unwrap(),
+            },
+            nics: vec![],
+            source_nat: SourceNatConfig {
+                ip: IpAddr::V6(Ipv6Addr::UNSPECIFIED),
+                first_port: 0,
+                last_port: 0,
+            },
+            ephemeral_ip: None,
+            floating_ips: vec![],
+            firewall_rules: vec![],
+            dhcp_config: DhcpConfig {
+                dns_servers: vec![],
+                host_domain: None,
+                search_domains: vec![],
+            },
+            disks: vec![],
+            cloud_init_bytes: None,
+        };
+
+        InstanceInitialState {
+            hardware,
+            instance_runtime: InstanceRuntimeState {
+                propolis_id: Some(propolis_id),
+                dst_propolis_id: None,
+                migration_id: None,
+                gen: Generation::new(),
+                time_updated: Default::default(),
+            },
+            vmm_runtime: VmmRuntimeState {
+                state: InstanceState::Creating,
+                gen: Generation::new(),
+                time_updated: Default::default(),
+            },
+            propolis_addr,
+        }
+    }
+
+    fn fake_instance_manager_services(
+        logctx: &LogContext,
+        storage_handle: StorageHandle,
+        nexus_client_with_resolver: NexusClientWithResolver,
+        temp_dir: &String,
+    ) -> InstanceManagerServices {
+        let vnic_allocator =
+            VnicAllocator::new("Foo", Etherstub("mystub".to_string()));
+        let port_manager = PortManager::new(
+            logctx.log.new(o!("component" => "PortManager")),
+            Ipv6Addr::new(0xfd00, 0x1de, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01),
+        );
+
+        let cleanup_context = CleanupContext::default();
+        let zone_bundler = ZoneBundler::new(
+            logctx.log.new(o!("component" => "ZoneBundler")),
+            storage_handle.clone(),
+            cleanup_context,
+        );
+
+        InstanceManagerServices {
+            nexus_client: nexus_client_with_resolver,
+            vnic_allocator,
+            port_manager,
+            storage: storage_handle,
+            zone_bundler,
+            zone_builder_factory: ZoneBuilderFactory::fake(Some(temp_dir)),
+        }
+    }
+
+    #[tokio::test]
+    async fn test_instance_create_events_normal() {
+        let logctx = omicron_test_utils::dev::test_setup_log(
+            "test_instance_create_events_normal",
+        );
+
+        let (propolis_server, _propolis_client) =
+            propolis_mock_server(&logctx.log);
+        let propolis_addr = propolis_server.local_addr();
+
+        // automock'd things used during this test
+        let _mock_vnic_contexts = mock_vnic_contexts();
+        let _mock_zone_contexts = mock_zone_contexts();
+
+        let FakeNexusParts { nexus_client, nexus_server, mut state_rx } =
+            FakeNexusParts::new(&logctx);
+
+        let (_dns_server, resolver, _dns_config_dir) =
+            timeout(TIMEOUT_DURATION, dns_server(&logctx, &nexus_server))
+                .await
+                .expect("timed out making DNS server and Resolver");
+
+        let nexus_client_with_resolver =
+            NexusClientWithResolver::new_with_client(nexus_client, resolver);
+
+        let storage_handle = fake_storage_manager_with_u2().await;
+
+        let temp_guard = Utf8TempDir::new().unwrap();
+        let temp_dir = temp_guard.path().to_string();
+
+        let inst = timeout(
+            TIMEOUT_DURATION,
+            instance_struct(
+                &logctx,
+                propolis_addr,
+                nexus_client_with_resolver,
+                storage_handle,
+                &temp_dir,
+            ),
+        )
+        .await
+        .expect("timed out creating Instance struct");
+
+        let (put_tx, put_rx) = oneshot::channel();
+
+        // pretending we're InstanceManager::ensure_state, start our "instance"
+        // (backed by fakes and propolis_mock_server)
+        inst.put_state(put_tx, InstanceStateRequested::Running)
+            .await
+            .expect("failed to send Instance::put_state");
+
+        // even though we ignore this result at instance creation time in
+        // practice (to avoid request timeouts), in this test let's make sure
+        // it actually completes.
+        timeout(TIMEOUT_DURATION, put_rx)
+            .await
+            .expect("timed out waiting for Instance::put_state result")
+            .expect("failed to receive Instance::put_state result")
+            .expect("Instance::put_state failed");
+
+        timeout(
+            TIMEOUT_DURATION,
+            state_rx.wait_for(|maybe_state| match maybe_state {
+                ReceivedInstanceState::InstancePut(sled_inst_state) => {
+                    sled_inst_state.vmm_state.state == InstanceState::Running
+                }
+                _ => false,
+            }),
+        )
+        .await
+        .expect("timed out waiting for InstanceState::Running in FakeNexus")
+        .expect("failed to receive FakeNexus' InstanceState");
+
+        logctx.cleanup_successful();
+    }
+
+    // tests around dropshot request timeouts during the blocking propolis setup
+    #[tokio::test]
+    async fn test_instance_create_timeout_while_starting_propolis() {
+        let logctx = omicron_test_utils::dev::test_setup_log(
+            "test_instance_create_timeout_while_starting_propolis",
+        );
+
+        // automock'd things used during this test
+        let _mock_vnic_contexts = mock_vnic_contexts();
+        let _mock_zone_contexts = mock_zone_contexts();
+
+        let FakeNexusParts { nexus_client, nexus_server, state_rx } =
+            FakeNexusParts::new(&logctx);
+
+        let (_dns_server, resolver, _dns_config_dir) =
+            timeout(TIMEOUT_DURATION, dns_server(&logctx, &nexus_server))
+                .await
+                .expect("timed out making DNS server and Resolver");
+
+        let nexus_client_with_resolver =
+            NexusClientWithResolver::new_with_client(nexus_client, resolver);
+
+        let storage_handle = fake_storage_manager_with_u2().await;
+
+        let temp_guard = Utf8TempDir::new().unwrap();
+        let temp_dir = temp_guard.path().to_string();
+
+        let inst = timeout(
+            TIMEOUT_DURATION,
+            instance_struct(
+                &logctx,
+                // we want to test propolis not ever coming up
+                SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::LOCALHOST, 1, 0, 0)),
+                nexus_client_with_resolver,
+                storage_handle,
+                &temp_dir,
+            ),
+        )
+        .await
+        .expect("timed out creating Instance struct");
+
+        let (put_tx, put_rx) = oneshot::channel();
+
+        tokio::time::pause();
+
+        // pretending we're InstanceManager::ensure_state, try in vain to start
+        // our "instance", but no propolis server is running
+        inst.put_state(put_tx, InstanceStateRequested::Running)
+            .await
+            .expect("failed to send Instance::put_state");
+
+        let timeout_fut = timeout(TIMEOUT_DURATION, put_rx);
+
+        tokio::time::advance(TIMEOUT_DURATION).await;
+
+        tokio::time::resume();
+
+        timeout_fut
+            .await
+            .expect_err("*should've* timed out waiting for Instance::put_state, but didn't?");
+
+        if let ReceivedInstanceState::InstancePut(SledInstanceState {
+            vmm_state: VmmRuntimeState { state: InstanceState::Running, .. },
+            ..
+        }) = state_rx.borrow().to_owned()
+        {
+            panic!("Nexus's InstanceState should never have reached running if zone creation timed out");
+        }
+
+        logctx.cleanup_successful();
+    }
+
+    #[tokio::test]
+    async fn test_instance_create_timeout_while_creating_zone() {
+        let logctx = omicron_test_utils::dev::test_setup_log(
+            "test_instance_create_timeout_while_creating_zone",
+        );
+
+        // automock'd things used during this test
+        let _mock_vnic_contexts = mock_vnic_contexts();
+
+        let rt_handle = tokio::runtime::Handle::current();
+
+        // time out while booting zone, on purpose!
+        let boot_ctx = MockZones::boot_context();
+        boot_ctx.expect().return_once(move |_| {
+            rt_handle.block_on(tokio::time::sleep(TIMEOUT_DURATION * 2));
+            Ok(())
+        });
+        let wait_ctx = illumos_utils::svc::wait_for_service_context();
+        wait_ctx.expect().times(..).returning(|_, _, _| Ok(()));
+        let zone_id_ctx = MockZones::id_context();
+        zone_id_ctx.expect().times(..).returning(|_| Ok(Some(1)));
+
+        let FakeNexusParts { nexus_client, nexus_server, state_rx } =
+            FakeNexusParts::new(&logctx);
+
+        let (_dns_server, resolver, _dns_config_dir) =
+            timeout(TIMEOUT_DURATION, dns_server(&logctx, &nexus_server))
+                .await
+                .expect("timed out making DNS server and Resolver");
+
+        let nexus_client_with_resolver =
+            NexusClientWithResolver::new_with_client(nexus_client, resolver);
+
+        let storage_handle = fake_storage_manager_with_u2().await;
+
+        let temp_guard = Utf8TempDir::new().unwrap();
+        let temp_dir = temp_guard.path().to_string();
+
+        let inst = timeout(
+            TIMEOUT_DURATION,
+            instance_struct(
+                &logctx,
+                // isn't running because the "zone" never "boots"
+                SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::LOCALHOST, 1, 0, 0)),
+                nexus_client_with_resolver,
+                storage_handle,
+                &temp_dir,
+            ),
+        )
+        .await
+        .expect("timed out creating Instance struct");
+
+        tokio::time::pause();
+
+        let (put_tx, put_rx) = oneshot::channel();
+
+        // pretending we're InstanceManager::ensure_state, try in vain to start
+        // our "instance", but the zone never finishes installing
+        inst.put_state(put_tx, InstanceStateRequested::Running)
+            .await
+            .expect("failed to send Instance::put_state");
+
+        let timeout_fut = timeout(TIMEOUT_DURATION, put_rx);
+
+        tokio::time::advance(TIMEOUT_DURATION * 2).await;
+
+        tokio::time::resume();
+
+        timeout_fut
+            .await
+            .expect_err("*should've* timed out waiting for Instance::put_state, but didn't?");
+
+        if let ReceivedInstanceState::InstancePut(SledInstanceState {
+            vmm_state: VmmRuntimeState { state: InstanceState::Running, .. },
+            ..
+        }) = state_rx.borrow().to_owned()
+        {
+            panic!("Nexus's InstanceState should never have reached running if zone creation timed out");
+        }
+
+        logctx.cleanup_successful();
+    }
+
+    #[tokio::test]
+    async fn test_instance_manager_creation() {
+        let logctx = omicron_test_utils::dev::test_setup_log(
+            "test_instance_manager_creation",
+        );
+
+        // automock'd things used during this test
+        let _mock_vnic_contexts = mock_vnic_contexts();
+        let _mock_zone_contexts = mock_zone_contexts();
+
+        let storage_handle = fake_storage_manager_with_u2().await;
+
+        let FakeNexusParts { nexus_client, nexus_server, mut state_rx } =
+            FakeNexusParts::new(&logctx);
+
+        let (_dns_server, resolver, _dns_config_dir) =
+            timeout(TIMEOUT_DURATION, dns_server(&logctx, &nexus_server))
+                .await
+                .expect("timed out making DNS server and Resolver");
+
+        let nexus_client_with_resolver =
+            NexusClientWithResolver::new_with_client(nexus_client, resolver);
+
+        let temp_guard = Utf8TempDir::new().unwrap();
+        let temp_dir = temp_guard.path().to_string();
+
+        let InstanceManagerServices {
+            nexus_client,
+            vnic_allocator: _,
+            port_manager,
+            storage,
+            zone_bundler,
+            zone_builder_factory,
+        } = fake_instance_manager_services(
+            &logctx,
+            storage_handle,
+            nexus_client_with_resolver,
+            &temp_dir,
+        );
+
+        let etherstub = Etherstub("mystub".to_string());
+
+        let vmm_reservoir_manager = VmmReservoirManagerHandle::stub_for_test();
+
+        let mgr = crate::instance_manager::InstanceManager::new(
+            logctx.log.new(o!("component" => "InstanceManager")),
+            nexus_client,
+            etherstub,
+            port_manager,
+            storage,
+            zone_bundler,
+            zone_builder_factory,
+            vmm_reservoir_manager,
+        )
+        .unwrap();
+
+        let (propolis_server, _propolis_client) =
+            propolis_mock_server(&logctx.log);
+        let propolis_addr = propolis_server.local_addr();
+
+        let instance_id = Uuid::new_v4();
+        let propolis_id = Uuid::new_v4();
+        let InstanceInitialState {
+            hardware,
+            instance_runtime,
+            vmm_runtime,
+            propolis_addr,
+        } = fake_instance_initial_state(propolis_id, propolis_addr);
+
+        let metadata = InstanceMetadata {
+            silo_id: Uuid::new_v4(),
+            project_id: Uuid::new_v4(),
+        };
+
+        mgr.ensure_registered(
+            instance_id,
+            propolis_id,
+            hardware,
+            instance_runtime,
+            vmm_runtime,
+            propolis_addr,
+            metadata,
+        )
+        .await
+        .unwrap();
+
+        mgr.ensure_state(instance_id, InstanceStateRequested::Running)
+            .await
+            .unwrap();
+
+        timeout(
+            TIMEOUT_DURATION,
+            state_rx.wait_for(|maybe_state| match maybe_state {
+                ReceivedInstanceState::InstancePut(sled_inst_state) => {
+                    sled_inst_state.vmm_state.state == InstanceState::Running
+                }
+                _ => false,
+            }),
+        )
+        .await
+        .expect("timed out waiting for InstanceState::Running in FakeNexus")
+        .expect("failed to receive FakeNexus' InstanceState");
+
+        logctx.cleanup_successful();
+    }
+}
diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs
index badaf2d7ba..2c9780b3ce 100644
--- a/sled-agent/src/instance_manager.rs
+++ b/sled-agent/src/instance_manager.rs
@@ -194,7 +194,23 @@ impl InstanceManager {
             })
             .await
             .map_err(|_| Error::FailedSendInstanceManagerClosed)?;
-        rx.await?
+
+        match target {
+            // these may involve a long-running zone creation, so avoid HTTP
+            // request timeouts by decoupling the response
+            // (see InstanceRunner::put_state)
+            InstanceStateRequested::MigrationTarget(_)
+            | InstanceStateRequested::Running => {
+                // We don't want the sending side of the channel to see an
+                // error if we drop rx without awaiting it.
+                // Since we don't care about the response here, we spawn rx
+                // into a task which will await it for us in the background.
+                tokio::spawn(rx);
+                Ok(InstancePutStateResponse { updated_runtime: None })
+            }
+            InstanceStateRequested::Stopped
+            | InstanceStateRequested::Reboot => rx.await?,
+        }
     }
 
     pub async fn put_migration_ids(
@@ -734,6 +750,11 @@ impl InstanceTicket {
         InstanceTicket { id, terminate_tx: Some(terminate_tx) }
     }
 
+    #[cfg(test)]
+    pub(crate) fn new_without_manager_for_test(id: Uuid) -> Self {
+        Self { id, terminate_tx: None }
+    }
+
     /// Idempotently removes this instance from the tracked set of
     /// instances. This acts as an "upcall" for instances to remove
     /// themselves after stopping.
diff --git a/sled-agent/src/nexus.rs b/sled-agent/src/nexus.rs
index 12fcc05ce3..3f24c6a806 100644
--- a/sled-agent/src/nexus.rs
+++ b/sled-agent/src/nexus.rs
@@ -60,6 +60,16 @@ impl NexusClientWithResolver {
         }
     }
 
+    // for when we have a NexusClient constructed from a FakeNexusServer
+    // (no need to expose this function outside of tests)
+    #[cfg(test)]
+    pub(crate) fn new_with_client(
+        client: NexusClient,
+        resolver: Arc<Resolver>,
+    ) -> Self {
+        Self { client, resolver }
+    }
+
     /// Access the progenitor-based Nexus Client.
     pub fn client(&self) -> &NexusClient {
         &self.client
diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs
index 71bd34c091..c9e0211690 100644
--- a/sled-agent/src/params.rs
+++ b/sled-agent/src/params.rs
@@ -135,7 +135,7 @@ pub struct InstancePutStateBody {
 
 /// The response sent from a request to move an instance into a specific runtime
 /// state.
-#[derive(Serialize, Deserialize, JsonSchema)]
+#[derive(Debug, Serialize, Deserialize, JsonSchema)]
 pub struct InstancePutStateResponse {
     /// The current runtime state of the instance after handling the request to
     /// change its state. If the instance's state did not change, this field is
diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs
index 45547cea64..1edde622a1 100644
--- a/sled-agent/src/sim/sled_agent.rs
+++ b/sled-agent/src/sim/sled_agent.rs
@@ -44,6 +44,7 @@ use std::collections::{HashMap, HashSet};
 use std::net::{IpAddr, Ipv6Addr, SocketAddr};
 use std::str::FromStr;
 use std::sync::Arc;
+use std::time::Duration;
 use uuid::Uuid;
 
 /// Simulates management of the control plane on a sled
@@ -73,6 +74,7 @@ pub struct SledAgent {
     config: Config,
     fake_zones: Mutex<OmicronZonesConfig>,
     instance_ensure_state_error: Mutex<Option<Error>>,
+    pub log: Logger,
 }
 
 fn extract_targets_from_volume_construction_request(
@@ -171,6 +173,7 @@ impl SledAgent {
                 zones: vec![],
             }),
             instance_ensure_state_error: Mutex::new(None),
+            log,
         })
     }
 
@@ -399,7 +402,28 @@ impl SledAgent {
                     ));
                 }
                 InstanceStateRequested::Running => {
-                    propolis_client::types::InstanceStateRequested::Run
+                    let instances = self.instances.clone();
+                    let log = self.log.new(
+                        o!("component" => "SledAgent-insure_instance_state"),
+                    );
+                    tokio::spawn(async move {
+                        tokio::time::sleep(Duration::from_secs(10)).await;
+                        match instances
+                            .sim_ensure(&instance_id, current, Some(state))
+                            .await
+                        {
+                            Ok(state) => {
+                                let instance_state: nexus_client::types::SledInstanceState = state.into();
+                                info!(log, "sim_ensure success"; "instance_state" => #?instance_state);
+                            }
+                            Err(instance_put_error) => {
+                                error!(log, "sim_ensure failure"; "error" => #?instance_put_error);
+                            }
+                        }
+                    });
+                    return Ok(InstancePutStateResponse {
+                        updated_runtime: None,
+                    });
                 }
                 InstanceStateRequested::Stopped => {
                     propolis_client::types::InstanceStateRequested::Stop
diff --git a/sled-agent/src/vmm_reservoir.rs b/sled-agent/src/vmm_reservoir.rs
index d7b6b64ecf..b16286f5f5 100644
--- a/sled-agent/src/vmm_reservoir.rs
+++ b/sled-agent/src/vmm_reservoir.rs
@@ -119,6 +119,19 @@ impl VmmReservoirManagerHandle {
         }
         rx.await.map_err(|_| Error::ReplySenderDropped)?
     }
+
+    #[cfg(test)]
+    pub fn stub_for_test() -> Self {
+        let (tx, _) = flume::bounded(1);
+        let (size_updated_tx, _) = broadcast::channel(1);
+        let _manager_handle = Arc::new(thread::spawn(|| {}));
+        Self {
+            reservoir_size: Arc::new(AtomicU64::new(0)),
+            tx,
+            size_updated_tx,
+            _manager_handle,
+        }
+    }
 }
 
 /// Manage the VMM reservoir in a background thread

From a3fa5404eeb6fa223f8925dbe4a1d0fb468d9484 Mon Sep 17 00:00:00 2001
From: Laura Abbott <labbott@users.noreply.github.com>
Date: Tue, 19 Mar 2024 09:09:56 -0400
Subject: [PATCH 130/157] RoT v1.0.7 (#5282)

---
 .github/buildomat/jobs/tuf-repo.sh | 4 ++--
 tools/dvt_dock_version             | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh
index aca43422d9..9c70dce2ac 100755
--- a/.github/buildomat/jobs/tuf-repo.sh
+++ b/.github/buildomat/jobs/tuf-repo.sh
@@ -278,8 +278,8 @@ EOF
     done
 }
 # usage:              SERIES           ROT_DIR      ROT_VERSION              BOARDS...
-add_hubris_artifacts  rot-staging-dev  staging/dev  cert-staging-dev-v1.0.6  "${ALL_BOARDS[@]}"
-add_hubris_artifacts  rot-prod-rel     prod/rel     cert-prod-rel-v1.0.6     "${ALL_BOARDS[@]}"
+add_hubris_artifacts  rot-staging-dev  staging/dev  cert-staging-dev-v1.0.7  "${ALL_BOARDS[@]}"
+add_hubris_artifacts  rot-prod-rel     prod/rel     cert-prod-rel-v1.0.7     "${ALL_BOARDS[@]}"
 
 for series in "${SERIES_LIST[@]}"; do
     /work/tufaceous assemble --no-generate-key /work/manifest-"$series".toml /work/repo-"$series".zip
diff --git a/tools/dvt_dock_version b/tools/dvt_dock_version
index fd988dc876..f6bd67fc2e 100644
--- a/tools/dvt_dock_version
+++ b/tools/dvt_dock_version
@@ -1 +1 @@
-COMMIT=9e8e16f508bb41f02455aeddcece0a7edae2f672
+COMMIT=8319bada89323373be2059404c25fe1162b3c714

From 353bfce4ca1be86b0a423f6eb47fd0b9ed9e6a0e Mon Sep 17 00:00:00 2001
From: Adam Leventhal <ahl@oxide.computer>
Date: Tue, 19 Mar 2024 15:48:32 -0700
Subject: [PATCH 131/157] update version number of API to 20240327.0 (#5288)

---
 nexus/src/lib.rs                          | 2 +-
 nexus/tests/integration_tests/commands.rs | 2 +-
 openapi/nexus.json                        | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs
index d143858bf4..cad90aab01 100644
--- a/nexus/src/lib.rs
+++ b/nexus/src/lib.rs
@@ -50,7 +50,7 @@ extern crate slog;
 /// to stdout.
 pub fn run_openapi_external() -> Result<(), String> {
     external_api()
-        .openapi("Oxide Region API", "0.0.6")
+        .openapi("Oxide Region API", "20240327.0")
         .description("API for interacting with the Oxide control plane")
         .contact_url("https://oxide.computer")
         .contact_email("api@oxide.computer")
diff --git a/nexus/tests/integration_tests/commands.rs b/nexus/tests/integration_tests/commands.rs
index bc79a7d5a2..fd7a6c60c0 100644
--- a/nexus/tests/integration_tests/commands.rs
+++ b/nexus/tests/integration_tests/commands.rs
@@ -109,7 +109,7 @@ fn test_nexus_openapi() {
         .expect("stdout was not valid OpenAPI");
     assert_eq!(spec.openapi, "3.0.3");
     assert_eq!(spec.info.title, "Oxide Region API");
-    assert_eq!(spec.info.version, "0.0.6");
+    assert_eq!(spec.info.version, "20240327.0");
 
     // Spot check a couple of items.
     assert!(!spec.paths.paths.is_empty());
diff --git a/openapi/nexus.json b/openapi/nexus.json
index 7d8e1909d2..54a2d3250f 100644
--- a/openapi/nexus.json
+++ b/openapi/nexus.json
@@ -7,7 +7,7 @@
       "url": "https://oxide.computer",
       "email": "api@oxide.computer"
     },
-    "version": "0.0.6"
+    "version": "20240327.0"
   },
   "paths": {
     "/device/auth": {

From c05c081cd6562ae799aa2f369dce89b23886af66 Mon Sep 17 00:00:00 2001
From: Kyle Simpson <kyle@oxide.computer>
Date: Tue, 19 Mar 2024 22:53:31 +0000
Subject: [PATCH 132/157] Chore: Bump OPTE to 7ee353a. (#5290)

Includes oxidecomputer/opte#477.
---
 Cargo.lock         | 14 +++++++-------
 Cargo.toml         |  4 ++--
 tools/opte_version |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 0bb79eece8..1540acfbd9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1749,7 +1749,7 @@ dependencies = [
 [[package]]
 name = "derror-macro"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
+source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce"
 dependencies = [
  "darling 0.20.3",
  "proc-macro2",
@@ -3436,7 +3436,7 @@ dependencies = [
 [[package]]
 name = "illumos-sys-hdrs"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
+source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce"
 
 [[package]]
 name = "illumos-utils"
@@ -3840,7 +3840,7 @@ dependencies = [
 [[package]]
 name = "kstat-macro"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
+source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce"
 dependencies = [
  "quote",
  "syn 2.0.52",
@@ -5832,7 +5832,7 @@ dependencies = [
 [[package]]
 name = "opte"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
+source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce"
 dependencies = [
  "cfg-if",
  "derror-macro",
@@ -5850,7 +5850,7 @@ dependencies = [
 [[package]]
 name = "opte-api"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
+source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce"
 dependencies = [
  "illumos-sys-hdrs",
  "ipnetwork",
@@ -5862,7 +5862,7 @@ dependencies = [
 [[package]]
 name = "opte-ioctl"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
+source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce"
 dependencies = [
  "libc",
  "libnet",
@@ -5936,7 +5936,7 @@ dependencies = [
 [[package]]
 name = "oxide-vpc"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/opte?rev=a4c956e44fc9b75b58b83ad2eec22f6bd9005262#a4c956e44fc9b75b58b83ad2eec22f6bd9005262"
+source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce"
 dependencies = [
  "cfg-if",
  "illumos-sys-hdrs",
diff --git a/Cargo.toml b/Cargo.toml
index 802f965965..70b40df81c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -298,7 +298,7 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.11.0"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "a4c956e44fc9b75b58b83ad2eec22f6bd9005262", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "7ee353a470ea59529ee1b34729681da887aa88ce", features = [ "api", "std" ] }
 once_cell = "1.19.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "2.0.0"
@@ -306,7 +306,7 @@ openapiv3 = "2.0.0"
 openssl = "0.10"
 openssl-sys = "0.9"
 openssl-probe = "0.1.5"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "a4c956e44fc9b75b58b83ad2eec22f6bd9005262" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "7ee353a470ea59529ee1b34729681da887aa88ce" }
 oso = "0.27"
 owo-colors = "4.0.0"
 oximeter = { path = "oximeter/oximeter" }
diff --git a/tools/opte_version b/tools/opte_version
index 6a9619566a..e1b3e11499 100644
--- a/tools/opte_version
+++ b/tools/opte_version
@@ -1 +1 @@
-0.28.232
+0.28.233

From 389206fc2ac2a7a08b77c3d0251b628109b81859 Mon Sep 17 00:00:00 2001
From: David Crespo <david-crespo@users.noreply.github.com>
Date: Wed, 20 Mar 2024 08:39:21 -0500
Subject: [PATCH 133/157] IP range add can only be IPv4 (#5107)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Closes #5085

- ~~Change range add request body from `IpRange` to `Ipv4Range`, see
what breaks~~
- [x] Error on Ipv6 ranges in the service layer
- [x] Update tests and add new tests
- ~~May change range remove body to `Ipv4Range` for consistency even
though it is technically fine as long as add is restricted to `Ipv4` —
need to think about it~~ decided not to so we can still remove any
existing IPv6 ranges through the API
---
 nexus/src/app/ip_pool.rs                   |  26 ++++
 nexus/src/external_api/http_entrypoints.rs |   4 +
 nexus/tests/integration_tests/ip_pools.rs  | 151 +++++++++++----------
 openapi/nexus.json                         |   2 +
 4 files changed, 108 insertions(+), 75 deletions(-)

diff --git a/nexus/src/app/ip_pool.rs b/nexus/src/app/ip_pool.rs
index 87a7d98c91..fd73a18355 100644
--- a/nexus/src/app/ip_pool.rs
+++ b/nexus/src/app/ip_pool.rs
@@ -27,6 +27,7 @@ use omicron_common::api::external::NameOrId;
 use omicron_common::api::external::ResourceType;
 use omicron_common::api::external::UpdateResult;
 use ref_cast::RefCast;
+use std::matches;
 use uuid::Uuid;
 
 /// Helper to make it easier to 404 on attempts to manipulate internal pools
@@ -291,6 +292,19 @@ impl super::Nexus {
             return Err(not_found_from_lookup(pool_lookup));
         }
 
+        // Disallow V6 ranges until IPv6 is fully supported by the networking
+        // subsystem. Instead of changing the API to reflect that (making this
+        // endpoint inconsistent with the rest) and changing it back when we
+        // add support, we accept them at the API layer and error here. It
+        // would be nice if we could do it in the datastore layer, but we'd
+        // have no way of creating IPv6 ranges for the purpose of testing IP
+        // pool utilization.
+        if matches!(range, IpRange::V6(_)) {
+            return Err(Error::invalid_request(
+                "IPv6 ranges are not allowed yet",
+            ));
+        }
+
         self.db_datastore.ip_pool_add_range(opctx, &authz_pool, range).await
     }
 
@@ -347,6 +361,18 @@ impl super::Nexus {
         let (authz_pool, ..) =
             self.db_datastore.ip_pools_service_lookup(opctx).await?;
         opctx.authorize(authz::Action::Modify, &authz_pool).await?;
+        // Disallow V6 ranges until IPv6 is fully supported by the networking
+        // subsystem. Instead of changing the API to reflect that (making this
+        // endpoint inconsistent with the rest) and changing it back when we
+        // add support, we accept them at the API layer and error here. It
+        // would be nice if we could do it in the datastore layer, but we'd
+        // have no way of creating IPv6 ranges for the purpose of testing IP
+        // pool utilization.
+        if matches!(range, IpRange::V6(_)) {
+            return Err(Error::invalid_request(
+                "IPv6 ranges are not allowed yet",
+            ));
+        }
         self.db_datastore.ip_pool_add_range(opctx, &authz_pool, range).await
     }
 
diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs
index 94cd35519b..41e5bce08a 100644
--- a/nexus/src/external_api/http_entrypoints.rs
+++ b/nexus/src/external_api/http_entrypoints.rs
@@ -1789,6 +1789,8 @@ async fn ip_pool_range_list(
 }
 
 /// Add range to IP pool
+///
+/// IPv6 ranges are not allowed yet.
 #[endpoint {
     method = POST,
     path = "/v1/system/ip-pools/{pool}/ranges/add",
@@ -1880,6 +1882,8 @@ async fn ip_pool_service_range_list(
 }
 
 /// Add IP range to Oxide service pool
+///
+/// IPv6 ranges are not allowed yet.
 #[endpoint {
     method = POST,
     path = "/v1/system/ip-pools-service/ranges/add",
diff --git a/nexus/tests/integration_tests/ip_pools.rs b/nexus/tests/integration_tests/ip_pools.rs
index c8390e8ce0..cb5eade735 100644
--- a/nexus/tests/integration_tests/ip_pools.rs
+++ b/nexus/tests/integration_tests/ip_pools.rs
@@ -9,6 +9,8 @@ use dropshot::HttpErrorResponseBody;
 use dropshot::ResultsPage;
 use http::method::Method;
 use http::StatusCode;
+use nexus_db_queries::authz;
+use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db::datastore::SERVICE_IP_POOL_NAME;
 use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO;
 use nexus_db_queries::db::fixed_data::silo::INTERNAL_SILO_ID;
@@ -38,7 +40,6 @@ use nexus_types::external_api::params::IpPoolSiloUpdate;
 use nexus_types::external_api::params::IpPoolUpdate;
 use nexus_types::external_api::shared::IpRange;
 use nexus_types::external_api::shared::Ipv4Range;
-use nexus_types::external_api::shared::Ipv6Range;
 use nexus_types::external_api::shared::SiloIdentityMode;
 use nexus_types::external_api::views::IpPool;
 use nexus_types::external_api::views::IpPoolRange;
@@ -46,7 +47,9 @@ use nexus_types::external_api::views::IpPoolSiloLink;
 use nexus_types::external_api::views::Silo;
 use nexus_types::external_api::views::SiloIpPool;
 use nexus_types::identity::Resource;
+use omicron_common::address::Ipv6Range;
 use omicron_common::api::external::IdentityMetadataUpdateParams;
+use omicron_common::api::external::LookupType;
 use omicron_common::api::external::NameOrId;
 use omicron_common::api::external::SimpleIdentity;
 use omicron_common::api::external::{IdentityMetadataCreateParams, Name};
@@ -765,7 +768,7 @@ async fn create_pool(client: &ClientTestContext, name: &str) -> IpPool {
 async fn test_ip_pool_utilization_total(cptestctx: &ControlPlaneTestContext) {
     let client = &cptestctx.external_client;
 
-    create_pool(client, "p0").await;
+    let pool = create_pool(client, "p0").await;
 
     assert_ip_pool_utilization(client, "p0", 0, 0, 0, 0).await;
 
@@ -783,20 +786,36 @@ async fn test_ip_pool_utilization_total(cptestctx: &ControlPlaneTestContext) {
 
     assert_ip_pool_utilization(client, "p0", 0, 5, 0, 0).await;
 
-    // now let's add a gigantic range just for fun
+    // Now let's add a gigantic range. This requires direct datastore
+    // shenanigans because adding IPv6 ranges through the API is currently not
+    // allowed. It's worth doing because we want this code to correctly handle
+    // IPv6 ranges when they are allowed again.
+
+    let nexus = &cptestctx.server.apictx().nexus;
+    let datastore = nexus.datastore();
+    let log = cptestctx.logctx.log.new(o!());
+    let opctx = OpContext::for_tests(log, datastore.clone());
+    let authz_pool = authz::IpPool::new(
+        authz::FLEET,
+        pool.identity.id,
+        LookupType::ByName("p0".to_string()),
+    );
+
     let big_range = IpRange::V6(
         Ipv6Range::new(
-            std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1),
+            std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0),
             std::net::Ipv6Addr::new(
-                0xfd00, 0, 0, 0, 0xffff, 0xfff, 0xffff, 0xffff,
+                0xfd00, 0, 0, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff,
             ),
         )
         .unwrap(),
     );
-    object_create::<IpRange, IpPoolRange>(client, &add_url, &big_range).await;
+    datastore
+        .ip_pool_add_range(&opctx, &authz_pool, &big_range)
+        .await
+        .expect("could not add range");
 
-    assert_ip_pool_utilization(client, "p0", 0, 5, 0, 18446480190918885375)
-        .await;
+    assert_ip_pool_utilization(client, "p0", 0, 5, 0, 2u128.pow(80)).await;
 }
 
 // Data for testing overlapping IP ranges
@@ -895,59 +914,9 @@ async fn test_ip_pool_range_overlapping_ranges_fails(
     };
     test_bad_ip_ranges(client, &ip_pool_add_range_url, &ipv4_range).await;
 
-    // Test data for IPv6 ranges that should fail due to overlap
-    let ipv6_range = TestRange {
-        base_range: IpRange::V6(
-            Ipv6Range::new(
-                std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 10),
-                std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 20),
-            )
-            .unwrap(),
-        ),
-        bad_ranges: vec![
-            // The exact same range
-            IpRange::V6(
-                Ipv6Range::new(
-                    std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 10),
-                    std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 20),
-                )
-                .unwrap(),
-            ),
-            // Overlaps below
-            IpRange::V6(
-                Ipv6Range::new(
-                    std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 5),
-                    std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 15),
-                )
-                .unwrap(),
-            ),
-            // Overlaps above
-            IpRange::V6(
-                Ipv6Range::new(
-                    std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 15),
-                    std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 25),
-                )
-                .unwrap(),
-            ),
-            // Contains the base range
-            IpRange::V6(
-                Ipv6Range::new(
-                    std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0),
-                    std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 100),
-                )
-                .unwrap(),
-            ),
-            // Contained by the base range
-            IpRange::V6(
-                Ipv6Range::new(
-                    std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 12),
-                    std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 13),
-                )
-                .unwrap(),
-            ),
-        ],
-    };
-    test_bad_ip_ranges(client, &ip_pool_add_range_url, &ipv6_range).await;
+    // IPv6 tests removed along with support for IPv6 ranges in
+    // https://github.com/oxidecomputer/omicron/pull/5107
+    // Put them back when IPv6 ranges are supported again.
 }
 
 async fn test_bad_ip_ranges(
@@ -994,6 +963,38 @@ async fn test_bad_ip_ranges(
     }
 }
 
+// Support for IPv6 ranges removed in
+// https://github.com/oxidecomputer/omicron/pull/5107
+// Delete this test when we support IPv6 again.
+#[nexus_test]
+async fn test_ip_pool_range_rejects_v6(cptestctx: &ControlPlaneTestContext) {
+    let client = &cptestctx.external_client;
+
+    create_ip_pool(client, "p0", None).await;
+
+    let range = IpRange::V6(
+        Ipv6Range::new(
+            std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 10),
+            std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 20),
+        )
+        .unwrap(),
+    );
+
+    let add_url = "/v1/system/ip-pools/p0/ranges/add";
+    let error =
+        object_create_error(client, add_url, &range, StatusCode::BAD_REQUEST)
+            .await;
+
+    assert_eq!(error.message, "IPv6 ranges are not allowed yet");
+
+    // same deal with service pool
+    let add_url = "/v1/system/ip-pools-service/ranges/add";
+    let error =
+        object_create_error(client, add_url, &range, StatusCode::BAD_REQUEST)
+            .await;
+    assert_eq!(error.message, "IPv6 ranges are not allowed yet");
+}
+
 #[nexus_test]
 async fn test_ip_pool_range_pagination(cptestctx: &ControlPlaneTestContext) {
     let client = &cptestctx.external_client;
@@ -1026,17 +1027,17 @@ async fn test_ip_pool_range_pagination(cptestctx: &ControlPlaneTestContext) {
     // address, which sorts all IPv4 before IPv6, then within protocol versions
     // by their first address.
     let ranges = [
-        IpRange::V6(
-            Ipv6Range::new(
-                std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 11),
-                std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 20),
+        IpRange::V4(
+            Ipv4Range::new(
+                std::net::Ipv4Addr::new(10, 0, 0, 3),
+                std::net::Ipv4Addr::new(10, 0, 0, 4),
             )
             .unwrap(),
         ),
-        IpRange::V6(
-            Ipv6Range::new(
-                std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0),
-                std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 10),
+        IpRange::V4(
+            Ipv4Range::new(
+                std::net::Ipv4Addr::new(10, 0, 0, 5),
+                std::net::Ipv4Addr::new(10, 0, 0, 6),
             )
             .unwrap(),
         ),
@@ -1304,15 +1305,15 @@ async fn test_ip_pool_service(cptestctx: &ControlPlaneTestContext) {
     let ranges = [
         IpRange::V4(
             Ipv4Range::new(
-                std::net::Ipv4Addr::new(10, 0, 0, 1),
-                std::net::Ipv4Addr::new(10, 0, 0, 2),
+                std::net::Ipv4Addr::new(10, 0, 0, 3),
+                std::net::Ipv4Addr::new(10, 0, 0, 4),
             )
             .unwrap(),
         ),
-        IpRange::V6(
-            Ipv6Range::new(
-                std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 0),
-                std::net::Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 10),
+        IpRange::V4(
+            Ipv4Range::new(
+                std::net::Ipv4Addr::new(10, 0, 0, 1),
+                std::net::Ipv4Addr::new(10, 0, 0, 2),
             )
             .unwrap(),
         ),
diff --git a/openapi/nexus.json b/openapi/nexus.json
index 54a2d3250f..5ea1a48d8b 100644
--- a/openapi/nexus.json
+++ b/openapi/nexus.json
@@ -5432,6 +5432,7 @@
           "system/networking"
         ],
         "summary": "Add range to IP pool",
+        "description": "IPv6 ranges are not allowed yet.",
         "operationId": "ip_pool_range_add",
         "parameters": [
           {
@@ -5847,6 +5848,7 @@
           "system/networking"
         ],
         "summary": "Add IP range to Oxide service pool",
+        "description": "IPv6 ranges are not allowed yet.",
         "operationId": "ip_pool_service_range_add",
         "requestBody": {
           "content": {

From 58adb453eb401daa461abfcfb2a57e8dc497438d Mon Sep 17 00:00:00 2001
From: David Pacheco <dap@oxidecomputer.com>
Date: Wed, 20 Mar 2024 09:58:12 -0700
Subject: [PATCH 134/157] make it a little harder to mess up new db schema
 migrations (#5274)

---
 Cargo.lock                                    |   4 +
 dev-tools/omdb/src/bin/omdb/db.rs             |   2 +-
 nexus/db-model/Cargo.toml                     |   4 +
 nexus/db-model/src/lib.rs                     |   2 +
 nexus/db-model/src/schema.rs                  |   9 -
 nexus/db-model/src/schema_versions.rs         | 800 ++++++++++++++++++
 .../src/db/datastore/db_metadata.rs           | 542 +++---------
 nexus/db-queries/src/db/datastore/mod.rs      |  11 +-
 nexus/src/app/mod.rs                          |  16 +-
 nexus/src/bin/schema-updater.rs               |  45 +-
 .../tests/integration_tests/initialization.rs |   4 +-
 nexus/tests/integration_tests/schema.rs       | 128 +--
 schema/crdb/README.adoc                       |  98 ++-
 schema/crdb/dbinit.sql                        |   2 +-
 schema/crdb/first-named-migration/up.sql      |   2 +
 15 files changed, 1111 insertions(+), 558 deletions(-)
 create mode 100644 nexus/db-model/src/schema_versions.rs
 create mode 100644 schema/crdb/first-named-migration/up.sql

diff --git a/Cargo.lock b/Cargo.lock
index 1540acfbd9..9ca0599fb3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4425,6 +4425,8 @@ name = "nexus-db-model"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "camino",
+ "camino-tempfile",
  "chrono",
  "db-macros",
  "derive-where",
@@ -4443,6 +4445,7 @@ dependencies = [
  "omicron-rpaths",
  "omicron-uuid-kinds",
  "omicron-workspace-hack",
+ "once_cell",
  "parse-display",
  "pq-sys",
  "rand 0.8.5",
@@ -4455,6 +4458,7 @@ dependencies = [
  "steno",
  "strum 0.26.1",
  "thiserror",
+ "tokio",
  "uuid 1.7.0",
 ]
 
diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs
index cc498d0ad1..9e0632d722 100644
--- a/dev-tools/omdb/src/bin/omdb/db.rs
+++ b/dev-tools/omdb/src/bin/omdb/db.rs
@@ -583,7 +583,7 @@ impl DbArgs {
 /// incompatible because in practice it may well not matter and it's very
 /// valuable for this tool to work if it possibly can.
 async fn check_schema_version(datastore: &DataStore) {
-    let expected_version = nexus_db_model::schema::SCHEMA_VERSION;
+    let expected_version = nexus_db_model::SCHEMA_VERSION;
     let version_check = datastore.database_schema_version().await;
 
     match version_check {
diff --git a/nexus/db-model/Cargo.toml b/nexus/db-model/Cargo.toml
index b1bb68b8f0..45a086a5b3 100644
--- a/nexus/db-model/Cargo.toml
+++ b/nexus/db-model/Cargo.toml
@@ -9,6 +9,7 @@ omicron-rpaths.workspace = true
 
 [dependencies]
 anyhow.workspace = true
+camino.workspace = true
 chrono.workspace = true
 derive-where.workspace = true
 diesel = { workspace = true, features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] }
@@ -17,6 +18,7 @@ ipnetwork.workspace = true
 macaddr.workspace = true
 newtype_derive.workspace = true
 omicron-uuid-kinds.workspace = true
+once_cell.workspace = true
 parse-display.workspace = true
 # See omicron-rpaths for more about the "pq-sys" dependency.
 pq-sys = "*"
@@ -29,6 +31,7 @@ serde_json.workspace = true
 steno.workspace = true
 strum.workspace = true
 thiserror.workspace = true
+tokio.workspace = true
 uuid.workspace = true
 
 db-macros.workspace = true
@@ -42,4 +45,5 @@ sled-agent-client.workspace = true
 omicron-workspace-hack.workspace = true
 
 [dev-dependencies]
+camino-tempfile.workspace = true
 expectorate.workspace = true
diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs
index d2b676a3da..7124103b30 100644
--- a/nexus/db-model/src/lib.rs
+++ b/nexus/db-model/src/lib.rs
@@ -67,6 +67,7 @@ mod role_assignment;
 mod role_builtin;
 pub mod saga_types;
 pub mod schema;
+mod schema_versions;
 mod service;
 mod service_kind;
 mod silo;
@@ -158,6 +159,7 @@ pub use region::*;
 pub use region_snapshot::*;
 pub use role_assignment::*;
 pub use role_builtin::*;
+pub use schema_versions::*;
 pub use semver_version::*;
 pub use service::*;
 pub use service_kind::*;
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index 771d6836dc..6becea4c2a 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -6,15 +6,6 @@
 //!
 //! NOTE: Should be kept up-to-date with dbinit.sql.
 
-use omicron_common::api::external::SemverVersion;
-
-/// The version of the database schema this particular version of Nexus was
-/// built against.
-///
-/// This should be updated whenever the schema is changed. For more details,
-/// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(45, 0, 0);
-
 table! {
     disk (id) {
         id -> Uuid,
diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs
new file mode 100644
index 0000000000..c267ce65d8
--- /dev/null
+++ b/nexus/db-model/src/schema_versions.rs
@@ -0,0 +1,800 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Database schema versions and upgrades
+//!
+//! For details, see schema/crdb/README.adoc in the root of this repository.
+
+use anyhow::{bail, ensure, Context};
+use camino::Utf8Path;
+use omicron_common::api::external::SemverVersion;
+use once_cell::sync::Lazy;
+use std::collections::BTreeMap;
+
+/// The version of the database schema this particular version of Nexus was
+/// built against
+///
+/// This must be updated when you change the database schema.  Refer to
+/// schema/crdb/README.adoc in the root of this repository for details.
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(46, 0, 0);
+
+/// List of all past database schema versions, in *reverse* order
+///
+/// If you want to change the Omicron database schema, you must update this.
+static KNOWN_VERSIONS: Lazy<Vec<KnownVersion>> = Lazy::new(|| {
+    vec![
+        // +- The next version goes here!  Duplicate this line, uncomment
+        // |  the *second* copy, then update that copy for your version,
+        // |  leaving the first copy as an example for the next person.
+        // v
+        // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(46, "first-named-migration"),
+        // The first many schema versions only vary by major or patch number and
+        // their path is predictable based on the version number.  (This was
+        // historically a problem because two pull requests both adding a new
+        // schema version might merge cleanly but produce an invalid result.)
+        KnownVersion::legacy(45, 0),
+        KnownVersion::legacy(44, 0),
+        KnownVersion::legacy(43, 0),
+        KnownVersion::legacy(42, 0),
+        KnownVersion::legacy(41, 0),
+        KnownVersion::legacy(40, 0),
+        KnownVersion::legacy(39, 0),
+        KnownVersion::legacy(38, 0),
+        KnownVersion::legacy(37, 1),
+        KnownVersion::legacy(37, 0),
+        KnownVersion::legacy(36, 0),
+        KnownVersion::legacy(35, 0),
+        KnownVersion::legacy(34, 0),
+        KnownVersion::legacy(33, 1),
+        KnownVersion::legacy(33, 0),
+        KnownVersion::legacy(32, 0),
+        KnownVersion::legacy(31, 0),
+        KnownVersion::legacy(30, 0),
+        KnownVersion::legacy(29, 0),
+        KnownVersion::legacy(28, 0),
+        KnownVersion::legacy(27, 0),
+        KnownVersion::legacy(26, 0),
+        KnownVersion::legacy(25, 0),
+        KnownVersion::legacy(24, 0),
+        KnownVersion::legacy(23, 1),
+        KnownVersion::legacy(23, 0),
+        KnownVersion::legacy(22, 0),
+        KnownVersion::legacy(21, 0),
+        KnownVersion::legacy(20, 0),
+        KnownVersion::legacy(19, 0),
+        KnownVersion::legacy(18, 0),
+        KnownVersion::legacy(17, 0),
+        KnownVersion::legacy(16, 0),
+        KnownVersion::legacy(15, 0),
+        KnownVersion::legacy(14, 0),
+        KnownVersion::legacy(13, 0),
+        KnownVersion::legacy(12, 0),
+        KnownVersion::legacy(11, 0),
+        KnownVersion::legacy(10, 0),
+        KnownVersion::legacy(9, 0),
+        KnownVersion::legacy(8, 0),
+        KnownVersion::legacy(7, 0),
+        KnownVersion::legacy(6, 0),
+        KnownVersion::legacy(5, 0),
+        KnownVersion::legacy(4, 0),
+        KnownVersion::legacy(3, 3),
+        KnownVersion::legacy(3, 2),
+        KnownVersion::legacy(3, 1),
+        KnownVersion::legacy(3, 0),
+        KnownVersion::legacy(2, 0),
+        KnownVersion::legacy(1, 0),
+    ]
+});
+
+/// The earliest supported schema version.
+pub const EARLIEST_SUPPORTED_VERSION: SemverVersion =
+    SemverVersion::new(1, 0, 0);
+
+/// Describes one version of the database schema
+#[derive(Debug, Clone)]
+struct KnownVersion {
+    /// All versions have an associated SemVer.  We only use the major number in
+    /// terms of determining compatibility.
+    semver: SemverVersion,
+
+    /// Path relative to the root of the schema ("schema/crdb" in the root of
+    /// this repo) where this version's update SQL files are stored
+    relative_path: String,
+}
+
+impl KnownVersion {
+    /// Generate a `KnownVersion` for a new schema version
+    ///
+    /// `major` should be the next available integer (one more than the previous
+    /// version's major number).
+    ///
+    /// `relative_path` is the path relative to "schema/crdb" (from the root of
+    /// this repository) where the SQL files live that will update the schema
+    /// from the previous version to this version.
+    fn new(major: u64, relative_path: &str) -> KnownVersion {
+        let semver = SemverVersion::new(major, 0, 0);
+        KnownVersion { semver, relative_path: relative_path.to_owned() }
+    }
+
+    /// Generate a `KnownVersion` for a version that predates the current
+    /// directory naming scheme
+    ///
+    /// These versions varied in both major and patch numbers and the path to
+    /// their SQL files was predictable based solely on the version.
+    ///
+    /// **This should not be used for new schema versions.**
+    fn legacy(major: u64, patch: u64) -> KnownVersion {
+        let semver = SemverVersion::new(major, 0, patch);
+        let relative_path = semver.to_string();
+        KnownVersion { semver, relative_path }
+    }
+}
+
+impl std::fmt::Display for KnownVersion {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.semver.fmt(f)
+    }
+}
+
+/// Load and inspect the set of all known schema versions
+#[derive(Debug, Clone)]
+pub struct AllSchemaVersions {
+    versions: BTreeMap<SemverVersion, SchemaVersion>,
+}
+
+impl AllSchemaVersions {
+    /// Load the set of all known schema versions from the given directory tree
+    ///
+    /// The directory should contain exactly one directory for each version.
+    /// Each version's directory should contain the SQL files that carry out
+    /// schema migration from the previous version.  See schema/crdb/README.adoc
+    /// for details.
+    pub fn load(
+        schema_directory: &Utf8Path,
+    ) -> Result<AllSchemaVersions, anyhow::Error> {
+        Self::load_known_versions(schema_directory, KNOWN_VERSIONS.iter())
+    }
+
+    /// Load a specific set of known schema versions using the legacy
+    /// conventions from the given directory tree
+    ///
+    /// This is only provided for certain integration tests.
+    #[doc(hidden)]
+    pub fn load_specific_legacy_versions<'a>(
+        schema_directory: &Utf8Path,
+        versions: impl Iterator<Item = &'a SemverVersion>,
+    ) -> Result<AllSchemaVersions, anyhow::Error> {
+        let known_versions: Vec<_> = versions
+            .map(|v| {
+                assert_eq!(v.0.minor, 0);
+                KnownVersion::legacy(v.0.major, v.0.patch)
+            })
+            .collect();
+
+        Self::load_known_versions(schema_directory, known_versions.iter())
+    }
+
+    fn load_known_versions<'a>(
+        schema_directory: &Utf8Path,
+        known_versions: impl Iterator<Item = &'a KnownVersion>,
+    ) -> Result<AllSchemaVersions, anyhow::Error> {
+        let mut versions = BTreeMap::new();
+        for known_version in known_versions {
+            let version_path =
+                schema_directory.join(&known_version.relative_path);
+            let schema_version = SchemaVersion::load_from_directory(
+                known_version.semver.clone(),
+                &version_path,
+            )
+            .with_context(|| {
+                format!(
+                    "loading schema version {} from {:?}",
+                    known_version.semver, schema_directory,
+                )
+            })?;
+
+            versions.insert(known_version.semver.clone(), schema_version);
+        }
+
+        Ok(AllSchemaVersions { versions })
+    }
+
+    /// Iterate over the set of all known schema versions in order starting with
+    /// the earliest supported version
+    pub fn iter_versions(&self) -> impl Iterator<Item = &SchemaVersion> {
+        self.versions.values()
+    }
+
+    /// Return whether `version` is a known schema version
+    pub fn contains_version(&self, version: &SemverVersion) -> bool {
+        self.versions.contains_key(version)
+    }
+
+    /// Iterate over the known schema versions within `bounds`
+    ///
+    /// This is generally used to iterate over all the schema versions between
+    /// two specific versions.
+    pub fn versions_range<R>(
+        &self,
+        bounds: R,
+    ) -> impl Iterator<Item = &'_ SchemaVersion>
+    where
+        R: std::ops::RangeBounds<SemverVersion>,
+    {
+        self.versions.range(bounds).map(|(_, v)| v)
+    }
+}
+
+/// Describes a single version of the schema, including the SQL steps to get
+/// from the previous version to the current one
+#[derive(Debug, Clone)]
+pub struct SchemaVersion {
+    semver: SemverVersion,
+    upgrade_from_previous: Vec<SchemaUpgradeStep>,
+}
+
+impl SchemaVersion {
+    /// Reads a "version directory" and reads all SQL changes into a result Vec.
+    ///
+    /// Files that do not begin with "up" and end with ".sql" are ignored. The
+    /// collection of `up*.sql` files must fall into one of these two
+    /// conventions:
+    ///
+    /// * "up.sql" with no other files
+    /// * "up1.sql", "up2.sql", ..., beginning from 1, optionally with leading
+    ///   zeroes (e.g., "up01.sql", "up02.sql", ...). There is no maximum value,
+    ///   but there may not be any gaps (e.g., if "up2.sql" and "up4.sql" exist,
+    ///   so must "up3.sql") and there must not be any repeats (e.g., if
+    ///   "up1.sql" exists, "up01.sql" must not exist).
+    ///
+    /// Any violation of these two rules will result in an error. Collections of
+    /// the second form (`up1.sql`, ...) will be sorted numerically.
+    fn load_from_directory(
+        semver: SemverVersion,
+        directory: &Utf8Path,
+    ) -> Result<SchemaVersion, anyhow::Error> {
+        let mut up_sqls = vec![];
+        let entries = directory
+            .read_dir_utf8()
+            .with_context(|| format!("Failed to readdir {directory}"))?;
+        for entry in entries {
+            let entry = entry.with_context(|| {
+                format!("Reading {directory:?}: invalid entry")
+            })?;
+            let pathbuf = entry.into_path();
+
+            // Ensure filename ends with ".sql"
+            if pathbuf.extension() != Some("sql") {
+                continue;
+            }
+
+            // Ensure filename begins with "up", and extract anything in between
+            // "up" and ".sql".
+            let Some(remaining_filename) = pathbuf
+                .file_stem()
+                .and_then(|file_stem| file_stem.strip_prefix("up"))
+            else {
+                continue;
+            };
+
+            // Ensure the remaining filename is either empty (i.e., the filename
+            // is exactly "up.sql") or parseable as an unsigned integer. We give
+            // "up.sql" the "up_number" 0 (checked in the loop below), and
+            // require any other number to be nonzero.
+            if remaining_filename.is_empty() {
+                up_sqls.push((0, pathbuf));
+            } else {
+                let Ok(up_number) = remaining_filename.parse::<u64>() else {
+                    bail!(
+                        "invalid filename (non-numeric `up*.sql`): {pathbuf}",
+                    );
+                };
+                ensure!(
+                    up_number != 0,
+                    "invalid filename (`up*.sql` numbering must start at 1): \
+                     {pathbuf}",
+                );
+                up_sqls.push((up_number, pathbuf));
+            }
+        }
+        up_sqls.sort();
+
+        // Validate that we have a reasonable sequence of `up*.sql` numbers.
+        match up_sqls.as_slice() {
+            [] => bail!("no `up*.sql` files found"),
+            [(up_number, path)] => {
+                // For a single file, we allow either `up.sql` (keyed as
+                // up_number=0) or `up1.sql`; reject any higher number.
+                ensure!(
+                    *up_number <= 1,
+                    "`up*.sql` numbering must start at 1: found first file \
+                     {path}"
+                );
+            }
+            _ => {
+                for (i, (up_number, path)) in up_sqls.iter().enumerate() {
+                    // We have 2 or more `up*.sql`; they should be numbered
+                    // exactly 1..=up_sqls.len().
+                    if i as u64 + 1 != *up_number {
+                        // We know we have at least two elements, so report an
+                        // error referencing either the next item (if we're
+                        // first) or the previous item (if we're not first).
+                        let (path_a, path_b) = if i == 0 {
+                            let (_, next_path) = &up_sqls[1];
+                            (path, next_path)
+                        } else {
+                            let (_, prev_path) = &up_sqls[i - 1];
+                            (prev_path, path)
+                        };
+                        bail!("invalid `up*.sql` sequence: {path_a}, {path_b}");
+                    }
+                }
+            }
+        }
+
+        // This collection of `up*.sql` files is valid.  Read them all, in
+        // order.
+        let mut steps = vec![];
+        for (_, path) in up_sqls.into_iter() {
+            let sql = std::fs::read_to_string(&path)
+                .with_context(|| format!("Cannot read {path}"))?;
+            // unwrap: `file_name()` is documented to return `None` only when
+            // the path is `..`.  But we got this path from reading the
+            // directory, and that process explicitly documents that it skips
+            // `..`.
+            steps.push(SchemaUpgradeStep {
+                label: path.file_name().unwrap().to_string(),
+                sql,
+            });
+        }
+
+        Ok(SchemaVersion { semver, upgrade_from_previous: steps })
+    }
+
+    /// Returns the semver for this schema version
+    pub fn semver(&self) -> &SemverVersion {
+        &self.semver
+    }
+
+    /// Returns true if this schema version is the one that the current program
+    /// thinks is the latest (current) one
+    pub fn is_current_software_version(&self) -> bool {
+        self.semver == SCHEMA_VERSION
+    }
+
+    /// Iterate over the SQL steps required to update the database schema from
+    /// the previous version to this one
+    pub fn upgrade_steps(&self) -> impl Iterator<Item = &SchemaUpgradeStep> {
+        self.upgrade_from_previous.iter()
+    }
+}
+
+impl std::fmt::Display for SchemaVersion {
+    fn fmt(
+        &self,
+        f: &mut std::fmt::Formatter<'_>,
+    ) -> Result<(), std::fmt::Error> {
+        self.semver.fmt(f)
+    }
+}
+
+/// Describes a single file containing a schema change, as SQL.
+#[derive(Debug, Clone)]
+pub struct SchemaUpgradeStep {
+    label: String,
+    sql: String,
+}
+
+impl SchemaUpgradeStep {
+    /// Returns a human-readable name for this step (the name of the file it
+    /// came from)
+    pub fn label(&self) -> &str {
+        self.label.as_ref()
+    }
+
+    /// Returns the actual SQL to execute for this step
+    pub fn sql(&self) -> &str {
+        self.sql.as_ref()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use camino_tempfile::Utf8TempDir;
+
+    #[test]
+    fn test_known_versions() {
+        if let Err(error) = verify_known_versions(
+            // The real list is defined in reverse order for developer
+            // convenience so we reverse it before processing.
+            KNOWN_VERSIONS.iter().rev(),
+            &EARLIEST_SUPPORTED_VERSION,
+            &SCHEMA_VERSION,
+            // Versions after 45 obey our modern, stricter rules.
+            45,
+        ) {
+            panic!("problem with static configuration: {:#}", error);
+        }
+    }
+
+    // (Test the test function)
+    #[test]
+    fn test_verify() {
+        // EARLIEST_SUPPORTED_VERSION is somehow wrong
+        let error = verify_known_versions(
+            [&KnownVersion::legacy(2, 0), &KnownVersion::legacy(3, 0)],
+            &SemverVersion::new(1, 0, 0),
+            &SemverVersion::new(3, 0, 0),
+            100,
+        )
+        .unwrap_err();
+        assert_eq!(
+            format!("{error:#}"),
+            "EARLIEST_SUPPORTED_VERSION is not the earliest in KNOWN_VERSIONS"
+        );
+
+        // SCHEMA_VERSION was not updated
+        let error = verify_known_versions(
+            [&KnownVersion::legacy(1, 0), &KnownVersion::legacy(2, 0)],
+            &SemverVersion::new(1, 0, 0),
+            &SemverVersion::new(1, 0, 0),
+            100,
+        )
+        .unwrap_err();
+        assert_eq!(
+            format!("{error:#}"),
+            "latest KNOWN_VERSION is 2.0.0, but SCHEMA_VERSION is 1.0.0"
+        );
+
+        // Latest version was duplicated instead of bumped (legacy)
+        let error = verify_known_versions(
+            [
+                &KnownVersion::legacy(1, 0),
+                &KnownVersion::legacy(2, 0),
+                &KnownVersion::legacy(2, 0),
+            ],
+            &EARLIEST_SUPPORTED_VERSION,
+            &SemverVersion::new(2, 0, 0),
+            100,
+        )
+        .unwrap_err();
+        assert_eq!(
+            format!("{error:#}"),
+            "KNOWN_VERSION 2.0.0 appears directly after 2.0.0, but is not later"
+        );
+
+        // Latest version was duplicated instead of bumped (modern case)
+        let error = verify_known_versions(
+            [
+                &KnownVersion::legacy(1, 0),
+                &KnownVersion::new(2, "dir1"),
+                &KnownVersion::new(2, "dir2"),
+            ],
+            &EARLIEST_SUPPORTED_VERSION,
+            &SemverVersion::new(2, 0, 0),
+            100,
+        )
+        .unwrap_err();
+        assert_eq!(
+            format!("{error:#}"),
+            "KNOWN_VERSION 2.0.0 appears directly after 2.0.0, but is not later"
+        );
+
+        // Version added out of order
+        let error = verify_known_versions(
+            [
+                &KnownVersion::legacy(1, 0),
+                &KnownVersion::legacy(2, 0),
+                &KnownVersion::legacy(1, 3),
+            ],
+            &EARLIEST_SUPPORTED_VERSION,
+            &SemverVersion::new(3, 0, 0),
+            100,
+        )
+        .unwrap_err();
+        assert_eq!(
+            format!("{error:#}"),
+            "KNOWN_VERSION 1.0.3 appears directly after 2.0.0, but is not later"
+        );
+
+        // Gaps are not allowed.
+        let error = verify_known_versions(
+            [
+                &KnownVersion::legacy(1, 0),
+                &KnownVersion::legacy(2, 0),
+                &KnownVersion::legacy(4, 0),
+            ],
+            &EARLIEST_SUPPORTED_VERSION,
+            &SemverVersion::new(4, 0, 0),
+            100,
+        )
+        .unwrap_err();
+        assert_eq!(
+            format!("{error:#}"),
+            "KNOWN_VERSION 4.0.0 appears directly after 2.0.0, but its major \
+            number is neither the same nor one greater"
+        );
+
+        // For the strict case, the patch level can't be non-zero.  You can only
+        // make this mistake by using `KnownVersion::legacy()` for a new
+        // version.
+        let error = verify_known_versions(
+            [
+                &KnownVersion::legacy(1, 0),
+                &KnownVersion::legacy(2, 0),
+                &KnownVersion::legacy(3, 2),
+            ],
+            &EARLIEST_SUPPORTED_VERSION,
+            &SemverVersion::new(3, 0, 2),
+            2,
+        )
+        .unwrap_err();
+        assert_eq!(format!("{error:#}"), "new patch versions must be zero");
+
+        // For the strict case, the directory name cannot contain the version at
+        // all.  You can only make this mistake by using
+        // `KnownVersion::legacy()` for a new version.
+        let error = verify_known_versions(
+            [
+                &KnownVersion::legacy(1, 0),
+                &KnownVersion::legacy(2, 0),
+                &KnownVersion::legacy(3, 0),
+            ],
+            &EARLIEST_SUPPORTED_VERSION,
+            &SemverVersion::new(3, 0, 0),
+            2,
+        )
+        .unwrap_err();
+        assert_eq!(
+            format!("{error:#}"),
+            "the relative path for a version should not contain the \
+            version itself"
+        );
+    }
+
+    fn verify_known_versions<'a, I>(
+        // list of known versions in order from earliest to latest
+        known_versions: I,
+        earliest: &SemverVersion,
+        latest: &SemverVersion,
+        min_strict_major: u64,
+    ) -> Result<(), anyhow::Error>
+    where
+        I: IntoIterator<Item = &'a KnownVersion>,
+    {
+        let mut known_versions = known_versions.into_iter();
+
+        // All known versions should be unique and increasing.
+        let first =
+            known_versions.next().expect("expected at least one KNOWN_VERSION");
+        ensure!(
+            first.semver == *earliest,
+            "EARLIEST_SUPPORTED_VERSION is not the earliest in KNOWN_VERSIONS"
+        );
+
+        let mut prev = first;
+        for v in known_versions {
+            println!("checking known version: {} -> {}", prev, v);
+            ensure!(
+                v.semver > prev.semver,
+                "KNOWN_VERSION {} appears directly after {}, but is not later",
+                v,
+                prev
+            );
+
+            // We currently make sure there are no gaps in the major number.
+            // This is not strictly necessary but if this isn't true then it was
+            // probably a mistake.
+            //
+            // It's allowed for the major numbers to be the same because a few
+            // past schema versions only bumped the patch number for whatever
+            // reason.
+            ensure!(
+                v.semver.0.major == prev.semver.0.major
+                    || v.semver.0.major == prev.semver.0.major + 1,
+                "KNOWN_VERSION {} appears directly after {}, but its major \
+                number is neither the same nor one greater",
+                v,
+                prev
+            );
+
+            // We never allowed minor versions to be zero and it is not
+            // currently possible to even construct one that had a non-zero
+            // minor number.
+            ensure!(v.semver.0.minor == 0, "new minor versions must be zero");
+
+            // We changed things after version 45 to require that:
+            //
+            // (1) the major always be bumped (the minor and patch must be zero)
+            // (2) users choose a unique directory name for the SQL files.  It
+            //     would defeat the point if people used the semver for
+            //
+            // After version 45, we do not allow non-zero minor or patch
+            // numbers.
+            if v.semver.0.major > min_strict_major {
+                ensure!(
+                    v.semver.0.patch == 0,
+                    "new patch versions must be zero"
+                );
+                ensure!(
+                    !v.relative_path.contains(&v.semver.to_string()),
+                    "the relative path for a version should not contain the \
+                    version itself"
+                );
+            }
+
+            prev = v;
+        }
+
+        ensure!(
+            prev.semver == *latest,
+            "latest KNOWN_VERSION is {}, but SCHEMA_VERSION is {}",
+            prev,
+            latest
+        );
+
+        Ok(())
+    }
+
+    // Confirm that `SchemaVersion::load_from_directory()` rejects `up*.sql`
+    // files where the `*` doesn't contain a positive integer.
+    #[tokio::test]
+    async fn test_reject_invalid_up_sql_names() {
+        for (invalid_filename, error_prefix) in [
+            ("upA.sql", "invalid filename (non-numeric `up*.sql`)"),
+            ("up1a.sql", "invalid filename (non-numeric `up*.sql`)"),
+            ("upaaa1.sql", "invalid filename (non-numeric `up*.sql`)"),
+            ("up-3.sql", "invalid filename (non-numeric `up*.sql`)"),
+            (
+                "up0.sql",
+                "invalid filename (`up*.sql` numbering must start at 1)",
+            ),
+            (
+                "up00.sql",
+                "invalid filename (`up*.sql` numbering must start at 1)",
+            ),
+            (
+                "up000.sql",
+                "invalid filename (`up*.sql` numbering must start at 1)",
+            ),
+        ] {
+            let tempdir = Utf8TempDir::new().unwrap();
+            let filename = tempdir.path().join(invalid_filename);
+            _ = tokio::fs::File::create(&filename).await.unwrap();
+            let maybe_schema = SchemaVersion::load_from_directory(
+                SemverVersion::new(12, 0, 0),
+                tempdir.path(),
+            );
+            match maybe_schema {
+                Ok(upgrade) => {
+                    panic!(
+                        "unexpected success on {invalid_filename} \
+                         (produced {upgrade:?})"
+                    );
+                }
+                Err(error) => {
+                    assert_eq!(
+                        format!("{error:#}"),
+                        format!("{error_prefix}: {filename}")
+                    );
+                }
+            }
+        }
+    }
+
+    // Confirm that `SchemaVersion::load_from_directory()` rejects a directory
+    // with no appropriately-named files.
+    #[tokio::test]
+    async fn test_reject_no_up_sql_files() {
+        for filenames in [
+            &[] as &[&str],
+            &["README.md"],
+            &["foo.sql", "bar.sql"],
+            &["up1sql", "up2sql"],
+        ] {
+            let tempdir = Utf8TempDir::new().unwrap();
+            for filename in filenames {
+                _ = tokio::fs::File::create(tempdir.path().join(filename))
+                    .await
+                    .unwrap();
+            }
+
+            let maybe_schema = SchemaVersion::load_from_directory(
+                SemverVersion::new(12, 0, 0),
+                tempdir.path(),
+            );
+            match maybe_schema {
+                Ok(upgrade) => {
+                    panic!(
+                        "unexpected success on {filenames:?} \
+                         (produced {upgrade:?})"
+                    );
+                }
+                Err(error) => {
+                    assert_eq!(
+                        format!("{error:#}"),
+                        "no `up*.sql` files found"
+                    );
+                }
+            }
+        }
+    }
+
+    // Confirm that `SchemaVersion::load_from_directory()` rejects collections
+    // of `up*.sql` files with individually-valid names but that do not pass the
+    // rules of the entire collection.
+    #[tokio::test]
+    async fn test_reject_invalid_up_sql_collections() {
+        for invalid_filenames in [
+            &["up.sql", "up1.sql"] as &[&str],
+            &["up1.sql", "up01.sql"],
+            &["up1.sql", "up3.sql"],
+            &["up1.sql", "up2.sql", "up3.sql", "up02.sql"],
+        ] {
+            let tempdir = Utf8TempDir::new().unwrap();
+            for filename in invalid_filenames {
+                _ = tokio::fs::File::create(tempdir.path().join(filename))
+                    .await
+                    .unwrap();
+            }
+
+            let maybe_schema = SchemaVersion::load_from_directory(
+                SemverVersion::new(12, 0, 0),
+                tempdir.path(),
+            );
+            match maybe_schema {
+                Ok(upgrade) => {
+                    panic!(
+                        "unexpected success on {invalid_filenames:?} \
+                         (produced {upgrade:?})"
+                    );
+                }
+                Err(error) => {
+                    let message = format!("{error:#}");
+                    assert!(
+                        message.starts_with("invalid `up*.sql` sequence: "),
+                        "message did not start with expected prefix: \
+                         {message:?}"
+                    );
+                }
+            }
+        }
+    }
+
+    // Confirm that `SchemaVersion::load_from_directory()` accepts legal
+    // collections of `up*.sql` filenames.
+    #[tokio::test]
+    async fn test_allows_valid_up_sql_collections() {
+        for filenames in [
+            &["up.sql"] as &[&str],
+            &["up1.sql", "up2.sql"],
+            &[
+                "up01.sql", "up02.sql", "up03.sql", "up04.sql", "up05.sql",
+                "up06.sql", "up07.sql", "up08.sql", "up09.sql", "up10.sql",
+                "up11.sql",
+            ],
+            &["up00001.sql", "up00002.sql", "up00003.sql"],
+        ] {
+            let tempdir = Utf8TempDir::new().unwrap();
+            for filename in filenames {
+                _ = tokio::fs::File::create(tempdir.path().join(filename))
+                    .await
+                    .unwrap();
+            }
+
+            let maybe_schema = SchemaVersion::load_from_directory(
+                SemverVersion::new(12, 0, 0),
+                tempdir.path(),
+            );
+            match maybe_schema {
+                Ok(_) => (),
+                Err(message) => {
+                    panic!("unexpected failure on {filenames:?}: {message:?}");
+                }
+            }
+        }
+    }
+}
diff --git a/nexus/db-queries/src/db/datastore/db_metadata.rs b/nexus/db-queries/src/db/datastore/db_metadata.rs
index cbbeaf2aa4..e6b57c1c72 100644
--- a/nexus/db-queries/src/db/datastore/db_metadata.rs
+++ b/nexus/db-queries/src/db/datastore/db_metadata.rs
@@ -8,144 +8,18 @@ use super::DataStore;
 use crate::db;
 use crate::db::error::public_error_from_diesel;
 use crate::db::error::ErrorHandler;
+use anyhow::{bail, ensure, Context};
 use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection};
-use camino::{Utf8Path, Utf8PathBuf};
 use chrono::Utc;
 use diesel::prelude::*;
-use nexus_config::SchemaConfig;
+use nexus_db_model::AllSchemaVersions;
+use nexus_db_model::SchemaVersion;
+use nexus_db_model::EARLIEST_SUPPORTED_VERSION;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::SemverVersion;
-use slog::Logger;
-use std::collections::BTreeSet;
-use std::ops::Bound;
+use slog::{error, info, o, Logger};
 use std::str::FromStr;
 
-pub const EARLIEST_SUPPORTED_VERSION: &'static str = "1.0.0";
-
-/// Describes a single file containing a schema change, as SQL.
-#[derive(Debug)]
-pub struct SchemaUpgradeStep {
-    pub path: Utf8PathBuf,
-    pub sql: String,
-}
-
-/// Describes a sequence of files containing schema changes.
-#[derive(Debug)]
-pub struct SchemaUpgrade {
-    pub steps: Vec<SchemaUpgradeStep>,
-}
-
-/// Reads a "version directory" and reads all SQL changes into
-/// a result Vec.
-///
-/// Files that do not begin with "up" and end with ".sql" are ignored. The
-/// collection of `up*.sql` files must fall into one of these two conventions:
-///
-/// * "up.sql" with no other files
-/// * "up1.sql", "up2.sql", ..., beginning from 1, optionally with leading
-///   zeroes (e.g., "up01.sql", "up02.sql", ...). There is no maximum value, but
-///   there may not be any gaps (e.g., if "up2.sql" and "up4.sql" exist, so must
-///   "up3.sql") and there must not be any repeats (e.g., if "up1.sql" exists,
-///   "up01.sql" must not exist).
-///
-/// Any violation of these two rules will result in an error. Collections of the
-/// second form (`up1.sql`, ...) will be sorted numerically.
-pub async fn all_sql_for_version_migration<P: AsRef<Utf8Path>>(
-    path: P,
-) -> Result<SchemaUpgrade, String> {
-    let target_dir = path.as_ref();
-    let mut up_sqls = vec![];
-    let entries = target_dir
-        .read_dir_utf8()
-        .map_err(|e| format!("Failed to readdir {target_dir}: {e}"))?;
-    for entry in entries {
-        let entry = entry.map_err(|err| format!("Invalid entry: {err}"))?;
-        let pathbuf = entry.into_path();
-
-        // Ensure filename ends with ".sql"
-        if pathbuf.extension() != Some("sql") {
-            continue;
-        }
-
-        // Ensure filename begins with "up", and extract anything in between
-        // "up" and ".sql".
-        let Some(remaining_filename) = pathbuf
-            .file_stem()
-            .and_then(|file_stem| file_stem.strip_prefix("up"))
-        else {
-            continue;
-        };
-
-        // Ensure the remaining filename is either empty (i.e., the filename is
-        // exactly "up.sql") or parseable as an unsigned integer. We give
-        // "up.sql" the "up_number" 0 (checked in the loop below), and require
-        // any other number to be nonzero.
-        if remaining_filename.is_empty() {
-            up_sqls.push((0, pathbuf));
-        } else {
-            let Ok(up_number) = remaining_filename.parse::<u64>() else {
-                return Err(format!(
-                    "invalid filename (non-numeric `up*.sql`): {pathbuf}",
-                ));
-            };
-            if up_number == 0 {
-                return Err(format!(
-                    "invalid filename (`up*.sql` numbering must start at 1): \
-                     {pathbuf}",
-                ));
-            }
-            up_sqls.push((up_number, pathbuf));
-        }
-    }
-    up_sqls.sort();
-
-    // Validate that we have a reasonable sequence of `up*.sql` numbers.
-    match up_sqls.as_slice() {
-        [] => return Err("no `up*.sql` files found".to_string()),
-        [(up_number, path)] => {
-            // For a single file, we allow either `up.sql` (keyed as
-            // up_number=0) or `up1.sql`; reject any higher number.
-            if *up_number > 1 {
-                return Err(format!(
-                    "`up*.sql` numbering must start at 1: found first file \
-                     {path}"
-                ));
-            }
-        }
-        _ => {
-            for (i, (up_number, path)) in up_sqls.iter().enumerate() {
-                // We have 2 or more `up*.sql`; they should be numbered exactly
-                // 1..=up_sqls.len().
-                if i as u64 + 1 != *up_number {
-                    // We know we have at least two elements, so report an error
-                    // referencing either the next item (if we're first) or the
-                    // previous item (if we're not first).
-                    let (path_a, path_b) = if i == 0 {
-                        let (_, next_path) = &up_sqls[1];
-                        (path, next_path)
-                    } else {
-                        let (_, prev_path) = &up_sqls[i - 1];
-                        (prev_path, path)
-                    };
-                    return Err(format!(
-                        "invalid `up*.sql` combination: {path_a}, {path_b}"
-                    ));
-                }
-            }
-        }
-    }
-
-    // This collection of `up*.sql` files is valid; read them all, in order.
-    let mut result = SchemaUpgrade { steps: vec![] };
-    for (_, path) in up_sqls.into_iter() {
-        let sql = tokio::fs::read_to_string(&path)
-            .await
-            .map_err(|e| format!("Cannot read {path}: {e}"))?;
-        result.steps.push(SchemaUpgradeStep { path: path.to_owned(), sql });
-    }
-    Ok(result)
-}
-
 impl DataStore {
     // Ensures that the database schema matches "desired_version".
     //
@@ -160,147 +34,118 @@ impl DataStore {
     // instances on the rack are operating on the same version of software.
     // If that assumption is broken, nothing would stop a "new deployment"
     // from making a change that invalidates the queries used by an "old
-    // deployment". This is fixable, but it requires slightly more knowledge
-    // about the deployment and liveness of Nexus services within the rack.
+    // deployment".
     pub async fn ensure_schema(
         &self,
         log: &Logger,
         desired_version: SemverVersion,
-        config: Option<&SchemaConfig>,
-    ) -> Result<(), String> {
-        let mut current_version = match self.database_schema_version().await {
-            Ok(current_version) => {
-                // NOTE: We could run with a less tight restriction.
-                //
-                // If we respect the meaning of the semver version, it should be possible
-                // to use subsequent versions, as long as they do not introduce breaking changes.
-                //
-                // However, at the moment, we opt for conservatism: if the database does not
-                // exactly match the schema version, we refuse to continue without modification.
-                if current_version == desired_version {
-                    info!(log, "Compatible database schema: {current_version}");
-                    return Ok(());
-                }
-                let observed = &current_version.0;
-                warn!(log, "Database schema {observed} does not match expected {desired_version}");
-                current_version
-            }
-            Err(e) => {
-                return Err(format!("Cannot read schema version: {e}"));
-            }
-        };
+        all_versions: Option<&AllSchemaVersions>,
+    ) -> Result<(), anyhow::Error> {
+        let found_version = self
+            .database_schema_version()
+            .await
+            .context("Cannot read database schema version")?;
+        let log = log.new(o!(
+            "found_version" => found_version.to_string(),
+            "desired_version" => desired_version.to_string(),
+        ));
 
-        let Some(config) = config else {
-            return Err(
-                "Not configured to automatically update schema".to_string()
-            );
-        };
+        // NOTE: We could run with a less tight restriction.
+        //
+        // If we respect the meaning of the semver version, it should be
+        // possible to use subsequent versions, as long as they do not introduce
+        // breaking changes.
+        //
+        // However, at the moment, we opt for conservatism: if the database does
+        // not exactly match the schema version, we refuse to continue without
+        // modification.
+        if found_version == desired_version {
+            info!(log, "Database schema version is up to date");
+            return Ok(());
+        }
 
-        if current_version > desired_version {
-            return Err("Nexus older than DB version: automatic downgrades are unsupported".to_string());
+        if found_version > desired_version {
+            error!(
+                log,
+                "Found schema version is newer than desired schema version";
+            );
+            bail!(
+                "Found schema version ({}) is newer than desired schema \
+                version ({})",
+                found_version,
+                desired_version,
+            )
         }
 
+        let Some(all_versions) = all_versions else {
+            error!(
+                log,
+                "Database schema version is out of date, but automatic update \
+                is disabled",
+            );
+            bail!("Schema is out of date but automatic update is disabled");
+        };
+
         // If we're here, we know the following:
         //
         // - The schema does not match our expected version (or at least, it
-        // didn't when we read it moments ago).
+        //   didn't when we read it moments ago).
         // - We should attempt to automatically upgrade the schema.
-        //
-        // We do the following:
-        // - Look in the schema directory for all the changes, in-order, to
-        // migrate from our current version to the desired version.
-
-        info!(log, "Reading schemas from {}", config.schema_dir);
-        let mut dir = tokio::fs::read_dir(&config.schema_dir)
-            .await
-            .map_err(|e| format!("Failed to read schema config dir: {e}"))?;
-        let mut all_versions = BTreeSet::new();
-        while let Some(entry) = dir
-            .next_entry()
-            .await
-            .map_err(|e| format!("Failed to read schema dir: {e}"))?
-        {
-            if entry.file_type().await.map_err(|e| e.to_string())?.is_dir() {
-                let name = entry
-                    .file_name()
-                    .into_string()
-                    .map_err(|_| "Non-unicode schema dir".to_string())?;
-                if let Ok(observed_version) = name.parse::<SemverVersion>() {
-                    all_versions.insert(observed_version);
-                } else {
-                    let err_msg =
-                        format!("Failed to parse {name} as a semver version");
-                    warn!(log, "{err_msg}");
-                    return Err(err_msg);
-                }
-            }
-        }
+        info!(log, "Database schema is out of date.  Attempting upgrade.");
+        ensure!(
+            all_versions.contains_version(&found_version),
+            "Found schema version {found_version} was not found",
+        );
 
-        if !all_versions.contains(&current_version) {
-            return Err(format!(
-                "Current DB version {current_version} was not found in {}",
-                config.schema_dir
-            ));
-        }
         // TODO: Test this?
-        if !all_versions.contains(&desired_version) {
-            return Err(format!(
-                "Target DB version {desired_version} was not found in {}",
-                config.schema_dir
-            ));
-        }
+        ensure!(
+            all_versions.contains_version(&desired_version),
+            "Desired version {desired_version} was not found",
+        );
 
-        let target_versions = all_versions.range((
-            Bound::Excluded(&current_version),
-            Bound::Included(&desired_version),
-        ));
+        let target_versions: Vec<&SchemaVersion> = all_versions
+            .versions_range(&found_version..=&desired_version)
+            .collect();
 
+        let mut current_version = found_version;
         for target_version in target_versions.into_iter() {
-            info!(
-                log,
-                "Attempting to upgrade schema";
+            let log = log.new(o!(
                 "current_version" => current_version.to_string(),
-                "target_version" => target_version.to_string(),
-            );
-
-            let target_dir = config.schema_dir.join(target_version.to_string());
-
-            let schema_change =
-                all_sql_for_version_migration(&target_dir).await?;
+                "target_version" => target_version.semver().to_string(),
+            ));
+            info!(log, "Attempting to upgrade schema");
 
             // Confirm the current version, set the "target_version"
             // column to indicate that a schema update is in-progress.
             //
             // Sets the following:
             // - db_metadata.target_version = new version
-            self.prepare_schema_update(&current_version, &target_version)
-                .await
-                .map_err(|e| e.to_string())?;
-
-            info!(
-                log,
-                "Marked schema upgrade as prepared";
-                "current_version" => current_version.to_string(),
-                "target_version" => target_version.to_string(),
-            );
+            self.prepare_schema_update(
+                &current_version,
+                &target_version.semver(),
+            )
+            .await
+            .context("preparing schema update")?;
+            info!(log, "Marked schema upgrade as prepared");
 
-            for SchemaUpgradeStep { path: _, sql } in &schema_change.steps {
-                // Perform the schema change.
+            for step in target_version.upgrade_steps() {
+                // Perform the schema change step.
                 self.apply_schema_update(
                     &current_version,
-                    &target_version,
-                    &sql,
+                    &target_version.semver(),
+                    step.sql(),
                 )
                 .await
-                .map_err(|e| e.to_string())?;
+                .with_context(|| {
+                    format!(
+                        "update to {}, applying step {:?}",
+                        target_version.semver(),
+                        step.label()
+                    )
+                })?;
             }
 
-            info!(
-                log,
-                "Applied schema upgrade";
-                "current_version" => current_version.to_string(),
-                "target_version" => target_version.to_string(),
-            );
+            info!(log, "Applied schema upgrade");
 
             // NOTE: We could execute the schema change in a background task,
             // and let it propagate, while observing it with the following
@@ -324,20 +169,23 @@ impl DataStore {
             // Now that the schema change has completed, set the following:
             // - db_metadata.version = new version
             // - db_metadata.target_version = NULL
-            self.finalize_schema_update(&current_version, &target_version)
-                .await
-                .map_err(|e| e.to_string())?;
+            self.finalize_schema_update(
+                &current_version,
+                target_version.semver(),
+            )
+            .await
+            .context("finalizing schema update")?;
 
             info!(
                 log,
                 "Finalized schema upgrade";
-                "current_version" => current_version.to_string(),
-                "target_version" => target_version.to_string(),
             );
 
-            current_version = target_version.clone();
+            current_version = target_version.semver().clone();
         }
 
+        info!(log, "Schema update complete");
+
         Ok(())
     }
 
@@ -407,13 +255,13 @@ impl DataStore {
         &self,
         current: &SemverVersion,
         target: &SemverVersion,
-        sql: &String,
+        sql: &str,
     ) -> Result<(), Error> {
         let conn = self.pool_connection_unauthorized().await?;
 
         let result = self.transaction_retry_wrapper("apply_schema_update")
             .transaction(&conn, |conn| async move {
-                if target.to_string() != EARLIEST_SUPPORTED_VERSION {
+                if *target != EARLIEST_SUPPORTED_VERSION {
                     let validate_version_query = format!("SELECT CAST(\
                             IF(\
                                 (\
@@ -460,9 +308,10 @@ impl DataStore {
         .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
 
         if rows_updated != 1 {
-            return Err(Error::internal_error(
-                &format!("Failed to finalize schema update from version {from_version} to {to_version}"),
-            ));
+            return Err(Error::internal_error(&format!(
+                "Failed to finalize schema update from version \
+                {from_version} to {to_version}"
+            )));
         }
         Ok(())
     }
@@ -472,149 +321,11 @@ impl DataStore {
 mod test {
     use super::*;
     use camino_tempfile::Utf8TempDir;
-    use nexus_db_model::schema::SCHEMA_VERSION;
+    use nexus_db_model::SCHEMA_VERSION;
     use nexus_test_utils::db as test_db;
     use omicron_test_utils::dev;
     use std::sync::Arc;
 
-    // Confirm that `all_sql_for_version_migration` rejects `up*.sql` files
-    // where the `*` doesn't contain a positive integer.
-    #[tokio::test]
-    async fn all_sql_for_version_migration_rejects_invalid_up_sql_names() {
-        for (invalid_filename, error_prefix) in [
-            ("upA.sql", "invalid filename (non-numeric `up*.sql`)"),
-            ("up1a.sql", "invalid filename (non-numeric `up*.sql`)"),
-            ("upaaa1.sql", "invalid filename (non-numeric `up*.sql`)"),
-            ("up-3.sql", "invalid filename (non-numeric `up*.sql`)"),
-            (
-                "up0.sql",
-                "invalid filename (`up*.sql` numbering must start at 1)",
-            ),
-            (
-                "up00.sql",
-                "invalid filename (`up*.sql` numbering must start at 1)",
-            ),
-            (
-                "up000.sql",
-                "invalid filename (`up*.sql` numbering must start at 1)",
-            ),
-        ] {
-            let tempdir = Utf8TempDir::new().unwrap();
-            let filename = tempdir.path().join(invalid_filename);
-            _ = tokio::fs::File::create(&filename).await.unwrap();
-
-            match all_sql_for_version_migration(tempdir.path()).await {
-                Ok(upgrade) => {
-                    panic!(
-                        "unexpected success on {invalid_filename} \
-                         (produced {upgrade:?})"
-                    );
-                }
-                Err(message) => {
-                    assert_eq!(message, format!("{error_prefix}: {filename}"));
-                }
-            }
-        }
-    }
-
-    // Confirm that `all_sql_for_version_migration` rejects a directory with no
-    // appriopriately-named files.
-    #[tokio::test]
-    async fn all_sql_for_version_migration_rejects_no_up_sql_files() {
-        for filenames in [
-            &[] as &[&str],
-            &["README.md"],
-            &["foo.sql", "bar.sql"],
-            &["up1sql", "up2sql"],
-        ] {
-            let tempdir = Utf8TempDir::new().unwrap();
-            for filename in filenames {
-                _ = tokio::fs::File::create(tempdir.path().join(filename))
-                    .await
-                    .unwrap();
-            }
-
-            match all_sql_for_version_migration(tempdir.path()).await {
-                Ok(upgrade) => {
-                    panic!(
-                        "unexpected success on {filenames:?} \
-                         (produced {upgrade:?})"
-                    );
-                }
-                Err(message) => {
-                    assert_eq!(message, "no `up*.sql` files found");
-                }
-            }
-        }
-    }
-
-    // Confirm that `all_sql_for_version_migration` rejects collections of
-    // `up*.sql` files with individually-valid names but that do not pass the
-    // rules of the entire collection.
-    #[tokio::test]
-    async fn all_sql_for_version_migration_rejects_invalid_up_sql_collections()
-    {
-        for invalid_filenames in [
-            &["up.sql", "up1.sql"] as &[&str],
-            &["up1.sql", "up01.sql"],
-            &["up1.sql", "up3.sql"],
-            &["up1.sql", "up2.sql", "up3.sql", "up02.sql"],
-        ] {
-            let tempdir = Utf8TempDir::new().unwrap();
-            for filename in invalid_filenames {
-                _ = tokio::fs::File::create(tempdir.path().join(filename))
-                    .await
-                    .unwrap();
-            }
-
-            match all_sql_for_version_migration(tempdir.path()).await {
-                Ok(upgrade) => {
-                    panic!(
-                        "unexpected success on {invalid_filenames:?} \
-                         (produced {upgrade:?})"
-                    );
-                }
-                Err(message) => {
-                    assert!(
-                        message.starts_with("invalid `up*.sql` combination: "),
-                        "message did not start with expected prefix: \
-                         {message:?}"
-                    );
-                }
-            }
-        }
-    }
-
-    // Confirm that `all_sql_for_version_migration` accepts legal collections of
-    // `up*.sql` filenames.
-    #[tokio::test]
-    async fn all_sql_for_version_migration_allows_valid_up_sql_collections() {
-        for filenames in [
-            &["up.sql"] as &[&str],
-            &["up1.sql", "up2.sql"],
-            &[
-                "up01.sql", "up02.sql", "up03.sql", "up04.sql", "up05.sql",
-                "up06.sql", "up07.sql", "up08.sql", "up09.sql", "up10.sql",
-                "up11.sql",
-            ],
-            &["up00001.sql", "up00002.sql", "up00003.sql"],
-        ] {
-            let tempdir = Utf8TempDir::new().unwrap();
-            for filename in filenames {
-                _ = tokio::fs::File::create(tempdir.path().join(filename))
-                    .await
-                    .unwrap();
-            }
-
-            match all_sql_for_version_migration(tempdir.path()).await {
-                Ok(_) => (),
-                Err(message) => {
-                    panic!("unexpected failure on {filenames:?}: {message:?}");
-                }
-            }
-        }
-    }
-
     // Confirms that calling the internal "ensure_schema" function can succeed
     // when the database is already at that version.
     #[tokio::test]
@@ -664,12 +375,14 @@ mod test {
             }
         };
 
-        // Create the old version directory, and also update the on-disk "current version" to
-        // this value.
+        // Create the old version directory, and also update the on-disk
+        // "current version" to this value.
         //
-        // Nexus will decide to upgrade to, at most, the version that its own binary understands.
+        // Nexus will decide to upgrade to, at most, the version that its own
+        // binary understands.
         //
-        // To trigger this action within a test, we manually set the "known to DB" version.
+        // To trigger this action within a test, we manually set the "known to
+        // DB" version.
         let v0 = SemverVersion::new(0, 0, 0);
         use db::schema::db_metadata::dsl;
         diesel::update(dsl::db_metadata.filter(dsl::singleton.eq(true)))
@@ -706,29 +419,40 @@ mod test {
             .await;
 
         // Show that the datastores can be created concurrently.
-        let config =
-            SchemaConfig { schema_dir: config_dir.path().to_path_buf() };
+        let all_versions = AllSchemaVersions::load_specific_legacy_versions(
+            config_dir.path(),
+            [&v0, &v1, &v2].into_iter(),
+        )
+        .expect("failed to load schema");
         let _ = futures::future::join_all((0..10).map(|_| {
+            let all_versions = all_versions.clone();
             let log = log.clone();
             let pool = pool.clone();
-            let config = config.clone();
             tokio::task::spawn(async move {
-                let datastore = DataStore::new(&log, pool, Some(&config)).await?;
+                let datastore =
+                    DataStore::new(&log, pool, Some(&all_versions)).await?;
 
                 // This is the crux of this test: confirm that, as each
                 // migration completes, it's not possible to see any artifacts
                 // of the "v1" migration (namely: the 'Widget' table should not
                 // exist).
-                let result = diesel::select(
-                        diesel::dsl::sql::<diesel::sql_types::Bool>(
-                            "EXISTS (SELECT * FROM pg_tables WHERE tablename = 'widget')"
-                        )
-                    )
-                    .get_result_async::<bool>(&*datastore.pool_connection_for_tests().await.unwrap())
-                    .await
-                    .expect("Failed to query for table");
-                assert_eq!(result, false, "The 'widget' table should have been deleted, but it exists.\
-                    This failure means an old update was re-applied after a newer update started.");
+                let result = diesel::select(diesel::dsl::sql::<
+                    diesel::sql_types::Bool,
+                >(
+                    "EXISTS (SELECT * FROM pg_tables WHERE \
+                            tablename = 'widget')",
+                ))
+                .get_result_async::<bool>(
+                    &*datastore.pool_connection_for_tests().await.unwrap(),
+                )
+                .await
+                .expect("Failed to query for table");
+                assert_eq!(
+                    result, false,
+                    "The 'widget' table should have been deleted, but it \
+                    exists.  This failure means an old update was re-applied \
+                    after a newer update started."
+                );
 
                 Ok::<_, String>(datastore)
             })
diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs
index ab00d876b8..8ae245aaa0 100644
--- a/nexus/db-queries/src/db/datastore/mod.rs
+++ b/nexus/db-queries/src/db/datastore/mod.rs
@@ -33,7 +33,6 @@ use diesel::prelude::*;
 use diesel::query_builder::{QueryFragment, QueryId};
 use diesel::query_dsl::methods::LoadQuery;
 use diesel::{ExpressionMethods, QueryDsl};
-use nexus_config::SchemaConfig;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::IdentityMetadataCreateParams;
 use omicron_common::api::external::LookupType;
@@ -100,13 +99,10 @@ mod vpc;
 mod zpool;
 
 pub use address_lot::AddressLotCreateResult;
-pub use db_metadata::{
-    all_sql_for_version_migration, SchemaUpgrade, SchemaUpgradeStep,
-    EARLIEST_SUPPORTED_VERSION,
-};
 pub use dns::DnsVersionUpdateBuilder;
 pub use instance::InstanceAndActiveVmm;
 pub use inventory::DataStoreInventoryTest;
+use nexus_db_model::AllSchemaVersions;
 pub use probe::ProbeInfo;
 pub use rack::RackInit;
 pub use silo::Discoverability;
@@ -197,14 +193,13 @@ impl DataStore {
     pub async fn new(
         log: &Logger,
         pool: Arc<Pool>,
-        config: Option<&SchemaConfig>,
+        config: Option<&AllSchemaVersions>,
     ) -> Result<Self, String> {
         let datastore =
             Self::new_unchecked(log.new(o!("component" => "datastore")), pool)?;
 
         // Keep looping until we find that the schema matches our expectation.
-        const EXPECTED_VERSION: SemverVersion =
-            nexus_db_model::schema::SCHEMA_VERSION;
+        const EXPECTED_VERSION: SemverVersion = nexus_db_model::SCHEMA_VERSION;
         retry_notify(
             retry_policy_internal_service(),
             || async {
diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs
index fdae303489..9e0b12d83d 100644
--- a/nexus/src/app/mod.rs
+++ b/nexus/src/app/mod.rs
@@ -87,6 +87,7 @@ pub(crate) mod sagas;
 
 pub(crate) use nexus_db_queries::db::queries::disk::MAX_DISKS_PER_INSTANCE;
 
+use nexus_db_model::AllSchemaVersions;
 pub(crate) use nexus_db_model::MAX_NICS_PER_INSTANCE;
 
 // XXX: Might want to recast as max *floating* IPs, we have at most one
@@ -202,13 +203,16 @@ impl Nexus {
         authz: Arc<authz::Authz>,
     ) -> Result<Arc<Nexus>, String> {
         let pool = Arc::new(pool);
+        let all_versions = config
+            .pkg
+            .schema
+            .as_ref()
+            .map(|s| AllSchemaVersions::load(&s.schema_dir))
+            .transpose()
+            .map_err(|error| format!("{error:#}"))?;
         let db_datastore = Arc::new(
-            db::DataStore::new(
-                &log,
-                Arc::clone(&pool),
-                config.pkg.schema.as_ref(),
-            )
-            .await?,
+            db::DataStore::new(&log, Arc::clone(&pool), all_versions.as_ref())
+                .await?,
         );
         db_datastore.register_producers(&producer_registry);
 
diff --git a/nexus/src/bin/schema-updater.rs b/nexus/src/bin/schema-updater.rs
index 6c7fe5adf3..2932a819d8 100644
--- a/nexus/src/bin/schema-updater.rs
+++ b/nexus/src/bin/schema-updater.rs
@@ -4,13 +4,14 @@
 
 //! Upgrades CRDB schema
 
-use anyhow::{anyhow, bail};
+use anyhow::anyhow;
 use camino::Utf8PathBuf;
 use clap::Parser;
 use clap::Subcommand;
 use nexus_config::PostgresConfigWithUrl;
 use nexus_config::SchemaConfig;
-use nexus_db_model::schema::SCHEMA_VERSION;
+use nexus_db_model::AllSchemaVersions;
+use nexus_db_model::SCHEMA_VERSION;
 use nexus_db_queries::db;
 use nexus_db_queries::db::DataStore;
 use omicron_common::api::external::SemverVersion;
@@ -20,7 +21,6 @@ use slog::LevelFilter;
 use slog::Logger;
 use slog_term::FullFormat;
 use slog_term::TermDecorator;
-use std::collections::BTreeSet;
 use std::sync::Arc;
 
 fn parse_log_level(s: &str) -> anyhow::Result<Level> {
@@ -73,6 +73,7 @@ async fn main() -> anyhow::Result<()> {
     let crdb_cfg = db::Config { url: args.url };
     let pool = Arc::new(db::Pool::new(&log, &crdb_cfg));
     let schema_config = SchemaConfig { schema_dir: args.schema_directory };
+    let all_versions = AllSchemaVersions::load(&schema_config.schema_dir)?;
 
     // We use the unchecked constructor of the datastore because we
     // don't want to block on someone else applying an upgrade.
@@ -87,50 +88,24 @@ async fn main() -> anyhow::Result<()> {
                 .map(|v| v.to_string())
                 .unwrap_or_else(|_| "Unknown".to_string());
 
-            println!("Current Version: {current_version}");
-
-            let mut dir =
-                tokio::fs::read_dir(&schema_config.schema_dir).await.map_err(
-                    |e| anyhow!("Failed to read from schema directory: {e}"),
-                )?;
-
-            let mut all_versions = BTreeSet::new();
-            while let Some(entry) = dir
-                .next_entry()
-                .await
-                .map_err(|e| anyhow!("Failed to read schema dir: {e}"))?
-            {
-                if entry.file_type().await.map_err(|e| anyhow!(e))?.is_dir() {
-                    let name = entry
-                        .file_name()
-                        .into_string()
-                        .map_err(|_| anyhow!("Non-unicode schema dir"))?;
-                    if let Ok(observed_version) = name.parse::<SemverVersion>()
-                    {
-                        all_versions.insert(observed_version);
-                    } else {
-                        bail!("Failed to parse {name} as a semver version");
-                    }
-                }
-            }
-
+            println!("Current Version in database: {current_version}");
             println!("Known Versions:");
-            for version in &all_versions {
+            for version in all_versions.iter_versions() {
                 let mut extra = String::new();
-                if version.to_string() == current_version {
+                if version.semver().to_string() == current_version {
                     extra.push_str(" (reported by database)");
                 };
-                if version == &SCHEMA_VERSION {
+                if version.is_current_software_version() {
                     extra.push_str(" (expected by Nexus)");
                 };
 
-                println!("  {version}{extra}")
+                println!("  {}{extra}", version.semver())
             }
         }
         Cmd::Upgrade { version } => {
             println!("Upgrading to {version}");
             datastore
-                .ensure_schema(&log, version.clone(), Some(&schema_config))
+                .ensure_schema(&log, version.clone(), Some(&all_versions))
                 .await
                 .map_err(|e| anyhow!(e))?;
             println!("Upgrade to {version} complete");
diff --git a/nexus/tests/integration_tests/initialization.rs b/nexus/tests/integration_tests/initialization.rs
index a76aef832e..a305a4178e 100644
--- a/nexus/tests/integration_tests/initialization.rs
+++ b/nexus/tests/integration_tests/initialization.rs
@@ -191,7 +191,7 @@ async fn test_nexus_boots_with_valid_schema() {
 
 #[tokio::test]
 async fn test_nexus_does_not_boot_without_valid_schema() {
-    let s = nexus_db_model::schema::SCHEMA_VERSION;
+    let s = nexus_db_model::SCHEMA_VERSION;
 
     let schemas_to_test = vec![
         semver::Version::new(s.0.major + 1, s.0.minor, s.0.patch),
@@ -240,7 +240,7 @@ async fn test_nexus_does_not_boot_without_valid_schema() {
 
 #[tokio::test]
 async fn test_nexus_does_not_boot_until_schema_updated() {
-    let good_schema = nexus_db_model::schema::SCHEMA_VERSION;
+    let good_schema = nexus_db_model::SCHEMA_VERSION;
     let bad_schema = semver::Version::new(
         good_schema.0.major + 1,
         good_schema.0.minor,
diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs
index f9bc4e1da7..dbe436a8f2 100644
--- a/nexus/tests/integration_tests/schema.rs
+++ b/nexus/tests/integration_tests/schema.rs
@@ -8,10 +8,9 @@ use dropshot::test_util::LogContext;
 use futures::future::BoxFuture;
 use nexus_config::NexusConfig;
 use nexus_config::SchemaConfig;
-use nexus_db_model::schema::SCHEMA_VERSION as LATEST_SCHEMA_VERSION;
-use nexus_db_queries::db::datastore::{
-    all_sql_for_version_migration, EARLIEST_SUPPORTED_VERSION,
-};
+use nexus_db_model::EARLIEST_SUPPORTED_VERSION;
+use nexus_db_model::SCHEMA_VERSION as LATEST_SCHEMA_VERSION;
+use nexus_db_model::{AllSchemaVersions, SchemaVersion};
 use nexus_db_queries::db::DISALLOW_FULL_TABLE_SCAN_SQL;
 use nexus_test_utils::{db, load_test_config, ControlPlaneTestContextBuilder};
 use omicron_common::api::external::SemverVersion;
@@ -20,7 +19,7 @@ use omicron_test_utils::dev::db::{Client, CockroachInstance};
 use pretty_assertions::{assert_eq, assert_ne};
 use similar_asserts;
 use slog::Logger;
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::BTreeMap;
 use std::net::IpAddr;
 use tokio::time::timeout;
 use tokio::time::Duration;
@@ -111,10 +110,10 @@ async fn apply_update_as_transaction(
 async fn apply_update(
     log: &Logger,
     crdb: &CockroachInstance,
-    version: &str,
+    version: &SchemaVersion,
     times_to_apply: usize,
 ) {
-    let log = log.new(o!("target version" => version.to_string()));
+    let log = log.new(o!("target version" => version.semver().to_string()));
     info!(log, "Performing upgrade");
 
     let client = crdb.connect().await.expect("failed to connect");
@@ -126,25 +125,27 @@ async fn apply_update(
 
     // We skip this for the earliest supported version because these tables
     // might not exist yet.
-    if version != EARLIEST_SUPPORTED_VERSION {
+    if *version.semver() != EARLIEST_SUPPORTED_VERSION {
         info!(log, "Updating schema version in db_metadata (setting target)");
-        let sql = format!("UPDATE omicron.public.db_metadata SET target_version = '{}' WHERE singleton = true;", version);
+        let sql = format!(
+            "UPDATE omicron.public.db_metadata SET target_version = '{}' \
+            WHERE singleton = true;",
+            version
+        );
         client
             .batch_execute(&sql)
             .await
             .expect("Failed to bump version number");
     }
 
-    let target_dir = Utf8PathBuf::from(SCHEMA_DIR).join(version);
-    let schema_change =
-        all_sql_for_version_migration(&target_dir).await.unwrap();
-
     for _ in 0..times_to_apply {
-        for nexus_db_queries::db::datastore::SchemaUpgradeStep { path, sql } in
-            &schema_change.steps
-        {
-            info!(log, "Applying sql schema upgrade step"; "path" => path.to_string());
-            apply_update_as_transaction(&log, &client, sql).await;
+        for step in version.upgrade_steps() {
+            info!(
+                log,
+                "Applying sql schema upgrade step";
+                "file" => step.label()
+            );
+            apply_update_as_transaction(&log, &client, step.sql()).await;
         }
     }
 
@@ -152,7 +153,11 @@ async fn apply_update(
     //
     // We do so explicitly here.
     info!(log, "Updating schema version in db_metadata (removing target)");
-    let sql = format!("UPDATE omicron.public.db_metadata SET version = '{}', target_version = NULL WHERE singleton = true;", version);
+    let sql = format!(
+        "UPDATE omicron.public.db_metadata SET version = '{}', \
+        target_version = NULL WHERE singleton = true;",
+        version
+    );
     client.batch_execute(&sql).await.expect("Failed to bump version number");
 
     client.cleanup().await.expect("cleaning up after wipe");
@@ -186,7 +191,8 @@ impl From<(&str, &str)> for SqlEnum {
 // interpret SQL types.
 //
 // Note that for the purposes of schema comparisons, we don't care about parsing
-// the contents of the database, merely the schema and equality of contained data.
+// the contents of the database, merely the schema and equality of contained
+// data.
 #[derive(PartialEq, Clone, Debug)]
 enum AnySqlType {
     Bool(bool),
@@ -443,20 +449,8 @@ async fn crdb_list_enums(crdb: &CockroachInstance) -> Vec<Row> {
     process_rows(&rows)
 }
 
-async fn read_all_schema_versions() -> BTreeSet<SemverVersion> {
-    let mut all_versions = BTreeSet::new();
-
-    let mut dir =
-        tokio::fs::read_dir(SCHEMA_DIR).await.expect("Access schema dir");
-    while let Some(entry) = dir.next_entry().await.expect("Read dirent") {
-        if entry.file_type().await.unwrap().is_dir() {
-            let name = entry.file_name().into_string().unwrap();
-            if let Ok(observed_version) = name.parse::<SemverVersion>() {
-                all_versions.insert(observed_version);
-            }
-        }
-    }
-    all_versions
+fn read_all_schema_versions() -> AllSchemaVersions {
+    AllSchemaVersions::load(camino::Utf8Path::new(SCHEMA_DIR)).unwrap()
 }
 
 // This test confirms the following behavior:
@@ -474,10 +468,16 @@ async fn nexus_applies_update_on_boot() {
     let crdb = builder.database.as_ref().expect("Should have started CRDB");
 
     // We started with an empty database -- apply an update here to bring
-    // us forward to our oldest supported schema version before trying to boot nexus.
-    apply_update(log, &crdb, EARLIEST_SUPPORTED_VERSION, 1).await;
+    // us forward to our oldest supported schema version before trying to boot
+    // nexus.
+    let all_versions = read_all_schema_versions();
+    let earliest = all_versions
+        .iter_versions()
+        .next()
+        .expect("missing earliest schema version");
+    apply_update(log, &crdb, earliest, 1).await;
     assert_eq!(
-        EARLIEST_SUPPORTED_VERSION,
+        EARLIEST_SUPPORTED_VERSION.to_string(),
         query_crdb_schema_version(&crdb).await
     );
 
@@ -542,16 +542,26 @@ async fn nexus_cannot_apply_update_from_unknown_version() {
     let log = &builder.logctx.log;
     let crdb = builder.database.as_ref().expect("Should have started CRDB");
 
-    apply_update(log, &crdb, EARLIEST_SUPPORTED_VERSION, 1).await;
+    let all_versions = read_all_schema_versions();
+    let earliest = all_versions
+        .iter_versions()
+        .next()
+        .expect("missing earliest schema version");
+    apply_update(log, &crdb, earliest, 1).await;
     assert_eq!(
-        EARLIEST_SUPPORTED_VERSION,
+        EARLIEST_SUPPORTED_VERSION.to_string(),
         query_crdb_schema_version(&crdb).await
     );
 
     // This version is not valid; it does not exist.
     let version = "0.0.0";
-    crdb.connect().await.expect("Failed to connect")
-        .batch_execute(&format!("UPDATE omicron.public.db_metadata SET version = '{version}' WHERE singleton = true"))
+    crdb.connect()
+        .await
+        .expect("Failed to connect")
+        .batch_execute(&format!(
+            "UPDATE omicron.public.db_metadata SET version = '{version}' \
+            WHERE singleton = true"
+        ))
         .await
         .expect("Failed to update schema");
 
@@ -583,11 +593,13 @@ async fn versions_have_idempotent_up() {
     let populate = false;
     let mut crdb = test_setup_just_crdb(&logctx.log, populate).await;
 
-    let all_versions = read_all_schema_versions().await;
-
-    for version in &all_versions {
-        apply_update(log, &crdb, &version.to_string(), 2).await;
-        assert_eq!(version.to_string(), query_crdb_schema_version(&crdb).await);
+    let all_versions = read_all_schema_versions();
+    for version in all_versions.iter_versions() {
+        apply_update(log, &crdb, &version, 2).await;
+        assert_eq!(
+            version.semver().to_string(),
+            query_crdb_schema_version(&crdb).await
+        );
     }
     assert_eq!(
         LATEST_SCHEMA_VERSION.to_string(),
@@ -910,12 +922,15 @@ async fn dbinit_equals_sum_of_all_up() {
     let populate = false;
     let mut crdb = test_setup_just_crdb(&logctx.log, populate).await;
 
-    let all_versions = read_all_schema_versions().await;
+    let all_versions = read_all_schema_versions();
 
     // Go from the first version to the latest version.
-    for version in &all_versions {
-        apply_update(log, &crdb, &version.to_string(), 1).await;
-        assert_eq!(version.to_string(), query_crdb_schema_version(&crdb).await);
+    for version in all_versions.iter_versions() {
+        apply_update(log, &crdb, version, 1).await;
+        assert_eq!(
+            version.semver().to_string(),
+            query_crdb_schema_version(&crdb).await
+        );
     }
     assert_eq!(
         LATEST_SCHEMA_VERSION.to_string(),
@@ -1190,19 +1205,22 @@ async fn validate_data_migration() {
     let mut crdb = test_setup_just_crdb(&logctx.log, populate).await;
     let client = crdb.connect().await.expect("Failed to access CRDB client");
 
-    let all_versions = read_all_schema_versions().await;
+    let all_versions = read_all_schema_versions();
     let all_checks = get_migration_checks();
 
     // Go from the first version to the latest version.
-    for version in &all_versions {
+    for version in all_versions.iter_versions() {
         // If this check has preconditions (or setup), run them.
-        let checks = all_checks.get(version);
+        let checks = all_checks.get(version.semver());
         if let Some(before) = checks.and_then(|check| check.before) {
             before(&client).await;
         }
 
-        apply_update(log, &crdb, &version.to_string(), 1).await;
-        assert_eq!(version.to_string(), query_crdb_schema_version(&crdb).await);
+        apply_update(log, &crdb, version, 1).await;
+        assert_eq!(
+            version.semver().to_string(),
+            query_crdb_schema_version(&crdb).await
+        );
 
         // If this check has postconditions (or cleanup), run them.
         if let Some(after) = checks.map(|check| check.after) {
diff --git a/schema/crdb/README.adoc b/schema/crdb/README.adoc
index 5b9c2f6a10..e017c01316 100644
--- a/schema/crdb/README.adoc
+++ b/schema/crdb/README.adoc
@@ -56,41 +56,75 @@ See RFD 319 for more discussion of the online upgrade plans.
 
 Assumptions:
 
-* The (previously) latest schema version is referred to as `OLD_VERSION`
-* Your new changes will bring the schema to a new version, `NEW_VERSION`
+* We'll call the (previously) latest schema version `OLD_VERSION`.
+* We'll call your new schema version `NEW_VERSION`.  This will always be a major
+  version bump from `OLD_VERSION`.  So if `OLD_VERSION` is 43.0.0, `NEW_VERSION`
+  should be `44.0.0`.
+* You can write a sequence of SQL statements to update a database that's
+  currently running `OLD_VERSION` to one running `NEW_VERSION`.
 
 Process:
 
-* Choose a `NEW_VERSION` number. This should almost certainly be a major
-  version bump over `OLD_VERSION`.
-* Create directory `schema/crdb/NEW_VERSION`.
-* If only one SQL statement is necessary to get from `OLD_VERSION` to
-  `NEW_VERSION`, put that statement into `schema/crdb/NEW_VERSION/up.sql`.  If
-  multiple statements are required, put each one into a separate file, naming
-  these `schema/crdb/NEW_VERSION/upN.sql` for as many `N` as you need, staring
-  with `N=1`.
-** Each file should contain _either_ one schema-modifying statement _or_ some
-   number of data-modifying statements.  You can combine multiple data-modifying
-   statements.  But you should not mix schema-modifying statements and
-   data-modifying statements in one file.  And you should not include multiple
-   schema-modifying statements in one file.
-** Beware that the entire file will be run in one transaction.  Expensive data-
-   modifying operations leading to long-running transactions are generally
-   to-be-avoided; however, there's no better way to do this today if you really
-   do need to update thousands of rows as part of the update.
-* Update `schema/crdb/dbinit.sql` to match what the database should look like
-  after your update is applied. Don't forget to update the version field of
-  `db_metadata` at the bottom of the file!
-** If necessary, do the same thing for `schema/crdb/dbwipe.sql`.
-* Update Nexus's idea of the latest schema, by updating its `SCHEMA_VERSION` to
-  `NEW_VERSION` within `nexus/db-model/src/schema.rs`.
-
-SQL Validation, via Automated Tests:
-
-* The `SCHEMA_VERSION` matches the version used in `dbinit.sql`
-* The combination of all `up.sql` files results in the same schema as
-  `dbinit.sql`
-* All `up.sql` files can be applied twice without error
+* Create a directory of SQL files describing how to update a database running
+  version `OLD_VERSION` to one running version `NEW_VERSION`:
+** Choose a unique, descriptive name for the directory.  It's suggested that
+   you stick to lowercase letters, numbers, and hyphen.  For example, if you're
+   adding a table called `widgets`, you might create a directory called
+   `create-table-widgets`.
+** Create the directory: `schema/crdb/NAME`.
+*** If only one SQL statement is necessary to get from `OLD_VERSION` to
+    `NEW_VERSION`, put that statement into `schema/crdb/NAME/up.sql`.  If
+    multiple statements are required, put each one into a separate file, naming
+    these `schema/crdb/NAME/upN.sql` for as many `N` as you need, starting with
+    `N=1`.
+*** Each file should contain _either_ one schema-modifying statement _or_ some
+    number of data-modifying statements.  You can combine multiple data-modifying
+    statements.  But you should not mix schema-modifying statements and
+    data-modifying statements in one file.  And you should not include multiple
+    schema-modifying statements in one file.
+*** Beware that the entire file will be run in one transaction.  Expensive data-
+    modifying operations leading to long-running transactions are generally
+    to-be-avoided; however, there's no better way to do this today if you really
+    do need to update thousands of rows as part of the update.
+* Update `schema/crdb/dbinit.sql`:
+** Update the SQL statements to match what the database should look like
+   after your up*.sql files are applied.
+** Update the version field of `db_metadata` at the bottom of the file.
+* Update `schema/crdb/dbwipe.sql` if needed.  (This is rare.)
+* Update `nexus/db-model/src/schema_versions.rs`:
+** Update the major number of `SCHEMA_VERSION` so that it matches `NEW_VERSION`.
+** Add a new entry to the *top* of `KNOWN_VERSIONS`.  It should be just one
+   line: `KnownVersion::new(NEW_VERSION.major, NAME)`
+* Optional: check some of this work by running `cargo nextest run -p nexus-db-model -- schema_versions`.  This is recommended because if you get
+  one of these steps wrong, these tests should be able to tell you, whereas
+  other tests might fail in much worse ways (e.g., they can hang if you've
+  updated `SCHEMA_VERSION` but not the database schema version).
+
+There are automated tests to validate many of these steps:
+
+* The `SCHEMA_VERSION` matches the version used in `dbinit.sql`.  (This catches
+  the case where you forget to update either one of these).
+* The `KNOWN_VERSIONS` are all strictly increasing semvers.  New known versions
+  must be sequential major numbers with minor and micro both being `0`.  (This
+  catches various mismerge errors: accidentally duplicating a version, putting
+  versions in the wrong order, etc.)
+* The combination of all `up*.sql` files results in the same schema as
+  `dbinit.sql`.  (This catches forgetting to update dbinit.sql, forgetting to
+  implement a schema update altogether, or a mismatch between dbinit.sql and
+  the update.)
+* All `up*.sql` files can be applied twice without error.  (This catches
+  non-idempotent update steps.)
+
+**If you've finished all this and another PR lands on "main" that chose the
+same `NEW_VERSION`:**, then your `OLD_VERSION` has changed and so _your_
+`NEW_VERSION` needs to change, too.  You'll need to:
+
+* In `nexus/db-model/src/schema_versions.rs`.
+** Make sure `SCHEMA_VERSION` reflects your new `NEW_VERSION`.
+** Make sure the `KNOWN_VERSIONS` entry that you added reflects your new
+   `NEW_VERSION` and still appears at the top of the list (logically after the
+   new version that came in from "main").
+* Update the version in `dbinit.sql` to match the new `NEW_VERSION`.
 
 === General notes
 
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index b56bec1ba9..82ddf6cd14 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -3707,7 +3707,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '45.0.0', NULL)
+    ( TRUE, NOW(), NOW(), '46.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;
diff --git a/schema/crdb/first-named-migration/up.sql b/schema/crdb/first-named-migration/up.sql
new file mode 100644
index 0000000000..3df411b9fc
--- /dev/null
+++ b/schema/crdb/first-named-migration/up.sql
@@ -0,0 +1,2 @@
+-- This is a demo migration showing how to use the new KNOWN_VERSIONS mechanism.
+SELECT TRUE;

From a655ff2a26b1c58335d4ecc17e8263df73045063 Mon Sep 17 00:00:00 2001
From: Levon Tarver <11586085+internet-diglett@users.noreply.github.com>
Date: Wed, 20 Mar 2024 13:30:18 -0500
Subject: [PATCH 135/157] Networking rpw bugfixes (#5286)

---
 clients/dpd-client/src/lib.rs                 |  40 ++++
 clients/sled-agent-client/src/lib.rs          |  46 +++++
 common/src/api/internal/shared.rs             |   8 +-
 nexus/src/app/background/nat_cleanup.rs       |  16 +-
 .../background/sync_switch_configuration.rs   | 177 +++++++++++++++---
 package-manifest.toml                         |  12 +-
 tools/dendrite_openapi_version                |   2 +-
 tools/dendrite_stub_checksums                 |   4 +-
 8 files changed, 266 insertions(+), 39 deletions(-)

diff --git a/clients/dpd-client/src/lib.rs b/clients/dpd-client/src/lib.rs
index 1186e1722f..a898c31781 100644
--- a/clients/dpd-client/src/lib.rs
+++ b/clients/dpd-client/src/lib.rs
@@ -17,6 +17,10 @@
 
 use slog::info;
 use slog::Logger;
+use types::LinkCreate;
+use types::LinkId;
+use types::LinkSettings;
+use types::PortSettings;
 
 include!(concat!(env!("OUT_DIR"), "/dpd-client.rs"));
 
@@ -787,3 +791,39 @@ impl From<u64> for MacAddr {
         }
     }
 }
+
+impl Eq for PortSettings {}
+
+impl PartialEq for PortSettings {
+    fn eq(&self, other: &Self) -> bool {
+        self.links == other.links
+    }
+}
+
+impl Eq for LinkSettings {}
+
+impl PartialEq for LinkSettings {
+    fn eq(&self, other: &Self) -> bool {
+        self.addrs == other.addrs && self.params == other.params
+    }
+}
+
+impl Eq for LinkCreate {}
+
+impl PartialEq for LinkCreate {
+    fn eq(&self, other: &Self) -> bool {
+        self.autoneg == other.autoneg
+            && self.fec == other.fec
+            && self.kr == other.kr
+            && self.lane == other.lane
+            && self.speed == other.speed
+    }
+}
+
+impl Eq for LinkId {}
+
+impl PartialEq for LinkId {
+    fn eq(&self, other: &Self) -> bool {
+        self.0 == other.0
+    }
+}
diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs
index 5303a7bafd..2e9174c16c 100644
--- a/clients/sled-agent-client/src/lib.rs
+++ b/clients/sled-agent-client/src/lib.rs
@@ -8,8 +8,10 @@ use anyhow::Context;
 use async_trait::async_trait;
 use omicron_common::api::internal::shared::NetworkInterface;
 use std::convert::TryFrom;
+use std::hash::Hash;
 use std::net::IpAddr;
 use std::net::SocketAddr;
+use types::{BgpConfig, BgpPeerConfig, PortConfigV1, RouteConfig};
 use uuid::Uuid;
 
 progenitor::generate_api!(
@@ -605,3 +607,47 @@ impl TestInterfaces for Client {
             .expect("disk_finish_transition() failed unexpectedly");
     }
 }
+
+impl Eq for BgpConfig {}
+
+impl Hash for BgpConfig {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.asn.hash(state);
+        self.originate.hash(state);
+    }
+}
+
+impl Hash for BgpPeerConfig {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.addr.hash(state);
+        self.asn.hash(state);
+        self.port.hash(state);
+        self.hold_time.hash(state);
+        self.connect_retry.hash(state);
+        self.delay_open.hash(state);
+        self.idle_hold_time.hash(state);
+        self.keepalive.hash(state);
+    }
+}
+
+impl Hash for RouteConfig {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.destination.hash(state);
+        self.nexthop.hash(state);
+    }
+}
+
+impl Eq for PortConfigV1 {}
+
+impl Hash for PortConfigV1 {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.addresses.hash(state);
+        self.autoneg.hash(state);
+        self.bgp_peers.hash(state);
+        self.port.hash(state);
+        self.routes.hash(state);
+        self.switch.hash(state);
+        self.uplink_port_fec.hash(state);
+        self.uplink_port_speed.hash(state);
+    }
+}
diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs
index bf825fd2e7..3fc1eb9879 100644
--- a/common/src/api/internal/shared.rs
+++ b/common/src/api/internal/shared.rs
@@ -266,7 +266,9 @@ pub enum ExternalPortDiscovery {
 }
 
 /// Switchport Speed options
-#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)]
+#[derive(
+    Copy, Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema, Hash,
+)]
 #[serde(rename_all = "snake_case")]
 pub enum PortSpeed {
     #[serde(alias = "0G")]
@@ -290,7 +292,9 @@ pub enum PortSpeed {
 }
 
 /// Switchport FEC options
-#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)]
+#[derive(
+    Copy, Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema, Hash,
+)]
 #[serde(rename_all = "snake_case")]
 pub enum PortFec {
     Firecode,
diff --git a/nexus/src/app/background/nat_cleanup.rs b/nexus/src/app/background/nat_cleanup.rs
index 16b1b7e357..844dbffefe 100644
--- a/nexus/src/app/background/nat_cleanup.rs
+++ b/nexus/src/app/background/nat_cleanup.rs
@@ -112,11 +112,21 @@ impl BackgroundTask for Ipv4NatGarbageCollector {
 
             let retention_threshold = Utc::now() - Duration::weeks(2);
 
-            let result = self
+            let result = match self
                 .datastore
                 .ipv4_nat_cleanup(opctx, min_gen, retention_threshold)
-                .await
-                .unwrap();
+                .await {
+                    Ok(v) => v,
+                    Err(e) => {
+                     return json!({
+                        "error":
+                            format!(
+                                "failed to perform cleanup operation: {:#}",
+                                e
+                            )
+                    });
+                    },
+                };
 
             let rv = serde_json::to_value(&result).unwrap_or_else(|error| {
                 json!({
diff --git a/nexus/src/app/background/sync_switch_configuration.rs b/nexus/src/app/background/sync_switch_configuration.rs
index e21f82b007..67d1d9fc1b 100644
--- a/nexus/src/app/background/sync_switch_configuration.rs
+++ b/nexus/src/app/background/sync_switch_configuration.rs
@@ -11,6 +11,7 @@ use crate::app::{
     },
     map_switch_zone_addrs,
 };
+use slog::o;
 
 use internal_dns::resolver::Resolver;
 use internal_dns::ServiceName;
@@ -35,6 +36,7 @@ use nexus_db_queries::{
     context::OpContext,
     db::{datastore::SwitchPortSettingsCombinedResult, DataStore},
 };
+use nexus_types::identity::Asset;
 use nexus_types::{external_api::params, identity::Resource};
 use omicron_common::OMICRON_DPD_TAG;
 use omicron_common::{
@@ -49,6 +51,7 @@ use sled_agent_client::types::{
 };
 use std::{
     collections::{hash_map::Entry, HashMap, HashSet},
+    hash::Hash,
     net::{IpAddr, Ipv4Addr},
     str::FromStr,
     sync::Arc,
@@ -211,6 +214,7 @@ impl SwitchPortSettingsManager {
     }
 }
 
+#[derive(Debug)]
 enum PortSettingsChange {
     Apply(Box<SwitchPortSettingsCombinedResult>),
     Clear,
@@ -222,7 +226,7 @@ impl BackgroundTask for SwitchPortSettingsManager {
         opctx: &'a OpContext,
     ) -> BoxFuture<'a, serde_json::Value> {
         async move {
-            let log = &opctx.log;
+            let log = opctx.log.clone();
 
             let racks = match self.datastore.rack_list_initialized(opctx, &DataPageParams::max_page()).await {
                 Ok(racks) => racks,
@@ -244,6 +248,8 @@ impl BackgroundTask for SwitchPortSettingsManager {
             // but our logic for pulling switch ports and their related configurations
             // *isn't* per-rack, so that's something we'll need to revisit in the future.
             for rack in &racks {
+                let rack_id = rack.id().to_string();
+                let log = log.new(o!("rack_id" => rack_id));
 
                 // lookup switch zones via DNS
                 // TODO https://github.com/oxidecomputer/omicron/issues/5201
@@ -261,21 +267,21 @@ impl BackgroundTask for SwitchPortSettingsManager {
 
                 // TODO https://github.com/oxidecomputer/omicron/issues/5201
                 let mappings =
-                    map_switch_zone_addrs(log, switch_zone_addresses).await;
+                    map_switch_zone_addrs(&log, switch_zone_addresses).await;
 
                 // TODO https://github.com/oxidecomputer/omicron/issues/5201
                 // build sled agent clients
-                let sled_agent_clients = build_sled_agent_clients(&mappings, log);
+                let sled_agent_clients = build_sled_agent_clients(&mappings, &log);
 
                 // TODO https://github.com/oxidecomputer/omicron/issues/5201
                 // build dpd clients
-                let dpd_clients = build_dpd_clients(&mappings, log);
+                let dpd_clients = build_dpd_clients(&mappings, &log);
 
                 // TODO https://github.com/oxidecomputer/omicron/issues/5201
                 // build mgd clients
-                let mgd_clients = build_mgd_clients(mappings, log);
+                let mgd_clients = build_mgd_clients(mappings, &log);
 
-                let port_list = match self.switch_ports(opctx, log).await {
+                let port_list = match self.switch_ports(opctx, &log).await {
                     Ok(value) => value,
                     Err(e) => {
                         error!(log, "failed to generate switchports for rack"; "error" => %e);
@@ -287,7 +293,7 @@ impl BackgroundTask for SwitchPortSettingsManager {
                 // calculate and apply switch port changes
                 //
 
-                let changes = match self.changes(port_list, opctx, log).await {
+                let changes = match self.changes(port_list, opctx, &log).await {
                     Ok(value) => value,
                     Err(e) => {
                         error!(log, "failed to generate changeset for switchport settings"; "error" => %e);
@@ -295,7 +301,8 @@ impl BackgroundTask for SwitchPortSettingsManager {
                     },
                 };
 
-                apply_switch_port_changes(&dpd_clients, &changes, log).await;
+                info!(&log, "applying switch port config changes"; "changes" => ?changes);
+                apply_switch_port_changes(&dpd_clients, &changes, &log).await;
 
                 //
                 // calculate and apply routing changes
@@ -303,7 +310,7 @@ impl BackgroundTask for SwitchPortSettingsManager {
 
                 // get the static routes on each switch
                 let current_static_routes =
-                    static_routes_on_switch(&mgd_clients, log).await;
+                    static_routes_on_switch(&mgd_clients, &log).await;
                 info!(&log, "retrieved existing routes"; "routes" => ?current_static_routes);
 
                 // generate the complete set of static routes that should be on a given switch
@@ -315,7 +322,7 @@ impl BackgroundTask for SwitchPortSettingsManager {
                 let routes_to_add = static_routes_to_add(
                     &desired_static_routes,
                     &current_static_routes,
-                    log,
+                    &log,
                 );
                 info!(&log, "calculated static routes to add"; "routes" => ?routes_to_add);
 
@@ -327,21 +334,26 @@ impl BackgroundTask for SwitchPortSettingsManager {
 
                 // delete the unneeded routes first, just in case there is a conflicting route for
                 // one we need to add
-                info!(&log, "deleting static routes"; "routes" => ?routes_to_del);
-                delete_static_routes(&mgd_clients, routes_to_del, log).await;
+                if !routes_to_del.is_empty() {
+                    info!(&log, "deleting static routes"; "routes" => ?routes_to_del);
+                    delete_static_routes(&mgd_clients, routes_to_del, &log).await;
+                }
 
                 // add the new routes
-                info!(&log, "adding static routes"; "routes" => ?routes_to_add);
-                add_static_routes(&mgd_clients, routes_to_add, log).await;
+                if !routes_to_add.is_empty() {
+                    info!(&log, "adding static routes"; "routes" => ?routes_to_add);
+                    add_static_routes(&mgd_clients, routes_to_add, &log).await;
+                }
 
 
                 //
                 // calculate and apply loopback address changes
                 //
 
-                match self.db_loopback_addresses(opctx, log).await {
+                info!(&log, "checking for changes to loopback addresses");
+                match self.db_loopback_addresses(opctx, &log).await {
                     Ok(desired_loopback_addresses) => {
-                        let current_loopback_addresses = switch_loopback_addresses(&dpd_clients, log).await;
+                        let current_loopback_addresses = switch_loopback_addresses(&dpd_clients, &log).await;
 
                         let loopbacks_to_add: Vec<(SwitchLocation, IpAddr)> = desired_loopback_addresses
                             .difference(&current_loopback_addresses)
@@ -352,8 +364,15 @@ impl BackgroundTask for SwitchPortSettingsManager {
                             .map(|i| (i.0, i.1))
                             .collect();
 
-                        delete_loopback_addresses_from_switch(&loopbacks_to_del, &dpd_clients, log).await;
-                        add_loopback_addresses_to_switch(&loopbacks_to_add, dpd_clients, log).await;
+                        if !loopbacks_to_del.is_empty() {
+                            info!(&log, "deleting loopback addresses"; "addresses" => ?loopbacks_to_del);
+                            delete_loopback_addresses_from_switch(&loopbacks_to_del, &dpd_clients, &log).await;
+                        }
+
+                        if !loopbacks_to_add.is_empty() {
+                            info!(&log, "adding loopback addresses"; "addresses" => ?loopbacks_to_add);
+                            add_loopback_addresses_to_switch(&loopbacks_to_add, dpd_clients, &log).await;
+                        }
                     },
                     Err(e) => {
                         error!(
@@ -769,7 +788,9 @@ impl BackgroundTask for SwitchPortSettingsManager {
                         error!(log, "no blocks assigned to infra lot");
                         continue;
                     },
-                };
+                }
+                ;
+
 
                 let mut desired_config = EarlyNetworkConfig {
                     generation: 0,
@@ -786,7 +807,7 @@ impl BackgroundTask for SwitchPortSettingsManager {
                     },
                 };
 
-                // should_update is a boolean value that determines whether or not we need to
+                // bootstore_needs_update is a boolean value that determines whether or not we need to
                 // increment the bootstore version and push a new config to the sled agents.
                 //
                 // * If the config we've built from the switchport configuration information is
@@ -804,15 +825,32 @@ impl BackgroundTask for SwitchPortSettingsManager {
                     Ok(Some(BootstoreConfig { data, .. })) => {
                         match serde_json::from_value::<EarlyNetworkConfig>(data.clone()) {
                             Ok(config) => {
-                                if config.body.ntp_servers != desired_config.body.ntp_servers {
+                                let current_ntp_servers: HashSet<String> = config.body.ntp_servers.clone().into_iter().collect();
+                                let desired_ntp_servers: HashSet<String> = desired_config.body.ntp_servers.clone().into_iter().collect();
+
+                                let rnc_differs = match (config.body.rack_network_config.clone(), desired_config.body.rack_network_config.clone()) {
+                                    (Some(current_rnc), Some(desired_rnc)) => {
+                                        !hashset_eq(current_rnc.bgp.clone(), desired_rnc.bgp.clone()) ||
+                                        !hashset_eq(current_rnc.ports.clone(), desired_rnc.ports.clone()) ||
+                                        current_rnc.rack_subnet != desired_rnc.rack_subnet ||
+                                        current_rnc.infra_ip_first != desired_rnc.infra_ip_first ||
+                                        current_rnc.infra_ip_last != desired_rnc.infra_ip_last
+                                    },
+                                    (None, Some(_)) => true,
+                                    _ => {
+                                        todo!("error")
+                                    }
+                                };
+
+                                if current_ntp_servers != desired_ntp_servers {
                                     info!(
                                         log,
                                         "ntp servers have changed";
-                                        "old" => ?config.body.ntp_servers,
-                                        "new" => ?desired_config.body.ntp_servers,
+                                        "old" => ?current_ntp_servers,
+                                        "new" => ?desired_ntp_servers,
                                     );
                                     true
-                                } else if config.body.rack_network_config != desired_config.body.rack_network_config {
+                                } else if rnc_differs {
                                     info!(
                                         log,
                                         "rack network config has changed";
@@ -855,6 +893,28 @@ impl BackgroundTask for SwitchPortSettingsManager {
                     },
                 };
 
+                // The following code is designed to give us the following
+                // properties
+                // * We only push updates to the bootstore (sled-agents) if
+                //   configuration on our side (nexus) has relevant changes.
+                // * If the RPW encounters a critical error or crashes at any
+                //   point of the operation, it will retry the configuration
+                //   again during the next run
+                // * We are able to accomplish the above without inspecting
+                //   the bootstore on the sled-agents
+                //
+                // For example, in the event that we crash after pushing to
+                // the sled-agents successfully, but before writing the
+                // results to the db
+                // 1. RPW will restart
+                // 2. RPW will build a new network config
+                // 3. RPW will compare against the last version stored in the db
+                // 4. RPW will decide to apply the config (again)
+                // 5. RPW will bump the version (again)
+                // 6. RPW will send a new bootstore update to the agents (with
+                //    the same info as last time, but with a new version)
+                // 7. RPW will record the update in the db
+                // 8. We are now back on the happy path
                 if bootstore_needs_update {
                     let generation = match self.datastore
                         .bump_bootstore_generation(opctx, NETWORK_KEY.into())
@@ -878,7 +938,7 @@ impl BackgroundTask for SwitchPortSettingsManager {
                         "config" => ?desired_config,
                     );
 
-                    // spush the updates to both scrimlets
+                    // push the updates to both scrimlets
                     // if both scrimlets are down, bootstore updates aren't happening anyway
                     let mut one_succeeded = false;
                     for (location, client) in &sled_agent_clients {
@@ -923,6 +983,15 @@ impl BackgroundTask for SwitchPortSettingsManager {
     }
 }
 
+fn hashset_eq<T>(left: Vec<T>, right: Vec<T>) -> bool
+where
+    T: Hash + Eq,
+{
+    let left = left.into_iter().collect::<HashSet<T>>();
+    let right = right.into_iter().collect::<HashSet<T>>();
+    left == right
+}
+
 async fn add_loopback_addresses_to_switch(
     loopbacks_to_add: &[(SwitchLocation, IpAddr)],
     dpd_clients: HashMap<SwitchLocation, dpd_client::Client>,
@@ -1112,6 +1181,13 @@ fn static_routes_to_del(
             continue;
         };
     }
+
+    // filter out switches with no routes to remove
+    let routes_to_del = routes_to_del
+        .into_iter()
+        .filter(|(_location, request)| !request.routes.list.is_empty())
+        .collect();
+
     routes_to_del
 }
 
@@ -1158,6 +1234,13 @@ fn static_routes_to_add(
             },
         );
     }
+
+    // filter out switches with no routes to add
+    let routes_to_add = routes_to_add
+        .into_iter()
+        .filter(|(_location, request)| !request.routes.list.is_empty())
+        .collect();
+
     routes_to_add
 }
 
@@ -1246,6 +1329,28 @@ async fn apply_switch_port_changes(
             }
         };
 
+        let config_on_switch =
+            match client.port_settings_get(&dpd_port_id, DPD_TAG).await {
+                Ok(v) => v,
+                Err(e) => {
+                    error!(
+                        log,
+                        "failed to retrieve port setttings from switch";
+                        "switch_port_id" => ?port_name,
+                        "switch_location" => ?location,
+                        "error" => format!("{:#}", e)
+                    );
+                    continue;
+                }
+            };
+
+        info!(
+            log,
+            "retrieved port settings from switch";
+            "switch_port_id" => ?port_name,
+            "settings" => ?config_on_switch,
+        );
+
         match change {
             PortSettingsChange::Apply(settings) => {
                 let dpd_port_settings = match api_to_dpd_port_settings(
@@ -1265,6 +1370,17 @@ async fn apply_switch_port_changes(
                     }
                 };
 
+                if config_on_switch.into_inner() == dpd_port_settings {
+                    info!(
+                        &log,
+                        "port settings up to date, skipping";
+                        "switch_port_id" => ?port_name,
+                        "switch_location" => ?location,
+                        "switch_port_settings_id" => ?settings.settings.id(),
+                    );
+                    continue;
+                }
+
                 // apply settings via dpd client
                 info!(
                     &log,
@@ -1301,6 +1417,17 @@ async fn apply_switch_port_changes(
                     "switch_location" => ?location,
                     "port_id" => ?dpd_port_id,
                 );
+
+                if config_on_switch.into_inner().links.is_empty() {
+                    info!(
+                        &log,
+                        "port settings up to date, skipping";
+                        "switch_port_id" => ?port_name,
+                        "switch_location" => ?location,
+                    );
+                    continue;
+                }
+
                 match client.port_settings_clear(&dpd_port_id, DPD_TAG).await {
                     Ok(_) => {}
                     Err(e) => {
diff --git a/package-manifest.toml b/package-manifest.toml
index e88f135985..0cc2e4939c 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -595,8 +595,8 @@ only_for_targets.image = "standard"
 # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "41ddeab9d43d90a51e6fc1c236dc9982fc76f922"
-source.sha256 = "8ebb889a555ce59cb0373a1ec9595536e015c951f6fc4d89308b4e3f09c83b20"
+source.commit = "b128500231b916802a9436dd438742424aa5c6ce"
+source.sha256 = "47e0b279cf5babb199ea4d7d1654ccef6f463eeae7cef24e8138658c57affb4a"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -620,8 +620,8 @@ only_for_targets.image = "standard"
 # 2. Copy the output zone image from dendrite/out to omicron/out
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "41ddeab9d43d90a51e6fc1c236dc9982fc76f922"
-source.sha256 = "3e8aa5483d22316e1fd629c77277190dafa875938a9ab3900e92a210c5e91e91"
+source.commit = "b128500231b916802a9436dd438742424aa5c6ce"
+source.sha256 = "3e6e785a3cfbf12dd9752c3bb026e4e3e75aa9d4433ba65045c7396d31e970a9"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -638,8 +638,8 @@ only_for_targets.image = "standard"
 # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "41ddeab9d43d90a51e6fc1c236dc9982fc76f922"
-source.sha256 = "5e5f2831f3c46253828ea237f701f1fa174061ab0bf73c200d31d09e94890ae7"
+source.commit = "b128500231b916802a9436dd438742424aa5c6ce"
+source.sha256 = "64cb9c62d3516c613ea70c36dff241cba2e1138e25a82a6367c40da262ca55ee"
 output.type = "zone"
 output.intermediate_only = true
 
diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version
index bbc2110a0a..3d5f21ffe0 100644
--- a/tools/dendrite_openapi_version
+++ b/tools/dendrite_openapi_version
@@ -1,2 +1,2 @@
-COMMIT="41ddeab9d43d90a51e6fc1c236dc9982fc76f922"
+COMMIT="b128500231b916802a9436dd438742424aa5c6ce"
 SHA2="50eff6d9f986b7b1af5970d11d8d01b812de37269731c6c691a244b3fdae82ae"
diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums
index 86cf1a56ec..3bca6cd174 100644
--- a/tools/dendrite_stub_checksums
+++ b/tools/dendrite_stub_checksums
@@ -1,3 +1,3 @@
-CIDL_SHA256_ILLUMOS="8ebb889a555ce59cb0373a1ec9595536e015c951f6fc4d89308b4e3f09c83b20"
-CIDL_SHA256_LINUX_DPD="f753444cae478cdedcde743a20a9df5965ed28cddab0f9632f3c263c66cd6397"
+CIDL_SHA256_ILLUMOS="47e0b279cf5babb199ea4d7d1654ccef6f463eeae7cef24e8138658c57affb4a"
+CIDL_SHA256_LINUX_DPD="11e464a38fa0858c3d896e82d7ee3123aee5f0cf4e3c2a029a0dd7cfd54d3adf"
 CIDL_SHA256_LINUX_SWADM="66eab497b955751d0704c3cd97ac5c1ed373aa656fc37ccba86ae9900b5ae96d"

From 0d5d5c0b75c4fa754a3d454ca4d2d36aa926af56 Mon Sep 17 00:00:00 2001
From: Laura Abbott <labbott@users.noreply.github.com>
Date: Thu, 21 Mar 2024 10:56:46 -0400
Subject: [PATCH 136/157] SP v1.0.12 (#5300)

---
 tools/hubris_checksums | 16 ++++++++--------
 tools/hubris_version   |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tools/hubris_checksums b/tools/hubris_checksums
index 6f47ff9f93..fcb9151be5 100644
--- a/tools/hubris_checksums
+++ b/tools/hubris_checksums
@@ -1,8 +1,8 @@
-4ad8e4bb4e96bafe988c4b26a0e39ff147d7ba3fd0fd04460dad20c9bdd4be93  build-gimlet-c-image-default-v1.0.11.zip
-171ee7fbff449447602b84c0e4d6f25c732a00485aba929add5bf5fd6209c8cb  build-gimlet-d-image-default-v1.0.11.zip
-0b48a4487d4683b09522aea7f66ac0be2977bcc0c0c93d1c6806c2dd3f58904f  build-gimlet-e-image-default-v1.0.11.zip
-3dcac446c7451ae4cbdd1ddf4081e1d9e9461d0935f67535bd8161fb9e84625d  build-gimlet-f-image-default-v1.0.11.zip
-060757892627a8c2bddf58621778e2bef825c3cc34906a81866486c413f917a3  build-psc-b-image-default-v1.0.11.zip
-fd712fe842a91b2fa60e680d4433d4951fd105c0fa81c7441a01f9208da8621d  build-psc-c-image-default-v1.0.11.zip
-9fe7856b0e92d03cd4ea2268d327af2479ae0a53c07cb820946b6b15c215d2a4  build-sidecar-b-image-default-v1.0.11.zip
-41ff08e08384f5015fd6a3d2ad07d48f159333087e89e88d58b2bfc4b6da9ef7  build-sidecar-c-image-default-v1.0.11.zip
+213f5e6fea9fea6356afc05e7769872c8d20b2c4c4ccd5841e161b785a3a858e  build-gimlet-c-image-default-v1.0.12.zip
+56185bc02a5b104106469da925edf7d9c80cfef6a75e1194231ba383fe68d765  build-gimlet-d-image-default-v1.0.12.zip
+36f8d2670422be87b55cab701655ce868ce0c6a0dae9293adcbb17f47bd021c6  build-gimlet-e-image-default-v1.0.12.zip
+98c9651da3317050c0c016c0936865c8aa409745db60057f39e49b702a8de432  build-gimlet-f-image-default-v1.0.12.zip
+16898114126d46ae5723823abb6b31da465f72d884e995b3b3360269fff1dde0  build-psc-b-image-default-v1.0.12.zip
+f4c0d46f38f0747b781efa1a306d223558cde89f60ed10edc582550b6f2c1ef1  build-psc-c-image-default-v1.0.12.zip
+dede524c6bd38d27b69011b28950a5e335591265ef75306fb87ce5b402e5ae6b  build-sidecar-b-image-default-v1.0.12.zip
+b114eb25ff7f48f1eceaafbbdaeba071d2f99b19335e793ab03f4f01f6d22c97  build-sidecar-c-image-default-v1.0.12.zip
diff --git a/tools/hubris_version b/tools/hubris_version
index 8bc373efcd..e6be0843de 100644
--- a/tools/hubris_version
+++ b/tools/hubris_version
@@ -1 +1 @@
-TAGS=(gimlet-v1.0.11 psc-v1.0.11 sidecar-v1.0.11)
+TAGS=(gimlet-v1.0.12 psc-v1.0.12 sidecar-v1.0.12)

From 8cdfa1c2af1b34d2733c955b06c34bb7db6faa1b Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Thu, 21 Mar 2024 14:42:13 -0400
Subject: [PATCH 137/157] Prune `inv_zpool` when pruning collections (#5301)

Noticed this while investigating #5299; I am unsure whether this will
actually resolve #5299 though.

I made this same mistake the first time I added something new to
inventory; @davepacheco and I discussed the issues with trying to test
for this at
https://github.com/oxidecomputer/omicron/pull/4496#discussion_r1393108135.
I'm not sure anything has materially changed there, but maybe testing
this warrants a closer look.
---
 nexus/db-queries/src/db/datastore/inventory.rs | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs
index 67d5684bbf..1a6d59337c 100644
--- a/nexus/db-queries/src/db/datastore/inventory.rs
+++ b/nexus/db-queries/src/db/datastore/inventory.rs
@@ -1096,6 +1096,7 @@ impl DataStore {
             nsled_agent_zones,
             nzones,
             nnics,
+            nzpools,
             nerrors,
         ) = conn
             .transaction_async(|conn| async move {
@@ -1206,6 +1207,16 @@ impl DataStore {
                     .await?
                 };
 
+                let nzpools = {
+                    use db::schema::inv_zpool::dsl;
+                    diesel::delete(
+                        dsl::inv_zpool
+                            .filter(dsl::inv_collection_id.eq(collection_id)),
+                    )
+                    .execute_async(&conn)
+                    .await?
+                };
+
                 // Remove rows for errors encountered.
                 let nerrors = {
                     use db::schema::inv_collection_error::dsl;
@@ -1228,6 +1239,7 @@ impl DataStore {
                     nsled_agent_zones,
                     nzones,
                     nnics,
+                    nzpools,
                     nerrors,
                 ))
             })
@@ -1251,6 +1263,7 @@ impl DataStore {
             "nsled_agent_zones" => nsled_agent_zones,
             "nzones" => nzones,
             "nnics" => nnics,
+            "nzpools" => nzpools,
             "nerrors" => nerrors,
         );
 

From 9c163401a6bd53acc94b6485d4d8a24acb9fa24b Mon Sep 17 00:00:00 2001
From: Ryan Goodfellow <ryan.goodfellow@oxide.computer>
Date: Thu, 21 Mar 2024 12:46:44 -0700
Subject: [PATCH 138/157] reduce a4x2 timeout to 30 minutes (#5295)

---
 .github/buildomat/jobs/a4x2-deploy.sh | 4 +++-
 end-to-end-tests/src/bin/commtest.rs  | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/buildomat/jobs/a4x2-deploy.sh b/.github/buildomat/jobs/a4x2-deploy.sh
index 0e37d4ae25..dfc9191611 100755
--- a/.github/buildomat/jobs/a4x2-deploy.sh
+++ b/.github/buildomat/jobs/a4x2-deploy.sh
@@ -198,7 +198,9 @@ pfexec route add 198.51.100.0/24 $customer_edge_addr
 # given sled.
 cp /input/a4x2/out/commtest .
 chmod +x commtest
-NO_COLOR=1 pfexec ./commtest http://198.51.100.23 run \
+NO_COLOR=1 pfexec ./commtest \
+    --api-timeout 30m \
+    http://198.51.100.23 run \
     --ip-pool-begin 198.51.100.40 \
     --ip-pool-end 198.51.100.70 \
     --icmp-loss-tolerance 500 \
diff --git a/end-to-end-tests/src/bin/commtest.rs b/end-to-end-tests/src/bin/commtest.rs
index 27ca4633ce..05e15faafc 100644
--- a/end-to-end-tests/src/bin/commtest.rs
+++ b/end-to-end-tests/src/bin/commtest.rs
@@ -102,7 +102,8 @@ async fn wait_until_oxide_api_is_available(cli: &Cli) -> Result<()> {
             > Into::<Duration>::into(cli.api_timeout)
         {
             return Err(anyhow!(
-                "One hour deadline for system startup exceeded"
+                "{} deadline for system startup exceeded",
+                cli.api_timeout
             ));
         }
         println!("no api response yet, wating 3s ...");

From 1d5738df4710c801749bb79d07bc9a590e81b17f Mon Sep 17 00:00:00 2001
From: Levon Tarver <11586085+internet-diglett@users.noreply.github.com>
Date: Thu, 21 Mar 2024 16:11:24 -0500
Subject: [PATCH 139/157] BUGFIX: Bootstore BGP configuration (#5297)

* Implement a more straightforward mechanism for generating bootstore
bgp peer config from db data
* Filter bootstore interface peer config data by switch and port number
---
 Cargo.lock                                    |  1 +
 nexus/db-model/src/bgp.rs                     | 21 ++++++-
 nexus/db-model/src/schema.rs                  | 14 +++++
 nexus/db-model/src/schema_versions.rs         |  3 +-
 nexus/db-queries/src/db/datastore/bgp.rs      | 28 +++++++--
 .../background/sync_switch_configuration.rs   | 62 +++++++++++--------
 .../add-view-for-bgp-peer-configs/up01.sql    | 16 +++++
 .../add-view-for-bgp-peer-configs/up02.sql    |  2 +
 .../add-view-for-bgp-peer-configs/up03.sql    |  1 +
 schema/crdb/dbinit.sql                        | 24 ++++++-
 sled-agent/Cargo.toml                         |  1 +
 sled-agent/src/hardware_monitor.rs            |  2 +-
 sled-agent/src/services.rs                    |  2 +-
 13 files changed, 138 insertions(+), 39 deletions(-)
 create mode 100644 schema/crdb/add-view-for-bgp-peer-configs/up01.sql
 create mode 100644 schema/crdb/add-view-for-bgp-peer-configs/up02.sql
 create mode 100644 schema/crdb/add-view-for-bgp-peer-configs/up03.sql

diff --git a/Cargo.lock b/Cargo.lock
index 9ca0599fb3..b94ef71980 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5538,6 +5538,7 @@ dependencies = [
  "slog",
  "slog-async",
  "slog-dtrace",
+ "slog-error-chain",
  "slog-term",
  "smf",
  "static_assertions",
diff --git a/nexus/db-model/src/bgp.rs b/nexus/db-model/src/bgp.rs
index cc9ebfb4f5..02201f60e4 100644
--- a/nexus/db-model/src/bgp.rs
+++ b/nexus/db-model/src/bgp.rs
@@ -2,14 +2,15 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use crate::schema::{bgp_announce_set, bgp_announcement, bgp_config};
+use crate::schema::{
+    bgp_announce_set, bgp_announcement, bgp_config, bgp_peer_view,
+};
 use crate::SqlU32;
 use db_macros::Resource;
 use ipnetwork::IpNetwork;
 use nexus_types::external_api::params;
 use nexus_types::identity::Resource;
-use omicron_common::api::external;
-use omicron_common::api::external::IdentityMetadataCreateParams;
+use omicron_common::api::external::{self, IdentityMetadataCreateParams};
 use serde::{Deserialize, Serialize};
 use uuid::Uuid;
 
@@ -117,3 +118,17 @@ impl Into<external::BgpAnnouncement> for BgpAnnouncement {
         }
     }
 }
+
+#[derive(Queryable, Selectable, Clone, Debug, Serialize, Deserialize)]
+#[diesel(table_name = bgp_peer_view)]
+pub struct BgpPeerView {
+    pub switch_location: String,
+    pub port_name: String,
+    pub addr: IpNetwork,
+    pub asn: SqlU32,
+    pub connect_retry: SqlU32,
+    pub delay_open: SqlU32,
+    pub hold_time: SqlU32,
+    pub idle_hold_time: SqlU32,
+    pub keepalive: SqlU32,
+}
diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs
index 6becea4c2a..c533b426bd 100644
--- a/nexus/db-model/src/schema.rs
+++ b/nexus/db-model/src/schema.rs
@@ -226,6 +226,20 @@ table! {
     }
 }
 
+table! {
+    bgp_peer_view (switch_location, port_name) {
+        switch_location -> Text,
+        port_name -> Text,
+        addr -> Inet,
+        asn -> Int8,
+        connect_retry -> Int8,
+        delay_open -> Int8,
+        hold_time -> Int8,
+        idle_hold_time -> Int8,
+        keepalive -> Int8,
+    }
+}
+
 table! {
     bgp_announce_set (id) {
         id -> Uuid,
diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs
index c267ce65d8..e35cc3c38a 100644
--- a/nexus/db-model/src/schema_versions.rs
+++ b/nexus/db-model/src/schema_versions.rs
@@ -17,7 +17,7 @@ use std::collections::BTreeMap;
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(46, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(47, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy<Vec<KnownVersion>> = Lazy::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(47, "add-view-for-bgp-peer-configs"),
         KnownVersion::new(46, "first-named-migration"),
         // The first many schema versions only vary by major or patch number and
         // their path is predictable based on the version number.  (This was
diff --git a/nexus/db-queries/src/db/datastore/bgp.rs b/nexus/db-queries/src/db/datastore/bgp.rs
index eb8285e4dc..bc103ecff9 100644
--- a/nexus/db-queries/src/db/datastore/bgp.rs
+++ b/nexus/db-queries/src/db/datastore/bgp.rs
@@ -5,22 +5,21 @@
 use super::DataStore;
 use crate::context::OpContext;
 use crate::db;
-use crate::db::error::public_error_from_diesel;
-use crate::db::error::ErrorHandler;
-use crate::db::model::Name;
-use crate::db::model::{BgpAnnounceSet, BgpAnnouncement, BgpConfig};
+use crate::db::error::{public_error_from_diesel, ErrorHandler};
+use crate::db::model::{BgpAnnounceSet, BgpAnnouncement, BgpConfig, Name};
 use crate::db::pagination::paginated;
 use crate::transaction_retry::OptionalError;
 use async_bb8_diesel::AsyncRunQueryDsl;
 use chrono::Utc;
 use diesel::{ExpressionMethods, QueryDsl, SelectableHelper};
 use ipnetwork::IpNetwork;
+use nexus_db_model::BgpPeerView;
 use nexus_types::external_api::params;
 use nexus_types::identity::Resource;
 use omicron_common::api::external::http_pagination::PaginatedBy;
 use omicron_common::api::external::{
     CreateResult, DeleteResult, Error, ListResultVec, LookupResult, NameOrId,
-    ResourceType,
+    ResourceType, SwitchLocation,
 };
 use ref_cast::RefCast;
 use uuid::Uuid;
@@ -474,4 +473,23 @@ impl DataStore {
                 }
             })
     }
+
+    pub async fn bgp_peer_configs(
+        &self,
+        opctx: &OpContext,
+        switch: SwitchLocation,
+        port: String,
+    ) -> ListResultVec<BgpPeerView> {
+        use db::schema::bgp_peer_view::dsl;
+
+        let results = dsl::bgp_peer_view
+            .filter(dsl::switch_location.eq(switch.to_string()))
+            .filter(dsl::port_name.eq(port))
+            .select(BgpPeerView::as_select())
+            .load_async(&*self.pool_connection_authorized(opctx).await?)
+            .await
+            .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
+
+        Ok(results)
+    }
 }
diff --git a/nexus/src/app/background/sync_switch_configuration.rs b/nexus/src/app/background/sync_switch_configuration.rs
index 67d1d9fc1b..7fa2b27966 100644
--- a/nexus/src/app/background/sync_switch_configuration.rs
+++ b/nexus/src/app/background/sync_switch_configuration.rs
@@ -18,8 +18,7 @@ use internal_dns::ServiceName;
 use ipnetwork::IpNetwork;
 use nexus_db_model::{
     AddressLotBlock, BgpConfig, BootstoreConfig, LoopbackAddress,
-    SwitchLinkFec, SwitchLinkSpeed, SwitchPortBgpPeerConfig, INFRA_LOT,
-    NETWORK_KEY,
+    SwitchLinkFec, SwitchLinkSpeed, INFRA_LOT, NETWORK_KEY,
 };
 use uuid::Uuid;
 
@@ -301,7 +300,6 @@ impl BackgroundTask for SwitchPortSettingsManager {
                     },
                 };
 
-                info!(&log, "applying switch port config changes"; "changes" => ?changes);
                 apply_switch_port_changes(&dpd_clients, &changes, &log).await;
 
                 //
@@ -436,8 +434,6 @@ impl BackgroundTask for SwitchPortSettingsManager {
                 // Prefixes are associated to BgpConfig via the config id
                 let mut bgp_announce_prefixes: HashMap<Uuid, Vec<Prefix4>> = HashMap::new();
 
-                let mut bootstore_bgp_peer_info: Vec<(SwitchPortBgpPeerConfig, u32, Ipv4Addr)> = vec![];
-
                 for (location, port, change) in &changes {
                     let PortSettingsChange::Apply(settings) = change else {
                         continue;
@@ -552,15 +548,6 @@ impl BackgroundTask for SwitchPortSettingsManager {
                             passive: false,
                         };
 
-                        // add it to data for the bootstore
-                        // only ipv4 is supported now
-                        match peer.addr {
-                            ipnetwork::IpNetwork::V4(addr) => {
-                                bootstore_bgp_peer_info.push((peer.clone(), bgp_config.asn.0, addr.ip()));
-                            },
-                            ipnetwork::IpNetwork::V6(_) => continue, //TODO v6
-                        };
-
                         // update the stored vec if it exists, create a new on if it doesn't exist
                         match peers.entry(port.port_name.clone()) {
                             Entry::Occupied(mut occupied_entry) => {
@@ -690,7 +677,7 @@ impl BackgroundTask for SwitchPortSettingsManager {
                 };
 
                 // TODO: @rcgoodfellow is this correct? Do we place the BgpConfig for both switches in a single Vec to send to the bootstore?
-                let bgp: Vec<SledBgpConfig> = switch_bgp_config.iter().map(|(_location, (_id, config))| {
+                let mut bgp: Vec<SledBgpConfig> = switch_bgp_config.iter().map(|(_location, (_id, config))| {
                     let announcements: Vec<Ipv4Network> = bgp_announce_prefixes
                         .get(&config.bgp_announce_set_id)
                         .expect("bgp config is present but announce set is not populated")
@@ -707,6 +694,8 @@ impl BackgroundTask for SwitchPortSettingsManager {
                     }
                 }).collect();
 
+                bgp.dedup();
+
                 let mut ports: Vec<PortConfigV1> = vec![];
 
                 for (location, port, change) in &changes {
@@ -714,6 +703,20 @@ impl BackgroundTask for SwitchPortSettingsManager {
                         continue;
                     };
 
+                    let peer_configs = match self.datastore.bgp_peer_configs(opctx, *location, port.port_name.clone()).await {
+                        Ok(v) => v,
+                        Err(e) => {
+                            error!(
+                                log,
+                                "failed to fetch bgp peer config for switch port";
+                                "switch_location" => ?location,
+                                "port" => &port.port_name,
+                                "error" => %e,
+                            );
+                            continue;
+                        },
+                    };
+
                     let port_config = PortConfigV1 {
                         addresses: info.addresses.iter().map(|a| a.address).collect(),
                         autoneg: info
@@ -721,19 +724,24 @@ impl BackgroundTask for SwitchPortSettingsManager {
                             .get(0) //TODO breakout support
                             .map(|l| l.autoneg)
                             .unwrap_or(false),
-                        bgp_peers: bootstore_bgp_peer_info
-                            .iter()
-                            .map(|(p, asn, addr)| SledBgpPeerConfig {
-                                addr: *addr,
-                                asn: *asn,
-                                port: port.port_name.clone(),
-                                hold_time: Some(p.hold_time.0.into()),
-                                connect_retry: Some(p.connect_retry.0.into()),
-                                delay_open: Some(p.delay_open.0.into()),
-                                idle_hold_time: Some(p.idle_hold_time.0.into()),
-                                keepalive: Some(p.keepalive.0.into()),
+                        bgp_peers: peer_configs.into_iter()
+                            // filter maps are cool
+                            .filter_map(|c| match c.addr.ip() {
+                                IpAddr::V4(addr) => Some((c, addr)),
+                                IpAddr::V6(_) => None,
                             })
-                            .collect(),
+                            .map(|(c, addr)| {
+                                SledBgpPeerConfig {
+                                    asn: *c.asn,
+                                    port: c.port_name,
+                                    addr,
+                                    hold_time: Some(c.hold_time.0.into()),
+                                    idle_hold_time: Some(c.idle_hold_time.0.into()),
+                                    delay_open: Some(c.delay_open.0.into()),
+                                    connect_retry: Some(c.connect_retry.0.into()),
+                                    keepalive: Some(c.keepalive.0.into()),
+                                }
+                        }).collect(),
                         port: port.port_name.clone(),
                         routes: info
                             .routes
diff --git a/schema/crdb/add-view-for-bgp-peer-configs/up01.sql b/schema/crdb/add-view-for-bgp-peer-configs/up01.sql
new file mode 100644
index 0000000000..495d6ea433
--- /dev/null
+++ b/schema/crdb/add-view-for-bgp-peer-configs/up01.sql
@@ -0,0 +1,16 @@
+CREATE VIEW IF NOT EXISTS omicron.public.bgp_peer_view
+AS
+SELECT
+ sp.switch_location,
+ sp.port_name,
+ bpc.addr,
+ bpc.hold_time,
+ bpc.idle_hold_time,
+ bpc.delay_open,
+ bpc.connect_retry,
+ bpc.keepalive,
+ bc.asn
+FROM omicron.public.switch_port sp
+JOIN omicron.public.switch_port_settings_bgp_peer_config bpc
+ON sp.port_settings_id = bpc.port_settings_id
+JOIN omicron.public.bgp_config bc ON bc.id = bpc.bgp_config_id;
diff --git a/schema/crdb/add-view-for-bgp-peer-configs/up02.sql b/schema/crdb/add-view-for-bgp-peer-configs/up02.sql
new file mode 100644
index 0000000000..d9aa9354a0
--- /dev/null
+++ b/schema/crdb/add-view-for-bgp-peer-configs/up02.sql
@@ -0,0 +1,2 @@
+CREATE INDEX IF NOT EXISTS switch_port_id_and_name
+ON omicron.public.switch_port (port_settings_id, port_name) STORING (switch_location);
diff --git a/schema/crdb/add-view-for-bgp-peer-configs/up03.sql b/schema/crdb/add-view-for-bgp-peer-configs/up03.sql
new file mode 100644
index 0000000000..797cf8f1e4
--- /dev/null
+++ b/schema/crdb/add-view-for-bgp-peer-configs/up03.sql
@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS switch_port_name ON omicron.public.switch_port (port_name);
diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql
index 82ddf6cd14..ec6a7c192f 100644
--- a/schema/crdb/dbinit.sql
+++ b/schema/crdb/dbinit.sql
@@ -3673,6 +3673,28 @@ CREATE TABLE IF NOT EXISTS omicron.public.bootstore_config (
 
 CREATE INDEX IF NOT EXISTS address_lot_names ON omicron.public.address_lot(name);
 
+CREATE VIEW IF NOT EXISTS omicron.public.bgp_peer_view
+AS
+SELECT
+ sp.switch_location,
+ sp.port_name,
+ bpc.addr,
+ bpc.hold_time,
+ bpc.idle_hold_time,
+ bpc.delay_open,
+ bpc.connect_retry,
+ bpc.keepalive,
+ bc.asn
+FROM omicron.public.switch_port sp
+JOIN omicron.public.switch_port_settings_bgp_peer_config bpc
+ON sp.port_settings_id = bpc.port_settings_id
+JOIN omicron.public.bgp_config bc ON bc.id = bpc.bgp_config_id;
+
+CREATE INDEX IF NOT EXISTS switch_port_id_and_name
+ON omicron.public.switch_port (port_settings_id, port_name) STORING (switch_location);
+
+CREATE INDEX IF NOT EXISTS switch_port_name ON omicron.public.switch_port (port_name);
+
 /*
  * Metadata for the schema itself. This version number isn't great, as there's
  * nothing to ensure it gets bumped when it should be, but it's a start.
@@ -3707,7 +3729,7 @@ INSERT INTO omicron.public.db_metadata (
     version,
     target_version
 ) VALUES
-    ( TRUE, NOW(), NOW(), '46.0.0', NULL)
+    ( TRUE, NOW(), NOW(), '47.0.0', NULL)
 ON CONFLICT DO NOTHING;
 
 COMMIT;
diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml
index d210b9b86b..c941ee2625 100644
--- a/sled-agent/Cargo.toml
+++ b/sled-agent/Cargo.toml
@@ -83,6 +83,7 @@ zeroize.workspace = true
 zone.workspace = true
 static_assertions.workspace = true
 omicron-workspace-hack.workspace = true
+slog-error-chain.workspace = true
 
 [target.'cfg(target_os = "illumos")'.dependencies]
 opte-ioctl.workspace = true
diff --git a/sled-agent/src/hardware_monitor.rs b/sled-agent/src/hardware_monitor.rs
index 34e9d8c9c7..cbd3134cf0 100644
--- a/sled-agent/src/hardware_monitor.rs
+++ b/sled-agent/src/hardware_monitor.rs
@@ -208,7 +208,7 @@ impl HardwareMonitor {
                     )
                     .await
                 {
-                    warn!(self.log, "Failed to activate switch: {e}");
+                    error!(self.log, "Failed to activate switch"; e);
                 }
             }
             TofinoManager::NotReady { tofino_loaded } => {
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
index 55ebdccb5c..7aeb0df838 100644
--- a/sled-agent/src/services.rs
+++ b/sled-agent/src/services.rs
@@ -118,7 +118,7 @@ use illumos_utils::zone::Zones;
 
 const IPV6_UNSPECIFIED: IpAddr = IpAddr::V6(Ipv6Addr::UNSPECIFIED);
 
-#[derive(thiserror::Error, Debug)]
+#[derive(thiserror::Error, Debug, slog_error_chain::SlogInlineError)]
 pub enum Error {
     #[error("Failed to initialize CockroachDb: {err}")]
     CockroachInit {

From 30c414a861f41edad9855debdbe28fb28af535bb Mon Sep 17 00:00:00 2001
From: Ryan Goodfellow <ryan.goodfellow@oxide.computer>
Date: Thu, 21 Mar 2024 15:41:17 -0700
Subject: [PATCH 140/157] Plumbing for maghemite + oximeter (#5262)

---
 Cargo.lock                                    | 87 ++-----------------
 Cargo.toml                                    |  4 +-
 clients/ddm-admin-client/Cargo.toml           |  1 +
 clients/ddm-admin-client/src/lib.rs           | 40 ++++++++-
 common/src/api/external/mod.rs                |  4 +-
 .../background/sync_switch_configuration.rs   |  3 +-
 package-manifest.toml                         | 12 +--
 sled-agent/src/bootstrap/early_networking.rs  |  4 +-
 sled-agent/src/bootstrap/server.rs            | 13 +++
 sled-agent/src/services.rs                    | 55 +++++++++++-
 smf/profile/profile                           |  2 +-
 test-utils/Cargo.toml                         |  1 +
 test-utils/src/dev/maghemite.rs               |  6 +-
 tools/maghemite_ddm_openapi_version           |  4 +-
 tools/maghemite_mg_openapi_version            |  4 +-
 tools/maghemite_mgd_checksums                 |  4 +-
 workspace-hack/Cargo.toml                     | 16 ++--
 17 files changed, 148 insertions(+), 112 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index b94ef71980..64ce6d7a5f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -466,26 +466,6 @@ dependencies = [
  "thiserror",
 ]
 
-[[package]]
-name = "bgp"
-version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/maghemite?rev=1d8b818b0fd000bc6d2739575c8868b0781dfe41#1d8b818b0fd000bc6d2739575c8868b0781dfe41"
-dependencies = [
- "anyhow",
- "chrono",
- "mg-common",
- "nom",
- "num_enum 0.7.2",
- "rdb",
- "schemars",
- "serde",
- "sled",
- "slog",
- "slog-async",
- "slog-bunyan",
- "thiserror",
-]
-
 [[package]]
 name = "bhyve_api"
 version = "0.0.0"
@@ -1608,15 +1588,15 @@ dependencies = [
 [[package]]
 name = "ddm-admin-client"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/maghemite?rev=1d8b818b0fd000bc6d2739575c8868b0781dfe41#1d8b818b0fd000bc6d2739575c8868b0781dfe41"
+source = "git+https://github.com/oxidecomputer/maghemite?rev=de065a84831e66c829603d9a098e237e8f5faaa1#de065a84831e66c829603d9a098e237e8f5faaa1"
 dependencies = [
- "mg-common",
  "percent-encoding",
  "progenitor",
  "reqwest",
  "serde",
  "serde_json",
  "slog",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -1909,7 +1889,7 @@ source = "git+https://github.com/oxidecomputer/dlpi-sys#1d587ea98cf2d36f1b1624b0
 dependencies = [
  "libc",
  "libdlpi-sys",
- "num_enum 0.5.11",
+ "num_enum",
  "pretty-hex 0.2.1",
  "thiserror",
  "tokio",
@@ -3430,7 +3410,7 @@ source = "git+https://github.com/oxidecomputer/illumos-devinfo?branch=main#4323b
 dependencies = [
  "anyhow",
  "libc",
- "num_enum 0.5.11",
+ "num_enum",
 ]
 
 [[package]]
@@ -3953,7 +3933,7 @@ dependencies = [
  "colored",
  "dlpi",
  "libc",
- "num_enum 0.5.11",
+ "num_enum",
  "nvpair",
  "nvpair-sys",
  "rusty-doors",
@@ -4186,14 +4166,12 @@ dependencies = [
 [[package]]
 name = "mg-admin-client"
 version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/maghemite?rev=1d8b818b0fd000bc6d2739575c8868b0781dfe41#1d8b818b0fd000bc6d2739575c8868b0781dfe41"
+source = "git+https://github.com/oxidecomputer/maghemite?rev=de065a84831e66c829603d9a098e237e8f5faaa1#de065a84831e66c829603d9a098e237e8f5faaa1"
 dependencies = [
  "anyhow",
- "bgp",
  "chrono",
  "percent-encoding",
  "progenitor",
- "rdb",
  "reqwest",
  "schemars",
  "serde",
@@ -4201,18 +4179,6 @@ dependencies = [
  "slog",
 ]
 
-[[package]]
-name = "mg-common"
-version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/maghemite?rev=1d8b818b0fd000bc6d2739575c8868b0781dfe41#1d8b818b0fd000bc6d2739575c8868b0781dfe41"
-dependencies = [
- "anstyle",
- "clap 4.5.1",
- "schemars",
- "serde",
- "thiserror",
-]
-
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -4966,16 +4932,7 @@ version = "0.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9"
 dependencies = [
- "num_enum_derive 0.5.11",
-]
-
-[[package]]
-name = "num_enum"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845"
-dependencies = [
- "num_enum_derive 0.7.2",
+ "num_enum_derive",
 ]
 
 [[package]]
@@ -4990,18 +4947,6 @@ dependencies = [
  "syn 1.0.109",
 ]
 
-[[package]]
-name = "num_enum_derive"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b"
-dependencies = [
- "proc-macro-crate",
- "proc-macro2",
- "quote",
- "syn 2.0.52",
-]
-
 [[package]]
 name = "num_threads"
 version = "0.1.6"
@@ -5147,6 +5092,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "toml 0.8.10",
+ "uuid 1.7.0",
 ]
 
 [[package]]
@@ -5590,6 +5536,7 @@ dependencies = [
  "tokio",
  "tokio-postgres",
  "usdt 0.5.0",
+ "uuid 1.7.0",
  "walkdir",
 ]
 
@@ -7180,22 +7127,6 @@ dependencies = [
  "yasna",
 ]
 
-[[package]]
-name = "rdb"
-version = "0.1.0"
-source = "git+https://github.com/oxidecomputer/maghemite?rev=1d8b818b0fd000bc6d2739575c8868b0781dfe41#1d8b818b0fd000bc6d2739575c8868b0781dfe41"
-dependencies = [
- "anyhow",
- "ciborium",
- "mg-common",
- "schemars",
- "serde",
- "serde_json",
- "sled",
- "slog",
- "thiserror",
-]
-
 [[package]]
 name = "rdrand"
 version = "0.4.0"
diff --git a/Cargo.toml b/Cargo.toml
index 70b40df81c..a8fa57d5d6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -265,8 +265,8 @@ maplit = "1.0.2"
 mime_guess = "2.0.4"
 mockall = "0.12"
 newtype_derive = "0.1.6"
-mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "1d8b818b0fd000bc6d2739575c8868b0781dfe41" }
-ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "1d8b818b0fd000bc6d2739575c8868b0781dfe41" }
+mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "de065a84831e66c829603d9a098e237e8f5faaa1" }
+ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "de065a84831e66c829603d9a098e237e8f5faaa1" }
 multimap = "0.10.0"
 nexus-client = { path = "clients/nexus-client" }
 nexus-config = { path = "nexus-config" }
diff --git a/clients/ddm-admin-client/Cargo.toml b/clients/ddm-admin-client/Cargo.toml
index 8254007c7c..1b0fca2951 100644
--- a/clients/ddm-admin-client/Cargo.toml
+++ b/clients/ddm-admin-client/Cargo.toml
@@ -16,6 +16,7 @@ omicron-common.workspace = true
 sled-hardware-types.workspace = true
 omicron-workspace-hack.workspace = true
 ddm-admin-client.workspace = true
+uuid.workspace = true
 
 [build-dependencies]
 anyhow.workspace = true
diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs
index c27532d7d7..5be2dd53bd 100644
--- a/clients/ddm-admin-client/src/lib.rs
+++ b/clients/ddm-admin-client/src/lib.rs
@@ -12,8 +12,8 @@
 pub use ddm_admin_client::types;
 pub use ddm_admin_client::Error;
 
+use ddm_admin_client::types::{Ipv6Prefix, TunnelOrigin};
 use ddm_admin_client::Client as InnerClient;
-use ddm_admin_client::{Ipv6Prefix, TunnelOrigin};
 use either::Either;
 use omicron_common::address::Ipv6Subnet;
 use omicron_common::address::SLED_PREFIX;
@@ -24,11 +24,14 @@ use sled_hardware_types::underlay::BOOTSTRAP_MASK;
 use sled_hardware_types::underlay::BOOTSTRAP_PREFIX;
 use slog::info;
 use slog::Logger;
+use std::net::IpAddr;
 use std::net::Ipv6Addr;
 use std::net::SocketAddr;
 use std::net::SocketAddrV6;
 use thiserror::Error;
 
+use crate::types::EnableStatsRequest;
+
 // TODO-cleanup Is it okay to hardcode this port number here?
 const DDMD_PORT: u16 = 8000;
 
@@ -145,4 +148,39 @@ impl Client {
             })
         }))
     }
+
+    /// Spawns a background task to instruct ddmd to advertise the given prefix
+    /// to peer sleds.
+    pub fn enable_stats(
+        &self,
+        addr: IpAddr,
+        dns_servers: Vec<SocketAddr>,
+        rack_id: uuid::Uuid,
+        sled_id: uuid::Uuid,
+    ) {
+        let me = self.clone();
+        tokio::spawn(async move {
+            retry_notify(retry_policy_internal_service_aggressive(), || async {
+                info!(
+                    me.log, "Enabling ddm stats";
+                    "addr" => ?addr,
+                    "dns_servers" => ?dns_servers
+                );
+
+                me.inner.enable_stats(&EnableStatsRequest{
+                    addr,
+                    dns_servers: dns_servers.iter().map(|x| x.to_string()).collect(),
+                    rack_id,
+                    sled_id,
+                }).await?;
+                Ok(())
+            }, |err, duration| {
+                info!(
+                    me.log,
+                    "Failed enable ddm stats (will retry after {duration:?}";
+                    "err" => %err,
+                );
+            }).await.unwrap();
+        });
+    }
 }
diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs
index 002f92ecc9..161e31cf87 100644
--- a/common/src/api/external/mod.rs
+++ b/common/src/api/external/mod.rs
@@ -2758,10 +2758,10 @@ pub struct BgpPeerStatus {
 /// Opaque object representing BGP message history for a given BGP peer. The
 /// contents of this object are not yet stable.
 #[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct BgpMessageHistory(mg_admin_client::MessageHistory);
+pub struct BgpMessageHistory(mg_admin_client::types::MessageHistory);
 
 impl BgpMessageHistory {
-    pub fn new(arg: mg_admin_client::MessageHistory) -> Self {
+    pub fn new(arg: mg_admin_client::types::MessageHistory) -> Self {
         Self(arg)
     }
 }
diff --git a/nexus/src/app/background/sync_switch_configuration.rs b/nexus/src/app/background/sync_switch_configuration.rs
index 7fa2b27966..97a8dcd7e3 100644
--- a/nexus/src/app/background/sync_switch_configuration.rs
+++ b/nexus/src/app/background/sync_switch_configuration.rs
@@ -28,9 +28,8 @@ use futures::future::BoxFuture;
 use futures::FutureExt;
 use mg_admin_client::types::{
     AddStaticRoute4Request, ApplyRequest, BgpPeerConfig,
-    DeleteStaticRoute4Request, StaticRoute4, StaticRoute4List,
+    DeleteStaticRoute4Request, Prefix4, StaticRoute4, StaticRoute4List,
 };
-use mg_admin_client::Prefix4;
 use nexus_db_queries::{
     context::OpContext,
     db::{datastore::SwitchPortSettingsCombinedResult, DataStore},
diff --git a/package-manifest.toml b/package-manifest.toml
index 0cc2e4939c..ba1a251ae2 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -535,10 +535,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "1d8b818b0fd000bc6d2739575c8868b0781dfe41"
+source.commit = "de065a84831e66c829603d9a098e237e8f5faaa1"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/maghemite.sha256.txt
-source.sha256 = "a3914f991ab1441486d42097d135df15c09604c02a8104810e634b05f2e3f525"
+source.sha256 = "8a7525f8329c5178ebf07cecc623a017806b81d5d1ca55cf76b88e737ae57dec"
 output.type = "tarball"
 
 [package.mg-ddm]
@@ -551,10 +551,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "1d8b818b0fd000bc6d2739575c8868b0781dfe41"
+source.commit = "de065a84831e66c829603d9a098e237e8f5faaa1"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mg-ddm.sha256.txt
-source.sha256 = "e973385b276c7910a76a58151723f30c39e2f8a302a506992b0543a7e95616fb"
+source.sha256 = "0cc9cbef39103d3e651334574ebdd0e6ef71670cbe6a720d22b1efb005b5a71c"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -566,10 +566,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "1d8b818b0fd000bc6d2739575c8868b0781dfe41"
+source.commit = "de065a84831e66c829603d9a098e237e8f5faaa1"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mg-ddm.sha256.txt
-source.sha256 = "88065a5d5051c4e684ea15bd182c1d734d649df2336b7612108e2aee330b81a6"
+source.sha256 = "4256d320b1ec310d56679eca5f65c9149072fe647f66021fd0cce1411fc39e0c"
 output.type = "zone"
 output.intermediate_only = true
 
diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs
index 0cc0f22f5d..2bb0d31797 100644
--- a/sled-agent/src/bootstrap/early_networking.rs
+++ b/sled-agent/src/bootstrap/early_networking.rs
@@ -16,10 +16,10 @@ use internal_dns::resolver::{ResolveError, Resolver as DnsResolver};
 use internal_dns::ServiceName;
 use ipnetwork::Ipv6Network;
 use mg_admin_client::types::{
-    AddStaticRoute4Request, ApplyRequest, BgpPeerConfig, StaticRoute4,
+    AddStaticRoute4Request, ApplyRequest, BgpPeerConfig, Prefix4, StaticRoute4,
     StaticRoute4List,
 };
-use mg_admin_client::{Client as MgdClient, Prefix4};
+use mg_admin_client::Client as MgdClient;
 use omicron_common::address::DENDRITE_PORT;
 use omicron_common::address::{MGD_PORT, MGS_PORT};
 use omicron_common::api::internal::shared::{
diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs
index d1f3fa0646..bca3350696 100644
--- a/sled-agent/src/bootstrap/server.rs
+++ b/sled-agent/src/bootstrap/server.rs
@@ -36,6 +36,8 @@ use illumos_utils::dladm;
 use illumos_utils::zfs;
 use illumos_utils::zone;
 use illumos_utils::zone::Zones;
+use internal_dns::resolver::Resolver;
+use omicron_common::address::{Ipv6Subnet, AZ_PREFIX};
 use omicron_common::ledger;
 use omicron_common::ledger::Ledger;
 use omicron_ddm_admin_client::Client as DdmAdminClient;
@@ -409,6 +411,17 @@ async fn start_sled_agent(
     // indicating which kind of address we're advertising).
     ddmd_client.advertise_prefix(request.body.subnet);
 
+    let az_prefix =
+        Ipv6Subnet::<AZ_PREFIX>::new(request.body.subnet.net().network());
+    let addr = request.body.subnet.net().iter().nth(1).unwrap();
+    let dns_servers = Resolver::servers_from_subnet(az_prefix);
+    ddmd_client.enable_stats(
+        addr.into(),
+        dns_servers,
+        request.body.rack_id,
+        request.body.id,
+    );
+
     // Server does not exist, initialize it.
     let server = SledAgentServer::start(
         config,
diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
index 7aeb0df838..c0dbaebcc2 100644
--- a/sled-agent/src/services.rs
+++ b/sled-agent/src/services.rs
@@ -2754,14 +2754,63 @@ impl ServiceManager {
                         }
                         SwitchService::Mgd => {
                             info!(self.inner.log, "Setting up mgd service");
-                            smfh.setprop("config/admin_host", "::")?;
+                            smfh.delpropvalue("config/dns_servers", "*")?;
+                            let info = self
+                                .inner
+                                .sled_info
+                                .get()
+                                .ok_or(Error::SledAgentNotReady)?;
+                            smfh.setprop("config/rack_uuid", info.rack_id)?;
+                            smfh.setprop(
+                                "config/sled_uuid",
+                                info.config.sled_id,
+                            )?;
+                            for address in &request.zone.addresses {
+                                if *address != Ipv6Addr::LOCALHOST {
+                                    let az_prefix =
+                                        Ipv6Subnet::<AZ_PREFIX>::new(*address);
+                                    for addr in
+                                        Resolver::servers_from_subnet(az_prefix)
+                                    {
+                                        smfh.addpropvalue(
+                                            "config/dns_servers",
+                                            &format!("{addr}"),
+                                        )?;
+                                    }
+                                    break;
+                                }
+                            }
                             smfh.refresh()?;
                         }
                         SwitchService::MgDdm { mode } => {
                             info!(self.inner.log, "Setting up mg-ddm service");
-
                             smfh.setprop("config/mode", &mode)?;
-                            smfh.setprop("config/admin_host", "::")?;
+                            let info = self
+                                .inner
+                                .sled_info
+                                .get()
+                                .ok_or(Error::SledAgentNotReady)?;
+                            smfh.setprop("config/rack_uuid", info.rack_id)?;
+                            smfh.setprop(
+                                "config/sled_uuid",
+                                info.config.sled_id,
+                            )?;
+                            smfh.delpropvalue("config/dns_servers", "*")?;
+                            for address in &request.zone.addresses {
+                                if *address != Ipv6Addr::LOCALHOST {
+                                    let az_prefix =
+                                        Ipv6Subnet::<AZ_PREFIX>::new(*address);
+                                    for addr in
+                                        Resolver::servers_from_subnet(az_prefix)
+                                    {
+                                        smfh.addpropvalue(
+                                            "config/dns_servers",
+                                            &format!("{addr}"),
+                                        )?;
+                                    }
+                                    break;
+                                }
+                            }
 
                             let is_gimlet = is_gimlet().map_err(|e| {
                                 Error::Underlay(
diff --git a/smf/profile/profile b/smf/profile/profile
index a8378f7403..e3802d1bd4 100644
--- a/smf/profile/profile
+++ b/smf/profile/profile
@@ -4,7 +4,7 @@ PATH+=:/opt/ooce/bin
 case "$HOSTNAME" in
     oxz_switch)
 	# Add tools like xcvradm, swadm & ddmadm to the PATH by default
-	PATH+=:/opt/oxide/bin:/opt/oxide/dendrite/bin:/opt/oxide/mg-ddm/bin:/opt/oxide/omdb/bin
+	PATH+=:/opt/oxide/bin:/opt/oxide/dendrite/bin:/opt/oxide/mg-ddm/bin:/opt/oxide/omdb/bin:/opt/oxide/mgd/bin
 	;;
     oxz_cockroachdb*)
 	PATH+=:/opt/oxide/cockroachdb/bin
diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml
index 49bdc030c7..d60f4f55e3 100644
--- a/test-utils/Cargo.toml
+++ b/test-utils/Cargo.toml
@@ -33,6 +33,7 @@ rcgen.workspace = true
 reqwest.workspace = true
 walkdir.workspace = true
 omicron-workspace-hack.workspace = true
+uuid.workspace = true
 
 [dev-dependencies]
 expectorate.workspace = true
diff --git a/test-utils/src/dev/maghemite.rs b/test-utils/src/dev/maghemite.rs
index fa1f353896..a2ee9b2253 100644
--- a/test-utils/src/dev/maghemite.rs
+++ b/test-utils/src/dev/maghemite.rs
@@ -47,6 +47,10 @@ impl MgdInstance {
             "--no-bgp-dispatcher".into(),
             "--data-dir".into(),
             temp_dir.path().display().to_string(),
+            "--rack-uuid".into(),
+            uuid::Uuid::new_v4().to_string(),
+            "--sled-uuid".into(),
+            uuid::Uuid::new_v4().to_string(),
         ];
 
         let child = tokio::process::Command::new("mgd")
@@ -133,7 +137,7 @@ async fn discover_port(logfile: String) -> Result<u16, anyhow::Error> {
 }
 
 async fn find_mgd_port_in_log(logfile: String) -> Result<u16, anyhow::Error> {
-    let re = regex::Regex::new(r#""local_addr":"\[::1\]:?([0-9]+)""#).unwrap();
+    let re = regex::Regex::new(r#""local_addr":"\[::\]:?([0-9]+)""#).unwrap();
     let reader = BufReader::new(File::open(logfile).await?);
     let mut lines = reader.lines();
     loop {
diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version
index 73cbfefad8..d2a6a9210d 100644
--- a/tools/maghemite_ddm_openapi_version
+++ b/tools/maghemite_ddm_openapi_version
@@ -1,2 +1,2 @@
-COMMIT="a19e88979ab8ab9a0136854182c410c29047bfcb"
-SHA2="0b0dbc2f8bbc5d2d9be92d64c4865f8f9335355aae62f7de9f67f81dfb3f1803"
+COMMIT="bc244d418368fcc3e66db2a0637501a900be161a"
+SHA2="004e873e4120aa26460271368485266b75b7f964e5ed4dbee8fb5db4519470d7"
diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version
index f46f293368..396a74d26a 100644
--- a/tools/maghemite_mg_openapi_version
+++ b/tools/maghemite_mg_openapi_version
@@ -1,2 +1,2 @@
-COMMIT="a19e88979ab8ab9a0136854182c410c29047bfcb"
-SHA2="768b38ad9b01299be66a899241158123fc90ea240708a517c8acfa52b4b2eb8e"
+COMMIT="bc244d418368fcc3e66db2a0637501a900be161a"
+SHA2="a5d2f275c99152711dec1df58fd49d459d3fcb8fbfc7a7f48f432be248d74639"
diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums
index face9b9b1b..5a9bd77910 100644
--- a/tools/maghemite_mgd_checksums
+++ b/tools/maghemite_mgd_checksums
@@ -1,2 +1,2 @@
-CIDL_SHA256="813fe3d408f538a9d7121a616fce5f87e6c42cf2800e20c1a2759c24c9e8b77b"
-MGD_LINUX_SHA256="03bf4e9fbe7d46be5dbcf83f9cd67520383f843767939e06d0db93ce4e584111"
\ No newline at end of file
+CIDL_SHA256="ddc19da8932e0ff4a91dea139369bb5b729748bdf698223bc2717a4abea89ed6"
+MGD_LINUX_SHA256="ee41d903d19b4ced4bbf139ed05ba8c060517642bc7d4bcef3790e697fa3f44d"
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index c4fd52aef7..a2d853ac23 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -28,8 +28,8 @@ byteorder = { version = "1.5.0" }
 bytes = { version = "1.5.0", features = ["serde"] }
 chrono = { version = "0.4.34", features = ["serde"] }
 cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] }
-clap = { version = "4.5.1", features = ["cargo", "derive", "env", "unstable-styles", "wrap_help"] }
-clap_builder = { version = "4.5.1", default-features = false, features = ["cargo", "env", "std", "suggestions", "unstable-styles", "usage", "wrap_help"] }
+clap = { version = "4.5.1", features = ["cargo", "derive", "env", "wrap_help"] }
+clap_builder = { version = "4.5.1", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
 console = { version = "0.15.8" }
 const-oid = { version = "0.9.6", default-features = false, features = ["db", "std"] }
 crossbeam-epoch = { version = "0.9.18" }
@@ -105,9 +105,7 @@ tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serd
 tokio-stream = { version = "0.1.14", features = ["net"] }
 tokio-util = { version = "0.7.10", features = ["codec", "io-util"] }
 toml = { version = "0.7.8" }
-toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] }
 toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.6", features = ["serde"] }
-toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] }
 tracing = { version = "0.1.40", features = ["log"] }
 trust-dns-proto = { version = "0.22.0" }
 unicode-bidi = { version = "0.3.15" }
@@ -135,8 +133,8 @@ byteorder = { version = "1.5.0" }
 bytes = { version = "1.5.0", features = ["serde"] }
 chrono = { version = "0.4.34", features = ["serde"] }
 cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] }
-clap = { version = "4.5.1", features = ["cargo", "derive", "env", "unstable-styles", "wrap_help"] }
-clap_builder = { version = "4.5.1", default-features = false, features = ["cargo", "env", "std", "suggestions", "unstable-styles", "usage", "wrap_help"] }
+clap = { version = "4.5.1", features = ["cargo", "derive", "env", "wrap_help"] }
+clap_builder = { version = "4.5.1", default-features = false, features = ["cargo", "color", "env", "std", "suggestions", "usage", "wrap_help"] }
 console = { version = "0.15.8" }
 const-oid = { version = "0.9.6", default-features = false, features = ["db", "std"] }
 crossbeam-epoch = { version = "0.9.18" }
@@ -213,9 +211,7 @@ tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serd
 tokio-stream = { version = "0.1.14", features = ["net"] }
 tokio-util = { version = "0.7.10", features = ["codec", "io-util"] }
 toml = { version = "0.7.8" }
-toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] }
 toml_edit-3c51e837cfc5589a = { package = "toml_edit", version = "0.22.6", features = ["serde"] }
-toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] }
 tracing = { version = "0.1.40", features = ["log"] }
 trust-dns-proto = { version = "0.22.0" }
 unicode-bidi = { version = "0.3.15" }
@@ -272,6 +268,8 @@ dof = { version = "0.3.0", default-features = false, features = ["des"] }
 mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
+toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] }
+toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] }
 
 [target.x86_64-unknown-illumos.build-dependencies]
 bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.2", default-features = false, features = ["std"] }
@@ -279,5 +277,7 @@ dof = { version = "0.3.0", default-features = false, features = ["des"] }
 mio = { version = "0.8.11", features = ["net", "os-ext"] }
 once_cell = { version = "1.19.0" }
 rustix = { version = "0.38.31", features = ["fs", "termios"] }
+toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] }
+toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] }
 
 ### END HAKARI SECTION

From f28d406391334b9b104b1b6154b072581b28d362 Mon Sep 17 00:00:00 2001
From: David Crespo <david-crespo@users.noreply.github.com>
Date: Thu, 21 Mar 2024 18:50:07 -0500
Subject: [PATCH 141/157] Bump web console (IP pool utilization)  (#5303)

https://github.com/oxidecomputer/console/compare/784e8aac...7c968443

* [7c968443](https://github.com/oxidecomputer/console/commit/7c968443)
oxidecomputer/console#2083
* [b844a42c](https://github.com/oxidecomputer/console/commit/b844a42c)
oxidecomputer/console#2078
* [47835ee3](https://github.com/oxidecomputer/console/commit/47835ee3)
oxidecomputer/console#2086
* [c4ddebc2](https://github.com/oxidecomputer/console/commit/c4ddebc2)
oxidecomputer/console#2082
* [530a0997](https://github.com/oxidecomputer/console/commit/530a0997)
upload-artifact@v4 requires unique filenames for each run in matrix
* [b996c34c](https://github.com/oxidecomputer/console/commit/b996c34c)
bump actions/upload-artifact to v4
* [bc51f530](https://github.com/oxidecomputer/console/commit/bc51f530)
bump API to latest main
* [743da3b0](https://github.com/oxidecomputer/console/commit/743da3b0)
oxidecomputer/console#2080
* [7596d633](https://github.com/oxidecomputer/console/commit/7596d633)
oxidecomputer/console#2073
* [65531bff](https://github.com/oxidecomputer/console/commit/65531bff)
oxidecomputer/console#2074
* [4d226cd3](https://github.com/oxidecomputer/console/commit/4d226cd3)
oxidecomputer/console#1891
* [7490104a](https://github.com/oxidecomputer/console/commit/7490104a)
update readme with cool new preview URL
* [a2218994](https://github.com/oxidecomputer/console/commit/a2218994)
oxidecomputer/console#2066
* [641ebe85](https://github.com/oxidecomputer/console/commit/641ebe85)
oxidecomputer/console#2061
* [1bb8dd3a](https://github.com/oxidecomputer/console/commit/1bb8dd3a)
oxidecomputer/console#2055
* [41dd9f06](https://github.com/oxidecomputer/console/commit/41dd9f06)
oxidecomputer/console#2059
* [04692802](https://github.com/oxidecomputer/console/commit/04692802)
oxidecomputer/console#2048
* [8c30305e](https://github.com/oxidecomputer/console/commit/8c30305e)
oxidecomputer/console#2050
* [53709d2a](https://github.com/oxidecomputer/console/commit/53709d2a)
oxidecomputer/console#2046
* [84caede7](https://github.com/oxidecomputer/console/commit/84caede7)
oxidecomputer/console#2026
* [bca9f42a](https://github.com/oxidecomputer/console/commit/bca9f42a)
oxidecomputer/console#2045
* [09e5f713](https://github.com/oxidecomputer/console/commit/09e5f713)
oxidecomputer/console#2043
* [d780e5d8](https://github.com/oxidecomputer/console/commit/d780e5d8)
fix casing on floating IP attach/detach toasts
---
 tools/console_version | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/console_version b/tools/console_version
index b3ccaf80b8..0946febc50 100644
--- a/tools/console_version
+++ b/tools/console_version
@@ -1,2 +1,2 @@
-COMMIT="784e8aac273dc282b6218994d09042aa5928a198"
-SHA2="b5afd1243e40e86d3a779f089fe61ed9cb3fbbd6d59e547e522a59c1b0e4b7d8"
+COMMIT="7c968443ce03e3fbadb8896f80fb9b156b46e3dd"
+SHA2="e6eb2df59804c0c85d8c0a262be00d3fc32668ef19abde0c1858e333f264b0c8"

From 5ffe09f62119587a4044393066c4cd85ca24de75 Mon Sep 17 00:00:00 2001
From: "oxide-reflector-bot[bot]"
 <130185838+oxide-reflector-bot[bot]@users.noreply.github.com>
Date: Fri, 22 Mar 2024 00:26:31 +0000
Subject: [PATCH 142/157] Update dendrite to c2a9f29 (#5240)

Updated dendrite to commit c2a9f29.

---------

Co-authored-by: reflector[bot] <130185838+reflector[bot]@users.noreply.github.com>
---
 package-manifest.toml          | 12 ++++++------
 tools/dendrite_openapi_version |  2 +-
 tools/dendrite_stub_checksums  |  6 +++---
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/package-manifest.toml b/package-manifest.toml
index ba1a251ae2..18cde0368b 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -595,8 +595,8 @@ only_for_targets.image = "standard"
 # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "b128500231b916802a9436dd438742424aa5c6ce"
-source.sha256 = "47e0b279cf5babb199ea4d7d1654ccef6f463eeae7cef24e8138658c57affb4a"
+source.commit = "c2a9f29f70b1e05d891c713997577be53826e1bb"
+source.sha256 = "1405185cc9645b3a6f1bf82d2ffd89f17505e4e625795cd0beb58f043fa7fd8a"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -620,8 +620,8 @@ only_for_targets.image = "standard"
 # 2. Copy the output zone image from dendrite/out to omicron/out
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "b128500231b916802a9436dd438742424aa5c6ce"
-source.sha256 = "3e6e785a3cfbf12dd9752c3bb026e4e3e75aa9d4433ba65045c7396d31e970a9"
+source.commit = "c2a9f29f70b1e05d891c713997577be53826e1bb"
+source.sha256 = "2124ffd76bb80bcb7063862a1516da3d805b1c062fe2339b95e4656355a55fd9"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -638,8 +638,8 @@ only_for_targets.image = "standard"
 # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "b128500231b916802a9436dd438742424aa5c6ce"
-source.sha256 = "64cb9c62d3516c613ea70c36dff241cba2e1138e25a82a6367c40da262ca55ee"
+source.commit = "c2a9f29f70b1e05d891c713997577be53826e1bb"
+source.sha256 = "3e89ff18a1209b82caafce91db42dec9f9f8d0fcaacbb1a8cfe7d1c0b0966700"
 output.type = "zone"
 output.intermediate_only = true
 
diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version
index 3d5f21ffe0..14221a70ab 100644
--- a/tools/dendrite_openapi_version
+++ b/tools/dendrite_openapi_version
@@ -1,2 +1,2 @@
-COMMIT="b128500231b916802a9436dd438742424aa5c6ce"
+COMMIT="c2a9f29f70b1e05d891c713997577be53826e1bb"
 SHA2="50eff6d9f986b7b1af5970d11d8d01b812de37269731c6c691a244b3fdae82ae"
diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums
index 3bca6cd174..74147127b1 100644
--- a/tools/dendrite_stub_checksums
+++ b/tools/dendrite_stub_checksums
@@ -1,3 +1,3 @@
-CIDL_SHA256_ILLUMOS="47e0b279cf5babb199ea4d7d1654ccef6f463eeae7cef24e8138658c57affb4a"
-CIDL_SHA256_LINUX_DPD="11e464a38fa0858c3d896e82d7ee3123aee5f0cf4e3c2a029a0dd7cfd54d3adf"
-CIDL_SHA256_LINUX_SWADM="66eab497b955751d0704c3cd97ac5c1ed373aa656fc37ccba86ae9900b5ae96d"
+CIDL_SHA256_ILLUMOS="1405185cc9645b3a6f1bf82d2ffd89f17505e4e625795cd0beb58f043fa7fd8a"
+CIDL_SHA256_LINUX_DPD="db3e419cf4381607482c55230c64c6afa80388a26db8efa48e437ebc8b82526a"
+CIDL_SHA256_LINUX_SWADM="1de82f6b7419aa0e6e0632def69d5b370a9cb3ac2261527fce96840eb5af5b8d"

From 946e4db568a033ba4f649b7f893cf0fb09785c05 Mon Sep 17 00:00:00 2001
From: Michael Zeller <mike@mikezeller.net>
Date: Thu, 21 Mar 2024 21:39:22 -0400
Subject: [PATCH 143/157] Remove missed cleanup in deploy.sh (#5306)

This PR removes code that should have been cleaned up when the following
was previously removed:

https://github.com/oxidecomputer/omicron/blob/978cf08adfc1fb6ec4fdf39814d625eba46447eb/.github/buildomat/jobs/deploy.sh#L104-L115
---
 .github/buildomat/jobs/deploy.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index e7990dafe5..6574ac839c 100755
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -34,9 +34,6 @@ _exit_trap() {
 	local status=$?
 	[[ $status -eq 0 ]] && exit 0
 
-    # XXX paranoia
-    pfexec cp /tmp/opteadm /opt/oxide/opte/bin/opteadm
-
 	set +o errexit
 	set -o xtrace
 	banner evidence

From 5d97551438d5186bd6e806425853c6b66bbf272c Mon Sep 17 00:00:00 2001
From: Michael Zeller <mike@mikezeller.net>
Date: Thu, 21 Mar 2024 22:02:37 -0400
Subject: [PATCH 144/157] Provide a way of detecting system library leakage
 (#5259)

After https://github.com/oxidecomputer/omicron/pull/5158 was integrated
@rain noticed that attempting to run a build of `omdb` in the switch
zone suddenly stopped working and filed
https://github.com/oxidecomputer/helios-omicron-brand/issues/15.
@jgallagher ended up fixing this by splitting out the sled-hardware
types into their own crate in
https://github.com/oxidecomputer/omicron/pull/5245.

We decided it would be good if we added some sort of CI check to omicron
to catch these library leakages earlier. This PR introduces that check
and adds it to the helios build and test buildomat job. I have also
added some notes to the readme for others that may end up adding a new
library dependency.

Locally I modified the allow list so that it would produce errors, those
errors end up looking like:
```
$ cargo xtask verify-libraries
    Finished dev [unoptimized + debuginfo] target(s) in 0.42s
     Running `target/debug/xtask verify-libraries`
    Finished dev [unoptimized + debuginfo] target(s) in 4.11s
Error: Found library issues with the following:
installinator
	UNEXPECTED dependency on libipcc.so.1
omicron-dev
	UNEXPECTED dependency on libipcc.so.1
	UNEXPECTED dependency on libresolv.so.2
sp-sim
	UNEXPECTED dependency on libipcc.so.1
	UNEXPECTED dependency on libresolv.so.2
sled-agent
	NEEDS libnvme.so.1 but is not allowed
mgs
	UNEXPECTED dependency on libipcc.so.1
	UNEXPECTED dependency on libresolv.so.2


If depending on a new library was intended please add it to xtask.toml
```
---
 .cargo/xtask.toml                   |  41 ++++++++
 .github/buildomat/build-and-test.sh |  13 +++
 Cargo.lock                          |   4 +
 README.adoc                         |   4 +
 dev-tools/xtask/Cargo.toml          |   4 +
 dev-tools/xtask/src/illumos.rs      | 142 ++++++++++++++++++++++++++++
 dev-tools/xtask/src/main.rs         |  18 +++-
 7 files changed, 224 insertions(+), 2 deletions(-)
 create mode 100644 .cargo/xtask.toml
 create mode 100644 dev-tools/xtask/src/illumos.rs

diff --git a/.cargo/xtask.toml b/.cargo/xtask.toml
new file mode 100644
index 0000000000..0055cf723c
--- /dev/null
+++ b/.cargo/xtask.toml
@@ -0,0 +1,41 @@
+# This config file is used by `cargo xtask verify-libraries`
+
+
+# These are libraries that we expect to show up in any executable produced
+# by the omicron repo.
+[libraries."libc.so.1"]
+[libraries."libcontract.so.1"]
+[libraries."libcrypto.so.3"]
+[libraries."libdevinfo.so.1"]
+[libraries."libdlpi.so.1"]
+[libraries."libdoor.so.1"]
+[libraries."libefi.so.1"]
+[libraries."libgcc_s.so.1"]
+[libraries."libipcc.so.1"]
+[libraries."libkstat.so.1"]
+[libraries."libm.so.2"]
+[libraries."libnsl.so.1"]
+[libraries."libnvpair.so.1"]
+[libraries."libpq.so.5"]
+[libraries."libpthread.so.1"]
+[libraries."libresolv.so.2"]
+[libraries."librt.so.1"]
+[libraries."libscf.so.1"]
+[libraries."libsocket.so.1"]
+[libraries."libssl.so.3"]
+[libraries."libumem.so.1"]
+[libraries."libxml2.so.2"]
+[libraries."libxmlsec1.so.1"]
+
+# libnvme is a global zone only library and therefore we must be sure that only
+# programs running in the gz require it. Additionally only sled-agent should be
+# managing a sled's hardware.
+[libraries."libnvme.so.1"]
+binary_allow_list = [
+  "installinator",
+  "omicron-dev",
+  "omicron-package",
+  "services-ledger-check-migrate",
+  "sled-agent",
+  "sled-agent-sim",
+]
diff --git a/.github/buildomat/build-and-test.sh b/.github/buildomat/build-and-test.sh
index 30abd02f90..7d0aadcb56 100755
--- a/.github/buildomat/build-and-test.sh
+++ b/.github/buildomat/build-and-test.sh
@@ -76,6 +76,19 @@ export RUSTC_BOOTSTRAP=1
 # We report build progress to stderr, and the "--timings=json" output goes to stdout.
 ptime -m cargo build -Z unstable-options --timings=json --workspace --tests --locked --verbose 1> "$OUTPUT_DIR/crate-build-timings.json"
 
+# If we are running on illumos we want to verify that we are not requiring
+# system libraries outside of specific binaries. If we encounter this situation
+# we bail.
+# NB: `cargo xtask verify-libraries` runs `cargo build --bins` to ensure it can
+# check the final executables.
+if [[ $target_os == "illumos" ]]; then
+    banner verify-libraries
+    # This has a separate timeout from `cargo nextest` since `timeout` expects
+    # to run an external command and therefore we cannot run bash functions or
+    # subshells.
+    ptime -m timeout 10m cargo xtask verify-libraries
+fi
+
 #
 # We apply our own timeout to ensure that we get a normal failure on timeout
 # rather than a buildomat timeout.  See oxidecomputer/buildomat#8.
diff --git a/Cargo.lock b/Cargo.lock
index 64ce6d7a5f..5b034fee62 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -10973,6 +10973,10 @@ dependencies = [
  "cargo_metadata",
  "cargo_toml",
  "clap 4.5.1",
+ "fs-err",
+ "serde",
+ "swrite",
+ "toml 0.8.10",
 ]
 
 [[package]]
diff --git a/README.adoc b/README.adoc
index 93d1fa4fb8..04bc33b2af 100644
--- a/README.adoc
+++ b/README.adoc
@@ -106,6 +106,10 @@ When you're happy with things and want to make sure you haven't missed something
 cargo nextest run
 ```
 
+=== Adding a new system library dependency
+
+We check that certain system library dependencies are not leaked outside of their intended binaries via `cargo xtask verify-libraries` in CI. If you are adding a new dependency on a illumos/helios library it is recommended that you update xref:.cargo/xtask.toml[] with an allow list of where you expect the dependency to show up. For example some libraries such as `libnvme.so.1` are only available in the global zone and therefore will not be present in any other zone. This check is here to help us catch any leakage before we go to deploy on a rack. You can inspect a compiled binary in the target directory for what it requires by using `elfedit` - for example `elfedit -r -e 'dyn:tag NEEDED' /path/to/omicron/target/debug/sled-agent`.
+
 === Rust packages in Omicron
 
 NOTE: The term "package" is overloaded: most programming languages and operating systems have their own definitions of a package.  On top of that, Omicron bundles up components into our own kind of "package" that gets delivered via the install and update systems.  These are described in the `package-manifest.toml` file in the root of the repo.  In this section, we're just concerned with Rust packages.
diff --git a/dev-tools/xtask/Cargo.toml b/dev-tools/xtask/Cargo.toml
index 0429fcae79..73bfe0b37a 100644
--- a/dev-tools/xtask/Cargo.toml
+++ b/dev-tools/xtask/Cargo.toml
@@ -10,3 +10,7 @@ camino.workspace = true
 cargo_toml = "0.19"
 cargo_metadata = "0.18"
 clap.workspace = true
+serde.workspace = true
+toml.workspace = true
+fs-err.workspace = true
+swrite.workspace = true
diff --git a/dev-tools/xtask/src/illumos.rs b/dev-tools/xtask/src/illumos.rs
new file mode 100644
index 0000000000..a2daab2c9e
--- /dev/null
+++ b/dev-tools/xtask/src/illumos.rs
@@ -0,0 +1,142 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use anyhow::{bail, Context, Result};
+use camino::Utf8Path;
+use cargo_metadata::Message;
+use fs_err as fs;
+use serde::Deserialize;
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    io::BufReader,
+    process::{Command, Stdio},
+};
+use swrite::{swriteln, SWrite};
+
+use crate::load_workspace;
+
+#[derive(Deserialize, Debug)]
+struct LibraryConfig {
+    binary_allow_list: Option<BTreeSet<String>>,
+}
+
+#[derive(Deserialize, Debug)]
+struct XtaskConfig {
+    libraries: BTreeMap<String, LibraryConfig>,
+}
+
+#[derive(Debug)]
+enum LibraryError {
+    Unexpected(String),
+    NotAllowed(String),
+}
+
+/// Verify that the binary at the provided path complies with the rules laid out
+/// in the xtask.toml config file. Errors are pushed to a hashmap so that we can
+/// display to a user the entire list of issues in one go.
+fn verify_executable(
+    config: &XtaskConfig,
+    path: &Utf8Path,
+    errors: &mut BTreeMap<String, Vec<LibraryError>>,
+) -> Result<()> {
+    let binary = path.file_name().context("basename of executable")?;
+
+    let command = Command::new("elfedit")
+        .args(["-o", "simple", "-r", "-e", "dyn:tag NEEDED"])
+        .arg(path)
+        .output()
+        .context("exec elfedit")?;
+
+    if !command.status.success() {
+        bail!("Failed to execute elfedit successfully {}", command.status);
+    }
+
+    let stdout = String::from_utf8(command.stdout)?;
+    // `elfedit -o simple -r -e "dyn:tag NEEDED" /file/path` will return
+    // a new line seperated list of required libraries so we walk over
+    // them looking for a match in our configuration file. If we find
+    // the library we make sure the binary is allowed to pull it in via
+    // the whitelist.
+    for library in stdout.lines() {
+        let library_config = match config.libraries.get(library.trim()) {
+            Some(config) => config,
+            None => {
+                errors
+                    .entry(binary.to_string())
+                    .or_default()
+                    .push(LibraryError::Unexpected(library.to_string()));
+
+                continue;
+            }
+        };
+
+        if let Some(allowed) = &library_config.binary_allow_list {
+            if !allowed.contains(binary) {
+                errors
+                    .entry(binary.to_string())
+                    .or_default()
+                    .push(LibraryError::NotAllowed(library.to_string()));
+            }
+        }
+    }
+
+    Ok(())
+}
+pub fn cmd_verify_libraries() -> Result<()> {
+    let metadata = load_workspace()?;
+    let mut config_path = metadata.workspace_root;
+    config_path.push(".cargo/xtask.toml");
+    let config = read_xtask_toml(&config_path)?;
+
+    let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string());
+    let mut command = Command::new(cargo)
+        .args(["build", "--bins", "--message-format=json-render-diagnostics"])
+        .stdout(Stdio::piped())
+        .spawn()
+        .context("failed to spawn cargo build")?;
+
+    let reader = BufReader::new(command.stdout.take().context("take stdout")?);
+
+    let mut errors = Default::default();
+    for message in cargo_metadata::Message::parse_stream(reader) {
+        if let Message::CompilerArtifact(artifact) = message? {
+            // We are only interested in artifacts that are binaries
+            if let Some(executable) = artifact.executable {
+                verify_executable(&config, &executable, &mut errors)?;
+            }
+        }
+    }
+
+    let status = command.wait()?;
+    if !status.success() {
+        bail!("Failed to execute cargo build successfully {}", status);
+    }
+
+    if !errors.is_empty() {
+        let mut msg = String::new();
+        errors.iter().for_each(|(binary, errors)| {
+            swriteln!(msg, "{binary}");
+            errors.iter().for_each(|error| match error {
+                LibraryError::Unexpected(lib) => {
+                    swriteln!(msg, "\tUNEXPECTED dependency on {lib}");
+                }
+                LibraryError::NotAllowed(lib) => {
+                    swriteln!(msg, "\tNEEDS {lib} but is not allowed");
+                }
+            });
+        });
+
+        bail!(
+            "Found library issues with the following:\n{msg}\n\n\
+        If depending on a new library was intended please add it to xtask.toml"
+        );
+    }
+
+    Ok(())
+}
+
+fn read_xtask_toml(path: &Utf8Path) -> Result<XtaskConfig> {
+    let config_str = fs::read_to_string(path)?;
+    toml::from_str(&config_str).with_context(|| format!("parse {:?}", path))
+}
diff --git a/dev-tools/xtask/src/main.rs b/dev-tools/xtask/src/main.rs
index 42e6c10d64..c682fc247e 100644
--- a/dev-tools/xtask/src/main.rs
+++ b/dev-tools/xtask/src/main.rs
@@ -11,8 +11,14 @@ use camino::Utf8Path;
 use cargo_metadata::Metadata;
 use cargo_toml::{Dependency, Manifest};
 use clap::{Parser, Subcommand};
+use fs_err as fs;
 use std::{collections::BTreeMap, process::Command};
 
+#[cfg(target_os = "illumos")]
+mod illumos;
+#[cfg(target_os = "illumos")]
+use illumos::cmd_verify_libraries;
+
 #[derive(Parser)]
 #[command(name = "cargo xtask", about = "Workspace-related developer tools")]
 struct Args {
@@ -27,6 +33,9 @@ enum Cmds {
     CheckWorkspaceDeps,
     /// Run configured clippy checks
     Clippy(ClippyArgs),
+    /// Verify we are not leaking library bindings outside of intended
+    /// crates
+    VerifyLibraries,
 }
 
 #[derive(Parser)]
@@ -41,6 +50,7 @@ fn main() -> Result<()> {
     match args.cmd {
         Cmds::Clippy(args) => cmd_clippy(args),
         Cmds::CheckWorkspaceDeps => cmd_check_workspace_deps(),
+        Cmds::VerifyLibraries => cmd_verify_libraries(),
     }
 }
 
@@ -213,9 +223,13 @@ fn cmd_check_workspace_deps() -> Result<()> {
     Ok(())
 }
 
+#[cfg(not(target_os = "illumos"))]
+fn cmd_verify_libraries() -> Result<()> {
+    unimplemented!("Library verification is only available on illumos!")
+}
+
 fn read_cargo_toml(path: &Utf8Path) -> Result<Manifest> {
-    let bytes =
-        std::fs::read(path).with_context(|| format!("read {:?}", path))?;
+    let bytes = fs::read(path)?;
     Manifest::from_slice(&bytes).with_context(|| format!("parse {:?}", path))
 }
 

From d2c2829c8e6c31ba0e07b94226a690ccad03e08f Mon Sep 17 00:00:00 2001
From: David Crespo <david-crespo@users.noreply.github.com>
Date: Thu, 21 Mar 2024 21:31:05 -0500
Subject: [PATCH 145/157] Dedupe IPs in utilization count (#5307)

Fixes #5304

<img width="586"
src="https://github.com/oxidecomputer/omicron/assets/3612203/1ced40d5-eeae-400d-91fb-43091cc34488"
/>
---
 nexus/db-queries/src/db/datastore/ip_pool.rs  | 10 ++++++++--
 nexus/tests/integration_tests/external_ips.rs | 14 +++++++-------
 nexus/tests/integration_tests/instances.rs    |  9 +++++----
 3 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/nexus/db-queries/src/db/datastore/ip_pool.rs b/nexus/db-queries/src/db/datastore/ip_pool.rs
index c0f4e0eea8..30ea2e89b4 100644
--- a/nexus/db-queries/src/db/datastore/ip_pool.rs
+++ b/nexus/db-queries/src/db/datastore/ip_pool.rs
@@ -348,9 +348,15 @@ impl DataStore {
         let (ipv4, ipv6) = external_ip::table
             .filter(external_ip::ip_pool_id.eq(authz_pool.id()))
             .filter(external_ip::time_deleted.is_null())
+            // need to do distinct IP because SNAT IPs are shared between
+            // multiple instances, and each gets its own row in the table
             .select((
-                sql::<BigInt>("count(*) FILTER (WHERE family(ip) = 4)"),
-                sql::<BigInt>("count(*) FILTER (WHERE family(ip) = 6)"),
+                sql::<BigInt>(
+                    "count(distinct ip) FILTER (WHERE family(ip) = 4)",
+                ),
+                sql::<BigInt>(
+                    "count(distinct ip) FILTER (WHERE family(ip) = 6)",
+                ),
             ))
             .first_async::<(i64, i64)>(
                 &*self.pool_connection_authorized(opctx).await?,
diff --git a/nexus/tests/integration_tests/external_ips.rs b/nexus/tests/integration_tests/external_ips.rs
index fa884f8731..203a1f35f7 100644
--- a/nexus/tests/integration_tests/external_ips.rs
+++ b/nexus/tests/integration_tests/external_ips.rs
@@ -654,8 +654,8 @@ async fn test_external_ip_live_attach_detach(
     }
 
     // the two instances above were deliberately not given ephemeral IPs, but
-    // they still always get SNAT IPs, so we went from 2 to 4
-    assert_ip_pool_utilization(client, "default", 4, 65536, 0, 0).await;
+    // they still always get SNAT IPs, but they share one, so we go from 2 to 3
+    assert_ip_pool_utilization(client, "default", 3, 65536, 0, 0).await;
 
     // Attach a floating IP and ephemeral IP to each instance.
     let mut recorded_ephs = vec![];
@@ -699,9 +699,9 @@ async fn test_external_ip_live_attach_detach(
         recorded_ephs.push(eph_resp);
     }
 
-    // now 6 because an ephemeral IP was added for each instance. floating IPs
+    // now 5 because an ephemeral IP was added for each instance. floating IPs
     // were attached, but they were already allocated
-    assert_ip_pool_utilization(client, "default", 6, 65536, 0, 0).await;
+    assert_ip_pool_utilization(client, "default", 5, 65536, 0, 0).await;
 
     // Detach a floating IP and ephemeral IP from each instance.
     for (instance, fip) in instances.iter().zip(&fips) {
@@ -733,8 +733,8 @@ async fn test_external_ip_live_attach_detach(
         );
     }
 
-    // 2 ephemeral go away on detachment but still 2 floating and 2 SNAT
-    assert_ip_pool_utilization(client, "default", 4, 65536, 0, 0).await;
+    // 2 ephemeral go away on detachment but still 2 floating and 1 SNAT
+    assert_ip_pool_utilization(client, "default", 3, 65536, 0, 0).await;
 
     // Finally, two kind of funny tests. There is special logic in the handler
     // for the case where the floating IP is specified by name but the instance
@@ -796,7 +796,7 @@ async fn test_external_ip_live_attach_detach(
     assert_eq!(eip_list[0].ip(), fips[1].ip);
 
     // none of that changed the number of allocated IPs
-    assert_ip_pool_utilization(client, "default", 4, 65536, 0, 0).await;
+    assert_ip_pool_utilization(client, "default", 3, 65536, 0, 0).await;
 }
 
 #[nexus_test]
diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs
index 128ed8967b..d5c4a1e7af 100644
--- a/nexus/tests/integration_tests/instances.rs
+++ b/nexus/tests/integration_tests/instances.rs
@@ -3950,9 +3950,10 @@ async fn test_instance_ephemeral_ip_from_correct_pool(
         "Expected ephemeral IP to come from pool2"
     );
 
-    // added 1 snat because snat comes from default pool.
-    // https://github.com/oxidecomputer/omicron/issues/5043 is about changing that
-    assert_ip_pool_utilization(client, "pool1", 3, 5, 0, 0).await;
+    // SNAT comes from default pool, but count does not change because
+    // SNAT IPs can be shared. https://github.com/oxidecomputer/omicron/issues/5043
+    // is about getting SNAT IP from specified pool instead of default.
+    assert_ip_pool_utilization(client, "pool1", 2, 5, 0, 0).await;
 
     // ephemeral IP comes from specified pool
     assert_ip_pool_utilization(client, "pool2", 1, 5, 0, 0).await;
@@ -3973,7 +3974,7 @@ async fn test_instance_ephemeral_ip_from_correct_pool(
     );
 
     // pool1 unchanged
-    assert_ip_pool_utilization(client, "pool1", 3, 5, 0, 0).await;
+    assert_ip_pool_utilization(client, "pool1", 2, 5, 0, 0).await;
     // +1 snat (now that pool2 is default) and +1 ephemeral
     assert_ip_pool_utilization(client, "pool2", 3, 5, 0, 0).await;
 

From 9d227640770a52ab19a0d98638bf5be1d538c840 Mon Sep 17 00:00:00 2001
From: David Crespo <david-crespo@users.noreply.github.com>
Date: Thu, 21 Mar 2024 22:39:22 -0500
Subject: [PATCH 146/157] 'List all' -> 'List' in API endpoint summaries
 (#5308)

Just a small thing I noticed!
---
 nexus/src/external_api/http_entrypoints.rs | 4 ++--
 openapi/nexus.json                         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs
index 41e5bce08a..551ef00817 100644
--- a/nexus/src/external_api/http_entrypoints.rs
+++ b/nexus/src/external_api/http_entrypoints.rs
@@ -1371,7 +1371,7 @@ async fn project_policy_update(
 
 // IP Pools
 
-/// List all IP pools
+/// List IP pools
 #[endpoint {
     method = GET,
     path = "/v1/ip-pools",
@@ -1927,7 +1927,7 @@ async fn ip_pool_service_range_remove(
 
 // Floating IP Addresses
 
-/// List all floating IPs
+/// List floating IPs
 #[endpoint {
     method = GET,
     path = "/v1/floating-ips",
diff --git a/openapi/nexus.json b/openapi/nexus.json
index 5ea1a48d8b..9c39829a3b 100644
--- a/openapi/nexus.json
+++ b/openapi/nexus.json
@@ -1009,7 +1009,7 @@
         "tags": [
           "floating-ips"
         ],
-        "summary": "List all floating IPs",
+        "summary": "List floating IPs",
         "operationId": "floating_ip_list",
         "parameters": [
           {
@@ -2681,7 +2681,7 @@
         "tags": [
           "projects"
         ],
-        "summary": "List all IP pools",
+        "summary": "List IP pools",
         "operationId": "project_ip_pool_list",
         "parameters": [
           {

From ac4f2b0b6c654790e6f6d86e488ae459d2ceafa7 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Thu, 21 Mar 2024 23:13:12 -0700
Subject: [PATCH 147/157] [nexus] During DB schema update, avoid re-applying
 sub-updates (#5293)

Fixes https://github.com/oxidecomputer/omicron/issues/5292
---
 Cargo.lock                                    |   1 +
 dev-tools/omdb/src/bin/omdb/db.rs             |   9 +-
 nexus/db-queries/Cargo.toml                   |   1 +
 .../src/db/datastore/db_metadata.rs           | 439 +++++++++++++++---
 nexus/db-queries/src/db/datastore/mod.rs      |   2 +-
 nexus/src/bin/schema-updater.rs               |   7 +-
 nexus/tests/integration_tests/schema.rs       |  31 +-
 7 files changed, 416 insertions(+), 74 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 5b034fee62..16e684d422 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4491,6 +4491,7 @@ dependencies = [
  "rustls 0.22.2",
  "samael",
  "schemars",
+ "semver 1.0.22",
  "serde",
  "serde_json",
  "serde_urlencoded",
diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs
index 9e0632d722..a4243fff31 100644
--- a/dev-tools/omdb/src/bin/omdb/db.rs
+++ b/dev-tools/omdb/src/bin/omdb/db.rs
@@ -587,7 +587,14 @@ async fn check_schema_version(datastore: &DataStore) {
     let version_check = datastore.database_schema_version().await;
 
     match version_check {
-        Ok(found_version) => {
+        Ok((found_version, found_target)) => {
+            if let Some(target) = found_target {
+                eprintln!(
+                    "note: database schema target exists (mid-upgrade?) ({})",
+                    target
+                );
+            }
+
             if found_version == expected_version {
                 eprintln!(
                     "note: database schema version matches expected ({})",
diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml
index 9bb7040af9..595280780e 100644
--- a/nexus/db-queries/Cargo.toml
+++ b/nexus/db-queries/Cargo.toml
@@ -36,6 +36,7 @@ rand.workspace = true
 ref-cast.workspace = true
 samael.workspace = true
 schemars.workspace = true
+semver.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 serde_urlencoded.workspace = true
diff --git a/nexus/db-queries/src/db/datastore/db_metadata.rs b/nexus/db-queries/src/db/datastore/db_metadata.rs
index e6b57c1c72..4169cc06bd 100644
--- a/nexus/db-queries/src/db/datastore/db_metadata.rs
+++ b/nexus/db-queries/src/db/datastore/db_metadata.rs
@@ -13,13 +13,94 @@ use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection};
 use chrono::Utc;
 use diesel::prelude::*;
 use nexus_db_model::AllSchemaVersions;
+use nexus_db_model::SchemaUpgradeStep;
 use nexus_db_model::SchemaVersion;
 use nexus_db_model::EARLIEST_SUPPORTED_VERSION;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::SemverVersion;
 use slog::{error, info, o, Logger};
+use std::ops::Bound;
 use std::str::FromStr;
 
+// A SchemaVersion which uses a pre-release value to indicate
+// "incremental progress".
+//
+// Although we organize our schema changes into higher-level versions,
+// transaction semantics of CockroachDB do not allow multiple DDL statements
+// to be applied together. As a result, an upgrade from "1.0.0" to "2.0.0"
+// may involve multiple smaller "step" versions, each separated by
+// an auto-generated pre-release version.
+#[derive(Clone)]
+struct StepSemverVersion {
+    version: SemverVersion,
+    i: usize,
+}
+
+impl StepSemverVersion {
+    // Durably store our progress in the target version as the
+    // pre-release extension of "target_version".
+    //
+    // Note that the format of "step.{i}" is load-bearing, see:
+    // https://docs.rs/semver/1.0.22/semver/struct.Prerelease.html#examples
+    //
+    // By using "dot number" notation, we can order the pre-release
+    // steps, which matters while comparing their order.
+    fn new(target_version: &SemverVersion, i: usize) -> anyhow::Result<Self> {
+        let mut target_step_version = target_version.clone();
+        target_step_version.0.pre =
+            semver::Prerelease::new(&format!("step.{i}"))
+                .context("Cannot parse step as semver pre-release")?;
+        Ok(Self { version: target_step_version, i })
+    }
+
+    // Drops the pre-release information about this target version.
+    //
+    // This is the version we are upgrading to, using these incremental steps.
+    fn without_prerelease(&self) -> SemverVersion {
+        let mut target_version = self.version.clone();
+        target_version.0.pre = semver::Prerelease::EMPTY;
+        target_version
+    }
+
+    // Returns the 'previous' step before this one, if such a step exists.
+    fn previous(&self) -> Option<Self> {
+        if self.i == 0 {
+            return None;
+        }
+        Self::new(&self.version, self.i - 1).ok()
+    }
+}
+
+// Identifies if we have already completed the "target_step_version", by
+// comparing with the `target_version` value stored in the DB as
+// "found_target_version".
+fn skippable_version(
+    log: &Logger,
+    target_step_version: &SemverVersion,
+    found_target_version: &Option<SemverVersion>,
+) -> bool {
+    if let Some(found_target_version) = found_target_version.as_ref() {
+        info!(
+            log,
+            "Considering found target version";
+            "found_target_version" => ?found_target_version,
+        );
+
+        // This case only occurs if an upgrade failed and needed to
+        // restart, for whatever reason. We skip all the incremental
+        // steps that we know have completed.
+        if found_target_version > target_step_version {
+            warn!(
+                log,
+                "Observed target version greater than this upgrade step. Skipping.";
+                "found_target_version" => %found_target_version,
+            );
+            return true;
+        }
+    }
+    return false;
+}
+
 impl DataStore {
     // Ensures that the database schema matches "desired_version".
     //
@@ -41,10 +122,11 @@ impl DataStore {
         desired_version: SemverVersion,
         all_versions: Option<&AllSchemaVersions>,
     ) -> Result<(), anyhow::Error> {
-        let found_version = self
+        let (found_version, found_target_version) = self
             .database_schema_version()
             .await
             .context("Cannot read database schema version")?;
+
         let log = log.new(o!(
             "found_version" => found_version.to_string(),
             "desired_version" => desired_version.to_string(),
@@ -104,9 +186,16 @@ impl DataStore {
         );
 
         let target_versions: Vec<&SchemaVersion> = all_versions
-            .versions_range(&found_version..=&desired_version)
+            .versions_range((
+                Bound::Excluded(&found_version),
+                Bound::Included(&desired_version),
+            ))
             .collect();
 
+        // Iterate over each of the higher-level user-defined versions.
+        //
+        // These are the user-defined `KNOWN_VERSIONS` defined in
+        // nexus/db-model/src/schema_versions.rs.
         let mut current_version = found_version;
         for target_version in target_versions.into_iter() {
             let log = log.new(o!(
@@ -115,37 +204,42 @@ impl DataStore {
             ));
             info!(log, "Attempting to upgrade schema");
 
-            // Confirm the current version, set the "target_version"
-            // column to indicate that a schema update is in-progress.
-            //
-            // Sets the following:
-            // - db_metadata.target_version = new version
-            self.prepare_schema_update(
-                &current_version,
-                &target_version.semver(),
-            )
-            .await
-            .context("preparing schema update")?;
-            info!(log, "Marked schema upgrade as prepared");
+            // For the rationale here, see: StepSemverVersion::new.
+            if target_version.semver().0.pre != semver::Prerelease::EMPTY {
+                bail!("Cannot upgrade to version which includes pre-release");
+            }
 
-            for step in target_version.upgrade_steps() {
-                // Perform the schema change step.
-                self.apply_schema_update(
+            // Iterate over each individual file that comprises a schema change.
+            //
+            // For example, this contains "up01.sql", "up02.sql", etc.
+            //
+            // While this update happens, the "current_version" will remain
+            // the same (unless another Nexus concurrently completes the
+            // update), but the "target_version" will keep shifting on each
+            // incremental step.
+            let mut last_step_version = None;
+
+            for (i, step) in target_version.upgrade_steps().enumerate() {
+                let target_step =
+                    StepSemverVersion::new(&target_version.semver(), i)?;
+                let log = log.new(o!("target_step.version" => target_step.version.to_string()));
+
+                self.apply_step_version_update(
+                    &log,
+                    &step,
+                    &target_step,
                     &current_version,
-                    &target_version.semver(),
-                    step.sql(),
+                    &found_target_version,
                 )
-                .await
-                .with_context(|| {
-                    format!(
-                        "update to {}, applying step {:?}",
-                        target_version.semver(),
-                        step.label()
-                    )
-                })?;
+                .await?;
+
+                last_step_version = Some(target_step.clone());
             }
 
-            info!(log, "Applied schema upgrade");
+            info!(
+                log,
+                "Applied schema upgrade";
+            );
 
             // NOTE: We could execute the schema change in a background task,
             // and let it propagate, while observing it with the following
@@ -169,12 +263,11 @@ impl DataStore {
             // Now that the schema change has completed, set the following:
             // - db_metadata.version = new version
             // - db_metadata.target_version = NULL
-            self.finalize_schema_update(
-                &current_version,
-                target_version.semver(),
-            )
-            .await
-            .context("finalizing schema update")?;
+            let last_step_version = last_step_version
+                .ok_or_else(|| anyhow::anyhow!("Missing final step version"))?;
+            self.finalize_schema_update(&current_version, &last_step_version)
+                .await
+                .context("Failed to finalize schema update")?;
 
             info!(
                 log,
@@ -185,31 +278,105 @@ impl DataStore {
         }
 
         info!(log, "Schema update complete");
+        Ok(())
+    }
+
+    // Executes (or skips, if unnecessary) a single "step" of a schema upgrade.
+    //
+    // - `step`: The schema upgrade step under consideration.
+    // - `target_step`: The target version, indicating the step by pre-release.
+    // - `current_version`: The last-known value of `db_metadata.version`.
+    // - `found_target_version`: The last-known value of
+    // `db_metadata.target_version`.
+    async fn apply_step_version_update(
+        &self,
+        log: &Logger,
+        step: &SchemaUpgradeStep,
+        target_step: &StepSemverVersion,
+        current_version: &SemverVersion,
+        found_target_version: &Option<SemverVersion>,
+    ) -> Result<(), anyhow::Error> {
+        if skippable_version(&log, &target_step.version, &found_target_version)
+        {
+            return Ok(());
+        }
+
+        info!(
+            log,
+            "Marking schema upgrade as prepared";
+        );
+
+        // Confirm the current version, set the "target_version"
+        // column to indicate that a schema update is in-progress.
+        //
+        // Sets the following:
+        // - db_metadata.target_version = new version
+        self.prepare_schema_update(&current_version, &target_step)
+            .await
+            .context("Failed to prepare schema change")?;
+
+        info!(
+            log,
+            "Marked schema upgrade as prepared";
+        );
 
+        // Perform the schema change.
+        self.apply_schema_update(
+            &current_version,
+            &target_step.version,
+            step.sql(),
+        )
+        .await
+        .with_context(|| {
+            format!(
+                "update to {}, applying step {:?}",
+                target_step.version,
+                step.label()
+            )
+        })?;
+
+        info!(
+            log,
+            "Applied subcomponent of schema upgrade";
+        );
         Ok(())
     }
 
     pub async fn database_schema_version(
         &self,
-    ) -> Result<SemverVersion, Error> {
+    ) -> Result<(SemverVersion, Option<SemverVersion>), Error> {
         use db::schema::db_metadata::dsl;
 
-        let version: String = dsl::db_metadata
+        let (version, target): (String, Option<String>) = dsl::db_metadata
             .filter(dsl::singleton.eq(true))
-            .select(dsl::version)
+            .select((dsl::version, dsl::target_version))
             .get_result_async(&*self.pool_connection_unauthorized().await?)
             .await
             .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?;
 
-        SemverVersion::from_str(&version).map_err(|e| {
+        let version = SemverVersion::from_str(&version).map_err(|e| {
             Error::internal_error(&format!("Invalid schema version: {e}"))
-        })
+        })?;
+
+        if let Some(target) = target {
+            let target = SemverVersion::from_str(&target).map_err(|e| {
+                Error::internal_error(&format!("Invalid schema version: {e}"))
+            })?;
+            return Ok((version, Some(target)));
+        };
+
+        Ok((version, None))
     }
 
     // Updates the DB metadata to indicate that a transition from
-    // `from_version` to `to_version` is occuring.
+    // `from_version` to `to_version` is occurring.
     //
-    // This is only valid if the current version matches `from_version`.
+    // This is only valid if the current version matches `from_version`,
+    // and the prior `target_version` is either:
+    // - None (no update in-progress)
+    // - target_step.version (another Nexus attempted to prepare this same update).
+    // - target_step.previous() (we are incrementally working through a
+    // multi-stage update).
     //
     // NOTE: This function should be idempotent -- if Nexus crashes mid-update,
     // a new Nexus instance should be able to re-call this function and
@@ -217,10 +384,15 @@ impl DataStore {
     async fn prepare_schema_update(
         &self,
         from_version: &SemverVersion,
-        to_version: &SemverVersion,
+        target_step: &StepSemverVersion,
     ) -> Result<(), Error> {
         use db::schema::db_metadata::dsl;
 
+        let mut valid_prior_targets = vec![target_step.version.to_string()];
+        if let Some(previous) = target_step.previous() {
+            valid_prior_targets.push(previous.version.to_string());
+        };
+
         let rows_updated = diesel::update(
             dsl::db_metadata
                 .filter(dsl::singleton.eq(true))
@@ -229,13 +401,13 @@ impl DataStore {
                 // in-progress.
                 .filter(
                     dsl::target_version
-                        .eq(Some(to_version.to_string()))
+                        .eq_any(valid_prior_targets)
                         .or(dsl::target_version.is_null()),
                 ),
         )
         .set((
             dsl::time_modified.eq(Utc::now()),
-            dsl::target_version.eq(Some(to_version.to_string())),
+            dsl::target_version.eq(Some(target_step.version.to_string())),
         ))
         .execute_async(&*self.pool_connection_unauthorized().await?)
         .await
@@ -285,18 +457,22 @@ impl DataStore {
     }
 
     // Completes a schema migration, upgrading to the new version.
+    //
+    // - from_version: What we expect "version" must be to proceed
+    // - last_step: What we expect "target_version" must be to proceed.
     async fn finalize_schema_update(
         &self,
         from_version: &SemverVersion,
-        to_version: &SemverVersion,
+        last_step: &StepSemverVersion,
     ) -> Result<(), Error> {
         use db::schema::db_metadata::dsl;
 
+        let to_version = last_step.without_prerelease();
         let rows_updated = diesel::update(
             dsl::db_metadata
                 .filter(dsl::singleton.eq(true))
                 .filter(dsl::version.eq(from_version.to_string()))
-                .filter(dsl::target_version.eq(to_version.to_string())),
+                .filter(dsl::target_version.eq(last_step.version.to_string())),
         )
         .set((
             dsl::time_modified.eq(Utc::now()),
@@ -320,6 +496,7 @@ impl DataStore {
 #[cfg(test)]
 mod test {
     use super::*;
+    use camino::Utf8Path;
     use camino_tempfile::Utf8TempDir;
     use nexus_db_model::SCHEMA_VERSION;
     use nexus_test_utils::db as test_db;
@@ -347,6 +524,30 @@ mod test {
         logctx.cleanup_successful();
     }
 
+    // Helper to create the version directory and "up.sql".
+    async fn add_upgrade<S: AsRef<str>>(
+        config_dir_path: &Utf8Path,
+        version: SemverVersion,
+        sql: S,
+    ) {
+        let dir = config_dir_path.join(version.to_string());
+        tokio::fs::create_dir_all(&dir).await.unwrap();
+        tokio::fs::write(dir.join("up.sql"), sql.as_ref()).await.unwrap();
+    }
+
+    async fn add_upgrade_subcomponent<S: AsRef<str>>(
+        config_dir_path: &Utf8Path,
+        version: SemverVersion,
+        sql: S,
+        i: usize,
+    ) {
+        let dir = config_dir_path.join(version.to_string());
+        tokio::fs::create_dir_all(&dir).await.unwrap();
+        tokio::fs::write(dir.join(format!("up{i}.sql")), sql.as_ref())
+            .await
+            .unwrap();
+    }
+
     // Confirms that calling ensure_schema from concurrent Nexus instances
     // only permit the latest schema migration, rather than re-applying old
     // schema updates.
@@ -364,17 +565,6 @@ mod test {
         // Mimic the layout of "schema/crdb".
         let config_dir = Utf8TempDir::new().unwrap();
 
-        // Helper to create the version directory and "up.sql".
-        let add_upgrade = |version: SemverVersion, sql: String| {
-            let config_dir_path = config_dir.path();
-            async move {
-                let dir = config_dir_path.join(version.to_string());
-                tokio::fs::create_dir_all(&dir).await.unwrap();
-
-                tokio::fs::write(dir.join("up.sql"), sql).await.unwrap();
-            }
-        };
-
         // Create the old version directory, and also update the on-disk
         // "current version" to this value.
         //
@@ -399,24 +589,28 @@ mod test {
 
         // This version must exist so Nexus can see the sequence of updates from
         // v0 to v1 to v2, but it doesn't need to re-apply it.
-        add_upgrade(v0.clone(), "SELECT true;".to_string()).await;
+        add_upgrade(config_dir.path(), v0.clone(), "SELECT true;").await;
 
         // This version adds a new table, but it takes a little while.
         //
         // This delay is intentional, so that some Nexus instances issuing
         // the update act quickly, while others lag behind.
         add_upgrade(
+            config_dir.path(),
             v1.clone(),
             "SELECT pg_sleep(RANDOM() / 10); \
              CREATE TABLE IF NOT EXISTS widget(); \
-             SELECT pg_sleep(RANDOM() / 10);"
-                .to_string(),
+             SELECT pg_sleep(RANDOM() / 10);",
         )
         .await;
 
         // The table we just created is deleted by a subsequent update.
-        add_upgrade(v2.clone(), "DROP TABLE IF EXISTS widget;".to_string())
-            .await;
+        add_upgrade(
+            config_dir.path(),
+            v2.clone(),
+            "DROP TABLE IF EXISTS widget;",
+        )
+        .await;
 
         // Show that the datastores can be created concurrently.
         let all_versions = AllSchemaVersions::load_specific_legacy_versions(
@@ -468,4 +662,125 @@ mod test {
         crdb.cleanup().await.unwrap();
         logctx.cleanup_successful();
     }
+
+    #[tokio::test]
+    async fn schema_version_subcomponents_save_progress() {
+        let logctx =
+            dev::test_setup_log("schema_version_subcomponents_save_progress");
+        let log = &logctx.log;
+        let mut crdb = test_db::test_setup_database(&logctx.log).await;
+
+        let cfg = db::Config { url: crdb.pg_config().clone() };
+        let pool = Arc::new(db::Pool::new(&logctx.log, &cfg));
+        let conn = pool.pool().get().await.unwrap();
+
+        // Mimic the layout of "schema/crdb".
+        let config_dir = Utf8TempDir::new().unwrap();
+
+        // Create the old version directory, and also update the on-disk "current version" to
+        // this value.
+        //
+        // Nexus will decide to upgrade to, at most, the version that its own binary understands.
+        //
+        // To trigger this action within a test, we manually set the "known to DB" version.
+        let v0 = SemverVersion::new(0, 0, 0);
+        use db::schema::db_metadata::dsl;
+        diesel::update(dsl::db_metadata.filter(dsl::singleton.eq(true)))
+            .set(dsl::version.eq(v0.to_string()))
+            .execute_async(&*conn)
+            .await
+            .expect("Failed to set version back to 0.0.0");
+
+        let v1 = SemverVersion::new(1, 0, 0);
+        let v2 = SCHEMA_VERSION;
+
+        assert!(v0 < v1);
+        assert!(v1 < v2);
+
+        // This version must exist so Nexus can see the sequence of updates from
+        // v0 to v1 to v2, but it doesn't need to re-apply it.
+        add_upgrade(config_dir.path(), v0.clone(), "SELECT true;").await;
+
+        // Add a new version with a table that we'll populate in V2.
+        add_upgrade(
+            config_dir.path(),
+            v1.clone(),
+            "CREATE TABLE t(id INT PRIMARY KEY, data TEXT NOT NULL);",
+        )
+        .await;
+
+        // Populate the table in a few incremental steps that might fail.
+        let add = |sql: &str, i: usize| {
+            let config_dir_path = config_dir.path();
+            let v2 = v2.clone();
+            let sql = sql.to_string();
+            async move {
+                add_upgrade_subcomponent(&config_dir_path, v2.clone(), &sql, i)
+                    .await
+            }
+        };
+
+        // This is just:
+        //   data = 'abcd',
+        // but with some spurious errors thrown in for good measure.
+        //
+        // Note that if we "retry the steps from the beginning", this would
+        // likely append the wrong string. This is intentional! If this occurs,
+        // we'd like to catch such a case.
+        //
+        // Normally, we expect that each of these steps should be idempotent (by
+        // themselves) but in this case, we're controlling the test environment,
+        // and we should not expect any of these steps to be replayed once they
+        // succeed.
+        add("INSERT INTO t (id, data) VALUES (1, '')", 1).await;
+        add("UPDATE t SET data = data || 'a' WHERE id = 1", 2).await;
+        add("SELECT CAST(IF ((SELECT RANDOM() < 0.5), 'true', 'failure') AS BOOL)", 3).await;
+        add("UPDATE t SET data = data || 'b' WHERE id = 1", 4).await;
+        add("SELECT CAST(IF ((SELECT RANDOM() < 0.5), 'true', 'failure') AS BOOL)", 5).await;
+        add("SELECT CAST(IF ((SELECT RANDOM() < 0.5), 'true', 'failure') AS BOOL)", 6).await;
+        add("UPDATE t SET data = data || 'c' WHERE id = 1", 7).await;
+        add("SELECT CAST(IF ((SELECT RANDOM() < 0.5), 'true', 'failure') AS BOOL)", 8).await;
+        add("SELECT CAST(IF ((SELECT RANDOM() < 0.5), 'true', 'failure') AS BOOL)", 9).await;
+        add("SELECT CAST(IF ((SELECT RANDOM() < 0.5), 'true', 'failure') AS BOOL)", 10).await;
+        add("UPDATE t SET data = data || 'd' WHERE id = 1", 11).await;
+
+        // Create the datastore, which should apply the update on boot
+        let all_versions = AllSchemaVersions::load_specific_legacy_versions(
+            config_dir.path(),
+            [&v0, &v1, &v2].into_iter(),
+        )
+        .expect("failed to load schema");
+
+        // Manually construct the datastore to avoid the backoff timeout.
+        // We want to trigger errors, but have no need to wait.
+        let datastore =
+            DataStore::new_unchecked(log.clone(), pool.clone()).unwrap();
+        while let Err(e) = datastore
+            .ensure_schema(&log, SCHEMA_VERSION, Some(&all_versions))
+            .await
+        {
+            warn!(log, "Failed to ensure schema"; "err" => %e);
+            continue;
+        }
+        let conn = datastore.pool_connection_for_tests().await.unwrap();
+
+        // Let's validate a couple things:
+        // 1. The version is what we expect
+        use diesel::dsl::sql;
+        use diesel::sql_types::Text;
+        let version = sql::<Text>("SELECT version FROM omicron.public.db_metadata WHERE singleton = true")
+            .get_result_async::<String>(&*conn)
+            .await
+            .expect("Failed to get DB version");
+        assert_eq!(version, SCHEMA_VERSION.to_string());
+        // 2. We only applied each incremental step once
+        let data = sql::<Text>("SELECT data FROM t WHERE id = 1")
+            .get_result_async::<String>(&*conn)
+            .await
+            .expect("Failed to get data");
+        assert_eq!(data, "abcd");
+
+        crdb.cleanup().await.unwrap();
+        logctx.cleanup_successful();
+    }
 }
diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs
index 8ae245aaa0..2a3f969183 100644
--- a/nexus/db-queries/src/db/datastore/mod.rs
+++ b/nexus/db-queries/src/db/datastore/mod.rs
@@ -209,7 +209,7 @@ impl DataStore {
                 {
                     Ok(()) => return Ok(()),
                     Err(e) => {
-                        warn!(log, "Failed to ensure schema version: {e}");
+                        warn!(log, "Failed to ensure schema version"; "error" => #%e);
                     }
                 };
                 return Err(BackoffError::transient(()));
diff --git a/nexus/src/bin/schema-updater.rs b/nexus/src/bin/schema-updater.rs
index 2932a819d8..7fe1ed84a4 100644
--- a/nexus/src/bin/schema-updater.rs
+++ b/nexus/src/bin/schema-updater.rs
@@ -82,13 +82,14 @@ async fn main() -> anyhow::Result<()> {
 
     match args.cmd {
         Cmd::List => {
-            let current_version = datastore
+            let (current_version, target_version) = datastore
                 .database_schema_version()
                 .await
-                .map(|v| v.to_string())
-                .unwrap_or_else(|_| "Unknown".to_string());
+                .map(|(v, t)| (v.to_string(), t.map(|t| t.to_string())))
+                .unwrap_or_else(|_| ("Unknown".to_string(), None));
 
             println!("Current Version in database: {current_version}");
+            println!("Target Version in database: {target_version:?}");
             println!("Known Versions:");
             for version in all_versions.iter_versions() {
                 let mut extra = String::new();
diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs
index dbe436a8f2..44c91bbacf 100644
--- a/nexus/tests/integration_tests/schema.rs
+++ b/nexus/tests/integration_tests/schema.rs
@@ -138,14 +138,31 @@ async fn apply_update(
             .expect("Failed to bump version number");
     }
 
-    for _ in 0..times_to_apply {
-        for step in version.upgrade_steps() {
-            info!(
-                log,
-                "Applying sql schema upgrade step";
-                "file" => step.label()
-            );
+    // Each step is applied `times_to_apply` times, but once we start applying
+    // a step within an upgrade, we will not attempt to apply prior steps.
+    for step in version.upgrade_steps() {
+        info!(
+            log,
+            "Applying sql schema upgrade step";
+            "file" => step.label()
+        );
+
+        for _ in 0..times_to_apply {
             apply_update_as_transaction(&log, &client, step.sql()).await;
+
+            // The following is a set of "versions exempt from being
+            // re-applied" multiple times. PLEASE AVOID ADDING TO THIS LIST.
+            const NOT_IDEMPOTENT_VERSIONS: [semver::Version; 1] = [
+                // Why: This calls "ALTER TYPE ... DROP VALUE", which does not
+                // support the "IF EXISTS" syntax in CockroachDB.
+                //
+                // https://github.com/cockroachdb/cockroach/issues/120801
+                semver::Version::new(10, 0, 0),
+            ];
+
+            if NOT_IDEMPOTENT_VERSIONS.contains(&version.semver().0) {
+                break;
+            }
         }
     }
 

From 5586f3a3748df48bbc5b4913d5984f12b8337147 Mon Sep 17 00:00:00 2001
From: David Crespo <david-crespo@users.noreply.github.com>
Date: Fri, 22 Mar 2024 12:57:29 -0500
Subject: [PATCH 148/157] Bump web console (IAM role column) (#5309)

https://github.com/oxidecomputer/console/compare/7c968443...0ed9587b

* [0ed9587b](https://github.com/oxidecomputer/console/commit/0ed9587b)
turn accessTypeLabel and getBadgeColor functions into records
* [420174b4](https://github.com/oxidecomputer/console/commit/420174b4)
oxidecomputer/console#2062
* [fc860a25](https://github.com/oxidecomputer/console/commit/fc860a25)
oxidecomputer/console#2090
---
 tools/console_version | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/console_version b/tools/console_version
index 0946febc50..dbacfcbdd0 100644
--- a/tools/console_version
+++ b/tools/console_version
@@ -1,2 +1,2 @@
-COMMIT="7c968443ce03e3fbadb8896f80fb9b156b46e3dd"
-SHA2="e6eb2df59804c0c85d8c0a262be00d3fc32668ef19abde0c1858e333f264b0c8"
+COMMIT="0ed9587b1966b1f81d67a3def7061cfa846801fd"
+SHA2="c2c4fc55693be9c50c5aca3a2def7f952a9b78ade2bd539c8a607b9dda7add2d"

From 00deaced68ac12e385ada9b12dddf98f32f85d93 Mon Sep 17 00:00:00 2001
From: David Crespo <david-crespo@users.noreply.github.com>
Date: Fri, 22 Mar 2024 17:36:26 -0500
Subject: [PATCH 149/157] Bump web console (fix instance create confirm flash)
 (#5310)

https://github.com/oxidecomputer/console/compare/0ed9587b...01142889

* [01142889](https://github.com/oxidecomputer/console/commit/01142889)
oxidecomputer/console#2094
---
 tools/console_version | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/console_version b/tools/console_version
index dbacfcbdd0..38a4cab257 100644
--- a/tools/console_version
+++ b/tools/console_version
@@ -1,2 +1,2 @@
-COMMIT="0ed9587b1966b1f81d67a3def7061cfa846801fd"
-SHA2="c2c4fc55693be9c50c5aca3a2def7f952a9b78ade2bd539c8a607b9dda7add2d"
+COMMIT="0114288974fb9e36bf392277fe59b0de357ec137"
+SHA2="12e4e0205665127b6f99911484b7d61e50ef740f7a94ac6c95e11918f0ce7ada"

From 26fad6980305b6925146a596c85a53c02b9aad35 Mon Sep 17 00:00:00 2001
From: Ryan Goodfellow <ryan.goodfellow@oxide.computer>
Date: Fri, 22 Mar 2024 19:25:50 -0700
Subject: [PATCH 150/157] add bfd to bootstore and early networking (#5313)

---
 clients/sled-agent-client/src/lib.rs          | 17 ++++-
 common/src/api/external/mod.rs                | 19 ++++++
 common/src/api/internal/shared.rs             | 15 ++++-
 nexus/db-model/src/bfd.rs                     |  8 +--
 .../background/sync_switch_configuration.rs   | 65 ++++++++++++++++++-
 nexus/src/app/bfd.rs                          | 12 +++-
 nexus/src/lib.rs                              |  1 +
 nexus/tests/integration_tests/endpoints.rs    |  2 +-
 nexus/types/src/external_api/params.rs        | 20 +-----
 nexus/types/src/external_api/shared.rs        |  3 +-
 openapi/bootstrap-agent.json                  | 53 +++++++++++++++
 openapi/nexus-internal.json                   | 53 +++++++++++++++
 openapi/nexus.json                            |  1 +
 openapi/sled-agent.json                       | 53 +++++++++++++++
 schema/rss-sled-plan.json                     | 55 ++++++++++++++++
 sled-agent/src/bootstrap/early_networking.rs  | 34 +++++++++-
 sled-agent/src/bootstrap/params.rs            |  1 +
 sled-agent/src/rack_setup/config.rs           |  1 +
 sled-agent/src/rack_setup/plan/service.rs     |  1 +
 sled-agent/src/rack_setup/service.rs          | 19 ++++++
 sled-agent/src/sim/http_entrypoints.rs        |  1 +
 sled-agent/src/sim/server.rs                  |  1 +
 .../tests/integration_tests/early_network.rs  |  1 +
 .../madrid-rss-sled-plan.json                 |  3 +-
 wicketd/src/rss_config.rs                     |  2 +
 25 files changed, 407 insertions(+), 34 deletions(-)

diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs
index 2e9174c16c..17d1225c6e 100644
--- a/clients/sled-agent-client/src/lib.rs
+++ b/clients/sled-agent-client/src/lib.rs
@@ -11,7 +11,9 @@ use std::convert::TryFrom;
 use std::hash::Hash;
 use std::net::IpAddr;
 use std::net::SocketAddr;
-use types::{BgpConfig, BgpPeerConfig, PortConfigV1, RouteConfig};
+use types::{
+    BfdPeerConfig, BgpConfig, BgpPeerConfig, PortConfigV1, RouteConfig,
+};
 use uuid::Uuid;
 
 progenitor::generate_api!(
@@ -651,3 +653,16 @@ impl Hash for PortConfigV1 {
         self.uplink_port_speed.hash(state);
     }
 }
+
+impl Eq for BfdPeerConfig {}
+
+impl Hash for BfdPeerConfig {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        self.local.hash(state);
+        self.remote.hash(state);
+        self.detection_threshold.hash(state);
+        self.required_rx.hash(state);
+        self.mode.hash(state);
+        self.switch.hash(state);
+    }
+}
diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs
index 161e31cf87..324231f469 100644
--- a/common/src/api/external/mod.rs
+++ b/common/src/api/external/mod.rs
@@ -2821,6 +2821,25 @@ pub struct BgpImportedRouteIpv4 {
     pub switch: SwitchLocation,
 }
 
+/// BFD connection mode.
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    Deserialize,
+    Serialize,
+    JsonSchema,
+    PartialEq,
+    Eq,
+    Ord,
+    PartialOrd,
+)]
+#[serde(rename_all = "snake_case")]
+pub enum BfdMode {
+    SingleHop,
+    MultiHop,
+}
+
 /// A description of an uploaded TUF repository.
 #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema)]
 pub struct TufRepoDescription {
diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs
index 3fc1eb9879..c123e1f9c8 100644
--- a/common/src/api/internal/shared.rs
+++ b/common/src/api/internal/shared.rs
@@ -4,7 +4,7 @@
 
 //! Types shared between Nexus and Sled Agent.
 
-use crate::api::external::{self, Name};
+use crate::api::external::{self, BfdMode, Name};
 use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
@@ -87,6 +87,9 @@ pub struct RackNetworkConfigV1 {
     pub ports: Vec<PortConfigV1>,
     /// BGP configurations for connecting the rack to external networks
     pub bgp: Vec<BgpConfig>,
+    /// BFD configuration for connecting the rack to external networks
+    #[serde(default)]
+    pub bfd: Vec<BfdPeerConfig>,
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)]
@@ -118,6 +121,16 @@ pub struct BgpPeerConfig {
     pub keepalive: Option<u64>,
 }
 
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)]
+pub struct BfdPeerConfig {
+    pub local: Option<IpAddr>,
+    pub remote: IpAddr,
+    pub detection_threshold: u8,
+    pub required_rx: u64,
+    pub mode: BfdMode,
+    pub switch: SwitchLocation,
+}
+
 #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)]
 pub struct RouteConfig {
     /// The destination of the route.
diff --git a/nexus/db-model/src/bfd.rs b/nexus/db-model/src/bfd.rs
index b71136ea2c..8a41c4c4ad 100644
--- a/nexus/db-model/src/bfd.rs
+++ b/nexus/db-model/src/bfd.rs
@@ -45,13 +45,13 @@ pub struct BfdSession {
     pub time_deleted: Option<DateTime<Utc>>,
 }
 
-impl From<nexus_types::external_api::params::BfdMode> for BfdMode {
-    fn from(value: nexus_types::external_api::params::BfdMode) -> Self {
+impl From<omicron_common::api::external::BfdMode> for BfdMode {
+    fn from(value: omicron_common::api::external::BfdMode) -> Self {
         match value {
-            nexus_types::external_api::params::BfdMode::SingleHop => {
+            omicron_common::api::external::BfdMode::SingleHop => {
                 BfdMode::SingleHop
             }
-            nexus_types::external_api::params::BfdMode::MultiHop => {
+            omicron_common::api::external::BfdMode::MultiHop => {
                 BfdMode::MultiHop
             }
         }
diff --git a/nexus/src/app/background/sync_switch_configuration.rs b/nexus/src/app/background/sync_switch_configuration.rs
index 97a8dcd7e3..84cc450d83 100644
--- a/nexus/src/app/background/sync_switch_configuration.rs
+++ b/nexus/src/app/background/sync_switch_configuration.rs
@@ -39,7 +39,10 @@ use nexus_types::{external_api::params, identity::Resource};
 use omicron_common::OMICRON_DPD_TAG;
 use omicron_common::{
     address::{get_sled_address, Ipv6Subnet},
-    api::external::{DataPageParams, SwitchLocation},
+    api::{
+        external::{DataPageParams, SwitchLocation},
+        internal::shared::ParseSwitchLocationError,
+    },
 };
 use serde_json::json;
 use sled_agent_client::types::{
@@ -210,6 +213,56 @@ impl SwitchPortSettingsManager {
 
         Ok(set)
     }
+
+    async fn bfd_peer_configs_from_db<'a>(
+        &'a mut self,
+        opctx: &OpContext,
+    ) -> Result<
+        Vec<sled_agent_client::types::BfdPeerConfig>,
+        omicron_common::api::external::Error,
+    > {
+        let db_data = self
+            .datastore
+            .bfd_session_list(opctx, &DataPageParams::max_page())
+            .await?;
+
+        let mut result = Vec::new();
+        for spec in db_data.into_iter() {
+            let config = sled_agent_client::types::BfdPeerConfig {
+                local: spec.local.map(|x| x.ip()),
+                remote: spec.remote.ip(),
+                detection_threshold: spec.detection_threshold.0.try_into().map_err(|_| {
+                    omicron_common::api::external::Error::InternalError {
+                        internal_message: format!(
+                            "db_bfd_peer_configs: detection threshold overflow: {}",
+                            spec.detection_threshold.0,
+                        ),
+                    }
+                })?,
+                required_rx: spec.required_rx.0.into(),
+                mode: match spec.mode {
+                    nexus_db_model::BfdMode::SingleHop => {
+                        sled_agent_client::types::BfdMode::SingleHop
+                    }
+                    nexus_db_model::BfdMode::MultiHop => {
+                        sled_agent_client::types::BfdMode::MultiHop
+                    }
+                },
+                switch: spec.switch.parse().map_err(|e: ParseSwitchLocationError| {
+                    omicron_common::api::external::Error::InternalError {
+                        internal_message: format!(
+                            "db_bfd_peer_configs: failed to parse switch name: {}: {:?}",
+                            spec.switch,
+                            e,
+                        ),
+                    }
+                })?,
+            };
+            result.push(config);
+        }
+
+        Ok(result)
+    }
 }
 
 #[derive(Debug)]
@@ -799,6 +852,14 @@ impl BackgroundTask for SwitchPortSettingsManager {
                 ;
 
 
+                let bfd = match self.bfd_peer_configs_from_db(opctx).await {
+                    Ok(bfd) => bfd,
+                    Err(e) => {
+                        error!(log, "error fetching bfd config from db"; "error" => %e);
+                        continue;
+                    }
+                };
+
                 let mut desired_config = EarlyNetworkConfig {
                     generation: 0,
                     schema_version: 1,
@@ -810,6 +871,7 @@ impl BackgroundTask for SwitchPortSettingsManager {
                             infra_ip_last,
                             ports,
                             bgp,
+                            bfd,
                         }),
                     },
                 };
@@ -838,6 +900,7 @@ impl BackgroundTask for SwitchPortSettingsManager {
                                 let rnc_differs = match (config.body.rack_network_config.clone(), desired_config.body.rack_network_config.clone()) {
                                     (Some(current_rnc), Some(desired_rnc)) => {
                                         !hashset_eq(current_rnc.bgp.clone(), desired_rnc.bgp.clone()) ||
+                                        !hashset_eq(current_rnc.bfd.clone(), desired_rnc.bfd.clone()) ||
                                         !hashset_eq(current_rnc.ports.clone(), desired_rnc.ports.clone()) ||
                                         current_rnc.rack_subnet != desired_rnc.rack_subnet ||
                                         current_rnc.infra_ip_first != desired_rnc.infra_ip_first ||
diff --git a/nexus/src/app/bfd.rs b/nexus/src/app/bfd.rs
index d3d0a731d0..0afa238ee3 100644
--- a/nexus/src/app/bfd.rs
+++ b/nexus/src/app/bfd.rs
@@ -42,6 +42,10 @@ impl super::Nexus {
         self.background_tasks
             .driver
             .activate(&self.background_tasks.bfd_manager);
+        // for timely propagation to bootstore
+        self.background_tasks
+            .driver
+            .activate(&self.background_tasks.task_switch_port_settings_manager);
         Ok(())
     }
 
@@ -56,6 +60,10 @@ impl super::Nexus {
         self.background_tasks
             .driver
             .activate(&self.background_tasks.bfd_manager);
+        // for timely propagation to bootstore
+        self.background_tasks
+            .driver
+            .activate(&self.background_tasks.task_switch_port_settings_manager);
         Ok(())
     }
 
@@ -93,10 +101,10 @@ impl super::Nexus {
                     required_rx: info.config.required_rx,
                     mode: match info.config.mode {
                         mg_admin_client::types::SessionMode::SingleHop => {
-                            params::BfdMode::SingleHop
+                            omicron_common::api::external::BfdMode::SingleHop
                         }
                         mg_admin_client::types::SessionMode::MultiHop => {
-                            params::BfdMode::MultiHop
+                            omicron_common::api::external::BfdMode::MultiHop
                         }
                     },
                 })
diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs
index cad90aab01..bd5a13dfd1 100644
--- a/nexus/src/lib.rs
+++ b/nexus/src/lib.rs
@@ -302,6 +302,7 @@ impl nexus_test_interface::NexusServer for Server {
                         infra_ip_last: Ipv4Addr::UNSPECIFIED,
                         ports: Vec::new(),
                         bgp: Vec::new(),
+                        bfd: Vec::new(),
                     },
                 },
             )
diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs
index 254bdabe8b..b2b1e72c23 100644
--- a/nexus/tests/integration_tests/endpoints.rs
+++ b/nexus/tests/integration_tests/endpoints.rs
@@ -595,7 +595,7 @@ pub static DEMO_BFD_ENABLE: Lazy<params::BfdSessionEnable> =
         detection_threshold: 3,
         required_rx: 1000000,
         switch: "switch0".parse().unwrap(),
-        mode: params::BfdMode::MultiHop,
+        mode: omicron_common::api::external::BfdMode::MultiHop,
     });
 
 pub static DEMO_BFD_DISABLE: Lazy<params::BfdSessionDisable> =
diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs
index aab25df7e8..51d0869821 100644
--- a/nexus/types/src/external_api/params.rs
+++ b/nexus/types/src/external_api/params.rs
@@ -9,7 +9,7 @@ use crate::external_api::shared;
 use base64::Engine;
 use chrono::{DateTime, Utc};
 use omicron_common::api::external::{
-    AddressLotKind, ByteCount, Hostname, IdentityMetadataCreateParams,
+    AddressLotKind, BfdMode, ByteCount, Hostname, IdentityMetadataCreateParams,
     IdentityMetadataUpdateParams, InstanceCpuCount, IpNet, Ipv4Net, Ipv6Net,
     Name, NameOrId, PaginationOrder, RouteDestination, RouteTarget,
     SemverVersion,
@@ -1814,24 +1814,6 @@ pub struct BgpStatusSelector {
     pub name_or_id: NameOrId,
 }
 
-#[derive(
-    Clone,
-    Copy,
-    Debug,
-    Deserialize,
-    Serialize,
-    JsonSchema,
-    PartialEq,
-    Eq,
-    Ord,
-    PartialOrd,
-)]
-#[serde(rename_all = "snake_case")]
-pub enum BfdMode {
-    SingleHop,
-    MultiHop,
-}
-
 /// Information about a bidirectional forwarding detection (BFD) session.
 #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)]
 pub struct BfdSessionEnable {
diff --git a/nexus/types/src/external_api/shared.rs b/nexus/types/src/external_api/shared.rs
index 5fb7705631..2f65f09c29 100644
--- a/nexus/types/src/external_api/shared.rs
+++ b/nexus/types/src/external_api/shared.rs
@@ -17,8 +17,7 @@ use strum::EnumIter;
 use uuid::Uuid;
 
 pub use omicron_common::address::{IpRange, Ipv4Range, Ipv6Range};
-
-use super::params::BfdMode;
+pub use omicron_common::api::external::BfdMode;
 
 /// Maximum number of role assignments allowed on any one resource
 // Today's implementation assumes a relatively small number of role assignments
diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json
index a55803eda9..688e444053 100644
--- a/openapi/bootstrap-agent.json
+++ b/openapi/bootstrap-agent.json
@@ -229,6 +229,51 @@
           }
         ]
       },
+      "BfdMode": {
+        "description": "BFD connection mode.",
+        "type": "string",
+        "enum": [
+          "single_hop",
+          "multi_hop"
+        ]
+      },
+      "BfdPeerConfig": {
+        "type": "object",
+        "properties": {
+          "detection_threshold": {
+            "type": "integer",
+            "format": "uint8",
+            "minimum": 0
+          },
+          "local": {
+            "nullable": true,
+            "type": "string",
+            "format": "ip"
+          },
+          "mode": {
+            "$ref": "#/components/schemas/BfdMode"
+          },
+          "remote": {
+            "type": "string",
+            "format": "ip"
+          },
+          "required_rx": {
+            "type": "integer",
+            "format": "uint64",
+            "minimum": 0
+          },
+          "switch": {
+            "$ref": "#/components/schemas/SwitchLocation"
+          }
+        },
+        "required": [
+          "detection_threshold",
+          "mode",
+          "remote",
+          "required_rx",
+          "switch"
+        ]
+      },
       "BgpConfig": {
         "type": "object",
         "properties": {
@@ -691,6 +736,14 @@
         "description": "Initial network configuration",
         "type": "object",
         "properties": {
+          "bfd": {
+            "description": "BFD configuration for connecting the rack to external networks",
+            "default": [],
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/BfdPeerConfig"
+            }
+          },
           "bgp": {
             "description": "BGP configurations for connecting the rack to external networks",
             "type": "array",
diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json
index 3938d2f081..0beab820a1 100644
--- a/openapi/nexus-internal.json
+++ b/openapi/nexus-internal.json
@@ -1502,6 +1502,51 @@
           "serial"
         ]
       },
+      "BfdMode": {
+        "description": "BFD connection mode.",
+        "type": "string",
+        "enum": [
+          "single_hop",
+          "multi_hop"
+        ]
+      },
+      "BfdPeerConfig": {
+        "type": "object",
+        "properties": {
+          "detection_threshold": {
+            "type": "integer",
+            "format": "uint8",
+            "minimum": 0
+          },
+          "local": {
+            "nullable": true,
+            "type": "string",
+            "format": "ip"
+          },
+          "mode": {
+            "$ref": "#/components/schemas/BfdMode"
+          },
+          "remote": {
+            "type": "string",
+            "format": "ip"
+          },
+          "required_rx": {
+            "type": "integer",
+            "format": "uint64",
+            "minimum": 0
+          },
+          "switch": {
+            "$ref": "#/components/schemas/SwitchLocation"
+          }
+        },
+        "required": [
+          "detection_threshold",
+          "mode",
+          "remote",
+          "required_rx",
+          "switch"
+        ]
+      },
       "BgpConfig": {
         "type": "object",
         "properties": {
@@ -6342,6 +6387,14 @@
         "description": "Initial network configuration",
         "type": "object",
         "properties": {
+          "bfd": {
+            "description": "BFD configuration for connecting the rack to external networks",
+            "default": [],
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/BfdPeerConfig"
+            }
+          },
           "bgp": {
             "description": "BGP configurations for connecting the rack to external networks",
             "type": "array",
diff --git a/openapi/nexus.json b/openapi/nexus.json
index 9c39829a3b..7d236de7a3 100644
--- a/openapi/nexus.json
+++ b/openapi/nexus.json
@@ -9010,6 +9010,7 @@
         ]
       },
       "BfdMode": {
+        "description": "BFD connection mode.",
         "type": "string",
         "enum": [
           "single_hop",
diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json
index 5c43c40e56..e5b3a1c56f 100644
--- a/openapi/sled-agent.json
+++ b/openapi/sled-agent.json
@@ -1361,6 +1361,51 @@
           "serial_number"
         ]
       },
+      "BfdMode": {
+        "description": "BFD connection mode.",
+        "type": "string",
+        "enum": [
+          "single_hop",
+          "multi_hop"
+        ]
+      },
+      "BfdPeerConfig": {
+        "type": "object",
+        "properties": {
+          "detection_threshold": {
+            "type": "integer",
+            "format": "uint8",
+            "minimum": 0
+          },
+          "local": {
+            "nullable": true,
+            "type": "string",
+            "format": "ip"
+          },
+          "mode": {
+            "$ref": "#/components/schemas/BfdMode"
+          },
+          "remote": {
+            "type": "string",
+            "format": "ip"
+          },
+          "required_rx": {
+            "type": "integer",
+            "format": "uint64",
+            "minimum": 0
+          },
+          "switch": {
+            "$ref": "#/components/schemas/SwitchLocation"
+          }
+        },
+        "required": [
+          "detection_threshold",
+          "mode",
+          "remote",
+          "required_rx",
+          "switch"
+        ]
+      },
       "BgpConfig": {
         "type": "object",
         "properties": {
@@ -6432,6 +6477,14 @@
         "description": "Initial network configuration",
         "type": "object",
         "properties": {
+          "bfd": {
+            "description": "BFD configuration for connecting the rack to external networks",
+            "default": [],
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/BfdPeerConfig"
+            }
+          },
           "bgp": {
             "description": "BGP configurations for connecting the rack to external networks",
             "type": "array",
diff --git a/schema/rss-sled-plan.json b/schema/rss-sled-plan.json
index f5ac5bd0ff..14ddc848be 100644
--- a/schema/rss-sled-plan.json
+++ b/schema/rss-sled-plan.json
@@ -91,6 +91,53 @@
         }
       ]
     },
+    "BfdMode": {
+      "description": "BFD connection mode.",
+      "type": "string",
+      "enum": [
+        "single_hop",
+        "multi_hop"
+      ]
+    },
+    "BfdPeerConfig": {
+      "type": "object",
+      "required": [
+        "detection_threshold",
+        "mode",
+        "remote",
+        "required_rx",
+        "switch"
+      ],
+      "properties": {
+        "detection_threshold": {
+          "type": "integer",
+          "format": "uint8",
+          "minimum": 0.0
+        },
+        "local": {
+          "type": [
+            "string",
+            "null"
+          ],
+          "format": "ip"
+        },
+        "mode": {
+          "$ref": "#/definitions/BfdMode"
+        },
+        "remote": {
+          "type": "string",
+          "format": "ip"
+        },
+        "required_rx": {
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0.0
+        },
+        "switch": {
+          "$ref": "#/definitions/SwitchLocation"
+        }
+      }
+    },
     "BgpConfig": {
       "type": "object",
       "required": [
@@ -558,6 +605,14 @@
         "rack_subnet"
       ],
       "properties": {
+        "bfd": {
+          "description": "BFD configuration for connecting the rack to external networks",
+          "default": [],
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/BfdPeerConfig"
+          }
+        },
         "bgp": {
           "description": "BGP configurations for connecting the rack to external networks",
           "type": "array",
diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs
index 2bb0d31797..57e28d530e 100644
--- a/sled-agent/src/bootstrap/early_networking.rs
+++ b/sled-agent/src/bootstrap/early_networking.rs
@@ -16,12 +16,13 @@ use internal_dns::resolver::{ResolveError, Resolver as DnsResolver};
 use internal_dns::ServiceName;
 use ipnetwork::Ipv6Network;
 use mg_admin_client::types::{
-    AddStaticRoute4Request, ApplyRequest, BgpPeerConfig, Prefix4, StaticRoute4,
-    StaticRoute4List,
+    AddStaticRoute4Request, ApplyRequest, BfdPeerConfig, BgpPeerConfig,
+    Prefix4, StaticRoute4, StaticRoute4List,
 };
 use mg_admin_client::Client as MgdClient;
 use omicron_common::address::DENDRITE_PORT;
 use omicron_common::address::{MGD_PORT, MGS_PORT};
+use omicron_common::api::external::BfdMode;
 use omicron_common::api::internal::shared::{
     BgpConfig, PortConfigV1, PortFec, PortSpeed, RackNetworkConfig,
     RackNetworkConfigV1, SwitchLocation, UplinkConfig,
@@ -63,6 +64,9 @@ pub enum EarlyNetworkSetupError {
     #[error("BGP configuration error: {0}")]
     BgpConfigurationError(String),
 
+    #[error("BFD configuration error: {0}")]
+    BfdConfigurationError(String),
+
     #[error("MGD error: {0}")]
     MgdError(String),
 }
@@ -551,6 +555,30 @@ impl<'a> EarlyNetworkSetup<'a> {
             ))
         })?;
 
+        // BFD config
+        for spec in &rack_network_config.bfd {
+            if spec.switch != switch_location {
+                continue;
+            }
+            let cfg = BfdPeerConfig {
+                detection_threshold: spec.detection_threshold,
+                listen: spec.local.unwrap_or(Ipv4Addr::UNSPECIFIED.into()),
+                mode: match spec.mode {
+                    BfdMode::SingleHop => {
+                        mg_admin_client::types::SessionMode::SingleHop
+                    }
+                    BfdMode::MultiHop => {
+                        mg_admin_client::types::SessionMode::MultiHop
+                    }
+                },
+                peer: spec.remote,
+                required_rx: spec.required_rx,
+            };
+            mgd.add_bfd_peer(&cfg).await.map_err(|e| {
+                EarlyNetworkSetupError::BfdConfigurationError(e.to_string())
+            })?;
+        }
+
         Ok(our_ports)
     }
 
@@ -804,6 +832,7 @@ impl RackNetworkConfigV0 {
                 .map(|uplink| PortConfigV1::from(uplink))
                 .collect(),
             bgp: vec![],
+            bfd: vec![],
         }
     }
 }
@@ -898,6 +927,7 @@ mod tests {
                         bgp_peers: vec![],
                     }],
                     bgp: vec![],
+                    bfd: vec![],
                 }),
             },
         };
diff --git a/sled-agent/src/bootstrap/params.rs b/sled-agent/src/bootstrap/params.rs
index 4835edf4a9..c901fb31ac 100644
--- a/sled-agent/src/bootstrap/params.rs
+++ b/sled-agent/src/bootstrap/params.rs
@@ -493,6 +493,7 @@ mod tests {
                 infra_ip_last: Ipv4Addr::LOCALHOST,
                 ports: Vec::new(),
                 bgp: Vec::new(),
+                bfd: Vec::new(),
             },
         };
 
diff --git a/sled-agent/src/rack_setup/config.rs b/sled-agent/src/rack_setup/config.rs
index 52bea295a5..fba753657e 100644
--- a/sled-agent/src/rack_setup/config.rs
+++ b/sled-agent/src/rack_setup/config.rs
@@ -127,6 +127,7 @@ mod test {
                 infra_ip_last: Ipv4Addr::LOCALHOST,
                 ports: Vec::new(),
                 bgp: Vec::new(),
+                bfd: Vec::new(),
             },
         };
 
diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs
index 144388a2de..dd6936132b 100644
--- a/sled-agent/src/rack_setup/plan/service.rs
+++ b/sled-agent/src/rack_setup/plan/service.rs
@@ -1169,6 +1169,7 @@ mod tests {
                 infra_ip_last: Ipv4Addr::LOCALHOST,
                 ports: Vec::new(),
                 bgp: Vec::new(),
+                bfd: Vec::new(),
             },
         };
 
diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs
index 0fae365242..8c678daf30 100644
--- a/sled-agent/src/rack_setup/service.rs
+++ b/sled-agent/src/rack_setup/service.rs
@@ -660,6 +660,25 @@ impl ServiceInner {
                         originate: config.originate.clone(),
                     })
                     .collect(),
+                bfd: config
+                    .bfd
+                    .iter()
+                    .map(|spec| NexusTypes::BfdPeerConfig {
+                        detection_threshold: spec.detection_threshold,
+                        local: spec.local,
+                        mode: match spec.mode {
+                            omicron_common::api::external::BfdMode::SingleHop => {
+                                nexus_client::types::BfdMode::SingleHop
+                            }
+                            omicron_common::api::external::BfdMode::MultiHop => {
+                                nexus_client::types::BfdMode::MultiHop
+                            }
+                        },
+                        remote: spec.remote,
+                        required_rx: spec.required_rx,
+                        switch: spec.switch.into(),
+                    })
+                    .collect(),
             }
         };
 
diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs
index c886c9534b..c3c92eb6fe 100644
--- a/sled-agent/src/sim/http_entrypoints.rs
+++ b/sled-agent/src/sim/http_entrypoints.rs
@@ -407,6 +407,7 @@ async fn read_network_bootstore_config(
                 infra_ip_last: Ipv4Addr::UNSPECIFIED,
                 ports: Vec::new(),
                 bgp: Vec::new(),
+                bfd: Vec::new(),
             }),
         },
     };
diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs
index b7577e6d02..fea6b738a6 100644
--- a/sled-agent/src/sim/server.rs
+++ b/sled-agent/src/sim/server.rs
@@ -516,6 +516,7 @@ pub async fn run_standalone_server(
             infra_ip_last: Ipv4Addr::LOCALHOST,
             ports: Vec::new(),
             bgp: Vec::new(),
+            bfd: Vec::new(),
         },
     };
 
diff --git a/sled-agent/tests/integration_tests/early_network.rs b/sled-agent/tests/integration_tests/early_network.rs
index c3a4a53ebf..b15694c7f1 100644
--- a/sled-agent/tests/integration_tests/early_network.rs
+++ b/sled-agent/tests/integration_tests/early_network.rs
@@ -146,6 +146,7 @@ fn current_config_example() -> (&'static str, EarlyNetworkConfig) {
                     asn: 20000,
                     originate: vec!["192.168.0.0/24".parse().unwrap()],
                 }],
+                bfd: vec![],
             }),
         },
     };
diff --git a/sled-agent/tests/output/new-rss-sled-plans/madrid-rss-sled-plan.json b/sled-agent/tests/output/new-rss-sled-plans/madrid-rss-sled-plan.json
index 69f68c60ad..5462c77d25 100644
--- a/sled-agent/tests/output/new-rss-sled-plans/madrid-rss-sled-plan.json
+++ b/sled-agent/tests/output/new-rss-sled-plans/madrid-rss-sled-plan.json
@@ -158,7 +158,8 @@
           "autoneg": false
         }
       ],
-      "bgp": []
+      "bgp": [],
+      "bfd": []
     }
   }
 }
\ No newline at end of file
diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs
index 3fc123480d..8d782754bc 100644
--- a/wicketd/src/rss_config.rs
+++ b/wicketd/src/rss_config.rs
@@ -560,6 +560,8 @@ fn validate_rack_network_config(
                 originate: config.originate.clone(),
             })
             .collect(),
+        //TODO(ry) bfd config in wicket
+        bfd: vec![],
     })
 }
 

From 2349feb36d39804e38539e134a9a3a4462e7d516 Mon Sep 17 00:00:00 2001
From: Rain <rain@oxide.computer>
Date: Sat, 23 Mar 2024 18:59:28 -0700
Subject: [PATCH 151/157] [reconfigurator] replace Blueprint::zones_in_service
 with a disposition enum next to each zone (#5238)

This greatly simplifies the handling of the expunged disposition in
#5211. It also is more understandable, and less prone to making
mistakes.

This PR only changes the in-memory representation of what is currently
`zones_in_service`. I've skipped doing the DB migration in this PR
because it'll be easier to just do it wholesale in #5211.

There are some small differences in diff output that I think make sense.
But I took a bigger swing at it in #5270, which depends on this.
---
 clients/sled-agent-client/src/lib.rs          |  18 +
 dev-tools/reconfigurator-cli/src/main.rs      |   3 +-
 .../tests/output/cmd-complex-stdout           | 313 ----------
 nexus/db-model/src/deployment.rs              |  22 +-
 .../db-queries/src/db/datastore/deployment.rs | 138 ++--
 nexus/db-queries/src/db/datastore/rack.rs     |   5 +-
 nexus/db-queries/src/db/datastore/vpc.rs      |  86 ++-
 .../reconfigurator/execution/src/datasets.rs  |   8 +-
 nexus/reconfigurator/execution/src/dns.rs     |  53 +-
 nexus/reconfigurator/execution/src/lib.rs     |  10 +-
 .../execution/src/omicron_zones.rs            | 112 ++--
 .../execution/src/resource_allocation.rs      | 244 ++++----
 .../planning/src/blueprint_builder.rs         | 185 +++---
 nexus/reconfigurator/planning/src/example.rs  |   7 +-
 nexus/reconfigurator/planning/src/planner.rs  |  46 +-
 .../output/planner_basic_add_sled_2_3.txt     |  74 +--
 .../output/planner_basic_add_sled_3_5.txt     |  94 +--
 .../output/planner_nonprovisionable_1_2.txt   | 142 ++---
 .../src/app/background/blueprint_execution.rs | 152 +++--
 nexus/src/app/background/blueprint_load.rs    |   5 +-
 nexus/test-utils/src/lib.rs                   |  29 +-
 nexus/types/src/deployment.rs                 | 591 +++++++++++-------
 openapi/nexus-internal.json                   | 123 ++--
 sled-agent/src/rack_setup/service.rs          |  55 +-
 24 files changed, 1262 insertions(+), 1253 deletions(-)
 delete mode 100644 dev-tools/reconfigurator-cli/tests/output/cmd-complex-stdout

diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs
index 17d1225c6e..0426982d3e 100644
--- a/clients/sled-agent-client/src/lib.rs
+++ b/clients/sled-agent-client/src/lib.rs
@@ -114,6 +114,24 @@ impl types::OmicronZoneType {
         }
     }
 
+    /// Identifies whether this a Crucible (not Crucible pantry) zone
+    pub fn is_crucible(&self) -> bool {
+        match self {
+            types::OmicronZoneType::Crucible { .. } => true,
+
+            types::OmicronZoneType::BoundaryNtp { .. }
+            | types::OmicronZoneType::InternalNtp { .. }
+            | types::OmicronZoneType::Clickhouse { .. }
+            | types::OmicronZoneType::ClickhouseKeeper { .. }
+            | types::OmicronZoneType::CockroachDb { .. }
+            | types::OmicronZoneType::CruciblePantry { .. }
+            | types::OmicronZoneType::ExternalDns { .. }
+            | types::OmicronZoneType::InternalDns { .. }
+            | types::OmicronZoneType::Nexus { .. }
+            | types::OmicronZoneType::Oximeter { .. } => false,
+        }
+    }
+
     /// This zone's external IP
     pub fn external_ip(&self) -> anyhow::Result<Option<IpAddr>> {
         match self {
diff --git a/dev-tools/reconfigurator-cli/src/main.rs b/dev-tools/reconfigurator-cli/src/main.rs
index 82b71cff49..b59fc96703 100644
--- a/dev-tools/reconfigurator-cli/src/main.rs
+++ b/dev-tools/reconfigurator-cli/src/main.rs
@@ -547,8 +547,7 @@ fn cmd_blueprint_diff_inventory(
         .get(&blueprint_id)
         .ok_or_else(|| anyhow!("no such blueprint: {}", blueprint_id))?;
 
-    let zones = collection.all_omicron_zones().map(|z| z.id).collect();
-    let diff = blueprint.diff_sleds_from_collection(&collection, &zones);
+    let diff = blueprint.diff_sleds_from_collection(&collection);
     Ok(Some(diff.display().to_string()))
 }
 
diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-complex-stdout b/dev-tools/reconfigurator-cli/tests/output/cmd-complex-stdout
deleted file mode 100644
index 0c8c164d6f..0000000000
--- a/dev-tools/reconfigurator-cli/tests/output/cmd-complex-stdout
+++ /dev/null
@@ -1,313 +0,0 @@
-> sled-list
-ID NZPOOLS SUBNET 
-
-> 
-
-> inventory-list
-ID NERRORS TIME_DONE 
-
-> 
-
-> blueprint-list
-ID 
-
-> 
-
-> file-contents tests/input/complex.json
-sled: 1f34b5cc-6dac-40cc-9ea1-95f32c33e59b (subnet: fd00:1122:3344:103::/64, zpools: 5)
-sled: 67a1268b-52d3-4159-b3ed-7a0777f1f340 (subnet: fd00:1122:3344:104::/64, zpools: 5)
-sled: 77851655-10a5-4e80-88b0-a001473cab7e (subnet: fd00:1122:3344:102::/64, zpools: 5)
-sled: f122b3a2-e383-4e42-b2a5-74a69a9d110a (subnet: fd00:1122:3344:101::/64, zpools: 5)
-collection: 3f61b723-cfe7-40ec-b22c-e3e1f35325f9 (errors: 0, completed at: 2024-03-12T18:59:34.397Z)
-collection: 37c212f1-bfdd-4cfd-a17c-e65bd93bea5f (errors: 0, completed at: 2024-03-12T19:07:28.472Z)
-collection: 3700dcd4-64bb-4997-bfdb-dcfc7cbbc998 (errors: 0, completed at: 2024-03-12T19:07:28.728Z)
-collection: 5e1d6ae0-c5b4-4884-ac41-680cd4f5762d (errors: 0, completed at: 2024-03-12T19:09:37.116Z)
-blueprint:  649e444d-d00f-4067-a915-12e15647bf37 (created at: 2024-03-12 19:09:41.953854 UTC)
-blueprint:  cd34e87b-d821-4768-b32f-f41c28db835e (created at: 2024-03-12 19:11:49.876621 UTC)
-
-
-> 
-
-> load tests/input/complex.json 3f61b723-cfe7-40ec-b22c-e3e1f35325f9
-using collection 3f61b723-cfe7-40ec-b22c-e3e1f35325f9 as source of sled inventory data
-sled 1f34b5cc-6dac-40cc-9ea1-95f32c33e59b loaded
-sled 67a1268b-52d3-4159-b3ed-7a0777f1f340 loaded
-sled 77851655-10a5-4e80-88b0-a001473cab7e loaded
-sled f122b3a2-e383-4e42-b2a5-74a69a9d110a loaded
-collection 3f61b723-cfe7-40ec-b22c-e3e1f35325f9 loaded
-collection 37c212f1-bfdd-4cfd-a17c-e65bd93bea5f loaded
-collection 3700dcd4-64bb-4997-bfdb-dcfc7cbbc998 loaded
-collection 5e1d6ae0-c5b4-4884-ac41-680cd4f5762d loaded
-blueprint 649e444d-d00f-4067-a915-12e15647bf37 loaded
-blueprint cd34e87b-d821-4768-b32f-f41c28db835e loaded
-loaded data from "tests/input/complex.json"
-
-
-> 
-
-> sled-list
-ID                                   NZPOOLS SUBNET                  
-1f34b5cc-6dac-40cc-9ea1-95f32c33e59b 5       fd00:1122:3344:103::/64 
-67a1268b-52d3-4159-b3ed-7a0777f1f340 5       fd00:1122:3344:104::/64 
-77851655-10a5-4e80-88b0-a001473cab7e 5       fd00:1122:3344:102::/64 
-f122b3a2-e383-4e42-b2a5-74a69a9d110a 5       fd00:1122:3344:101::/64 
-
-> 
-
-> inventory-list
-ID                                   NERRORS TIME_DONE                
-3f61b723-cfe7-40ec-b22c-e3e1f35325f9 0       2024-03-12T18:59:34.397Z 
-37c212f1-bfdd-4cfd-a17c-e65bd93bea5f 0       2024-03-12T19:07:28.472Z 
-3700dcd4-64bb-4997-bfdb-dcfc7cbbc998 0       2024-03-12T19:07:28.728Z 
-5e1d6ae0-c5b4-4884-ac41-680cd4f5762d 0       2024-03-12T19:09:37.116Z 
-
-> 
-
-> blueprint-list
-ID                                   
-649e444d-d00f-4067-a915-12e15647bf37 
-cd34e87b-d821-4768-b32f-f41c28db835e 
-
-> 
-
-> blueprint-show cd34e87b-d821-4768-b32f-f41c28db835e
-blueprint  cd34e87b-d821-4768-b32f-f41c28db835e
-parent:    <none>
-created by 0d459323-e414-4f32-9944-76c7331da622 (likely a Nexus instance)
-created at 2024-03-12T19:11:49.876Z
-internal DNS version: 1
-comment: from collection 5e1d6ae0-c5b4-4884-ac41-680cd4f5762d
-zones:
-
-  sled 1f34b5cc-6dac-40cc-9ea1-95f32c33e59b: Omicron zones at generation 5
-    0db88baf-911d-47bb-af71-3aebd7c96934 in service     nexus
-    213d7dd3-95a4-45a7-99ca-e4899991dab4 in service     crucible
-    218f36a5-9e51-4b88-ad1a-da4d85c246c4 in service     internal_dns
-    40c47ead-3be8-4724-a8ef-b66233fc5c65 in service     crucible
-    425d9d2f-72ea-4f53-80b6-04e002771cdb in service     crucible
-    46a050b6-37b8-47b9-b4ae-ccb736ecd6d5 in service     crucible
-    4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd in service     clickhouse
-    7611a549-effc-41b6-b6cb-c03e0c70a31f in service     internal_ntp
-    af7ccead-2bd8-4a8a-9fbf-3b788d180139 in service     cockroach_db
-    d4428e41-ad7b-41f0-bde6-7c7c955f20c1 in service     crucible
-  sled 67a1268b-52d3-4159-b3ed-7a0777f1f340: Omicron zones at generation 5
-    1ebea0c3-1bfd-472c-9f84-ac9e23b32707 in service     crucible
-    2e9dfa8f-5371-48e5-b7f7-62a6af8283c1 in service     cockroach_db
-    57c712a6-6429-44b7-a9b2-9957cb9054d4 in service     crucible
-    59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9 in service     crucible
-    73916098-788f-48d8-b745-e13fdcfb771b in service     internal_ntp
-    9bb513e3-7505-416c-aa70-b26f1152bbdc in service     crucible
-    9c14c879-137b-4f5b-8068-1292bffc83ed in service     cockroach_db
-    9cf90589-5da1-4b42-b09b-d784e4134637 in service     nexus
-    d48b2dfa-c749-47dd-98b6-3f478cd63934 in service     crucible_pantry
-    fd003bee-818a-4c95-aae5-e378ac601522 in service     crucible
-  sled 77851655-10a5-4e80-88b0-a001473cab7e: Omicron zones at generation 5
-    084cac6e-44b5-4333-a2ff-b0993b534898 in service     crucible
-    11c527c5-0598-428e-9270-be5b74d3461b in service     cockroach_db
-    20be9299-495d-428d-b315-694c2c4b64af in service     crucible_pantry
-    2b29cb95-f8c6-4aff-acc0-1c0a627be392 in service     crucible
-    8d452bdd-e8b8-4108-a690-46fc59197db0 in service     crucible
-    953019c0-8e85-4417-b231-d42820470125 in service     oximeter
-    99c635b9-8e97-414a-8316-8261d2d767d0 in service     crucible
-    cbe1abfa-0063-44b2-9ce7-7ad76b403a94 in service     internal_dns
-    d52e797a-8fff-4e1e-a5ff-0ab8c8293ded in service     boundary_ntp
-    dc7f703a-27d1-4802-b85d-5629ddb2d67c in service     crucible
-    f0ed9838-6f1b-4646-a478-9f74f2e6e802 in service     external_dns
-  sled f122b3a2-e383-4e42-b2a5-74a69a9d110a: Omicron zones at generation 5
-    0d459323-e414-4f32-9944-76c7331da622 in service     nexus
-    2902972b-3ae4-489d-b937-6480352e214a in service     crucible
-    2ee832de-af75-47ac-abac-f91304857ca4 in service     external_dns
-    30050a7e-7fc3-4965-b07b-1635212393eb in service     crucible_pantry
-    8494befe-6df6-42d1-9baa-599ba4228430 in service     boundary_ntp
-    92cba955-3a5a-4d82-b0f2-30e89d4556d1 in service     crucible
-    9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b in service     cockroach_db
-    e668e3f5-5457-4689-97da-b2af537787ac in service     crucible
-    f32fd972-7f40-4c58-9e00-7fc5c44273f4 in service     crucible
-    f8c79ba4-194a-4cfe-8090-5315e297b780 in service     internal_dns
-    fa5e0176-b4a8-4598-aa31-40fd562d246c in service     crucible
-
-
-> 
-
-> blueprint-diff-inventory 3f61b723-cfe7-40ec-b22c-e3e1f35325f9 cd34e87b-d821-4768-b32f-f41c28db835e
-diff collection 3f61b723-cfe7-40ec-b22c-e3e1f35325f9 blueprint cd34e87b-d821-4768-b32f-f41c28db835e
---- collection 3f61b723-cfe7-40ec-b22c-e3e1f35325f9
-+++ blueprint cd34e87b-d821-4768-b32f-f41c28db835e
-  sled 1f34b5cc-6dac-40cc-9ea1-95f32c33e59b
-      zone config generation 5
-         zone 0db88baf-911d-47bb-af71-3aebd7c96934 type nexus underlay IP fd00:1122:3344:103::4 (unchanged)
-         zone 213d7dd3-95a4-45a7-99ca-e4899991dab4 type crucible underlay IP fd00:1122:3344:103::6 (unchanged)
-         zone 218f36a5-9e51-4b88-ad1a-da4d85c246c4 type internal_dns underlay IP fd00:1122:3344:3::1 (unchanged)
-         zone 40c47ead-3be8-4724-a8ef-b66233fc5c65 type crucible underlay IP fd00:1122:3344:103::7 (unchanged)
-         zone 425d9d2f-72ea-4f53-80b6-04e002771cdb type crucible underlay IP fd00:1122:3344:103::9 (unchanged)
-         zone 46a050b6-37b8-47b9-b4ae-ccb736ecd6d5 type crucible underlay IP fd00:1122:3344:103::8 (unchanged)
-         zone 4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd type clickhouse underlay IP fd00:1122:3344:103::5 (unchanged)
-         zone 7611a549-effc-41b6-b6cb-c03e0c70a31f type internal_ntp underlay IP fd00:1122:3344:103::b (unchanged)
-         zone af7ccead-2bd8-4a8a-9fbf-3b788d180139 type cockroach_db underlay IP fd00:1122:3344:103::3 (unchanged)
-         zone d4428e41-ad7b-41f0-bde6-7c7c955f20c1 type crucible underlay IP fd00:1122:3344:103::a (unchanged)
-  sled 67a1268b-52d3-4159-b3ed-7a0777f1f340
-      zone config generation 5
-         zone 1ebea0c3-1bfd-472c-9f84-ac9e23b32707 type crucible underlay IP fd00:1122:3344:104::b (unchanged)
-         zone 2e9dfa8f-5371-48e5-b7f7-62a6af8283c1 type cockroach_db underlay IP fd00:1122:3344:104::4 (unchanged)
-         zone 57c712a6-6429-44b7-a9b2-9957cb9054d4 type crucible underlay IP fd00:1122:3344:104::a (unchanged)
-         zone 59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9 type crucible underlay IP fd00:1122:3344:104::7 (unchanged)
-         zone 73916098-788f-48d8-b745-e13fdcfb771b type internal_ntp underlay IP fd00:1122:3344:104::c (unchanged)
-         zone 9bb513e3-7505-416c-aa70-b26f1152bbdc type crucible underlay IP fd00:1122:3344:104::9 (unchanged)
-         zone 9c14c879-137b-4f5b-8068-1292bffc83ed type cockroach_db underlay IP fd00:1122:3344:104::3 (unchanged)
-         zone 9cf90589-5da1-4b42-b09b-d784e4134637 type nexus underlay IP fd00:1122:3344:104::5 (unchanged)
-         zone d48b2dfa-c749-47dd-98b6-3f478cd63934 type crucible_pantry underlay IP fd00:1122:3344:104::6 (unchanged)
-         zone fd003bee-818a-4c95-aae5-e378ac601522 type crucible underlay IP fd00:1122:3344:104::8 (unchanged)
-  sled 77851655-10a5-4e80-88b0-a001473cab7e
-      zone config generation 5
-         zone 084cac6e-44b5-4333-a2ff-b0993b534898 type crucible underlay IP fd00:1122:3344:102::8 (unchanged)
-         zone 11c527c5-0598-428e-9270-be5b74d3461b type cockroach_db underlay IP fd00:1122:3344:102::3 (unchanged)
-         zone 20be9299-495d-428d-b315-694c2c4b64af type crucible_pantry underlay IP fd00:1122:3344:102::6 (unchanged)
-         zone 2b29cb95-f8c6-4aff-acc0-1c0a627be392 type crucible underlay IP fd00:1122:3344:102::b (unchanged)
-         zone 8d452bdd-e8b8-4108-a690-46fc59197db0 type crucible underlay IP fd00:1122:3344:102::a (unchanged)
-         zone 953019c0-8e85-4417-b231-d42820470125 type oximeter underlay IP fd00:1122:3344:102::5 (unchanged)
-         zone 99c635b9-8e97-414a-8316-8261d2d767d0 type crucible underlay IP fd00:1122:3344:102::7 (unchanged)
-         zone cbe1abfa-0063-44b2-9ce7-7ad76b403a94 type internal_dns underlay IP fd00:1122:3344:2::1 (unchanged)
-         zone d52e797a-8fff-4e1e-a5ff-0ab8c8293ded type boundary_ntp underlay IP fd00:1122:3344:102::c (unchanged)
-         zone dc7f703a-27d1-4802-b85d-5629ddb2d67c type crucible underlay IP fd00:1122:3344:102::9 (unchanged)
-         zone f0ed9838-6f1b-4646-a478-9f74f2e6e802 type external_dns underlay IP fd00:1122:3344:102::4 (unchanged)
-  sled f122b3a2-e383-4e42-b2a5-74a69a9d110a
-      zone config generation 5
-         zone 0d459323-e414-4f32-9944-76c7331da622 type nexus underlay IP fd00:1122:3344:101::5 (unchanged)
-         zone 2902972b-3ae4-489d-b937-6480352e214a type crucible underlay IP fd00:1122:3344:101::8 (unchanged)
-         zone 2ee832de-af75-47ac-abac-f91304857ca4 type external_dns underlay IP fd00:1122:3344:101::4 (unchanged)
-         zone 30050a7e-7fc3-4965-b07b-1635212393eb type crucible_pantry underlay IP fd00:1122:3344:101::6 (unchanged)
-         zone 8494befe-6df6-42d1-9baa-599ba4228430 type boundary_ntp underlay IP fd00:1122:3344:101::c (unchanged)
-         zone 92cba955-3a5a-4d82-b0f2-30e89d4556d1 type crucible underlay IP fd00:1122:3344:101::7 (unchanged)
-         zone 9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b type cockroach_db underlay IP fd00:1122:3344:101::3 (unchanged)
-         zone e668e3f5-5457-4689-97da-b2af537787ac type crucible underlay IP fd00:1122:3344:101::b (unchanged)
-         zone f32fd972-7f40-4c58-9e00-7fc5c44273f4 type crucible underlay IP fd00:1122:3344:101::9 (unchanged)
-         zone f8c79ba4-194a-4cfe-8090-5315e297b780 type internal_dns underlay IP fd00:1122:3344:1::1 (unchanged)
-         zone fa5e0176-b4a8-4598-aa31-40fd562d246c type crucible underlay IP fd00:1122:3344:101::a (unchanged)
-
-
-> 
-
-> blueprint-diff 649e444d-d00f-4067-a915-12e15647bf37 cd34e87b-d821-4768-b32f-f41c28db835e
-diff blueprint 649e444d-d00f-4067-a915-12e15647bf37 blueprint cd34e87b-d821-4768-b32f-f41c28db835e
---- blueprint 649e444d-d00f-4067-a915-12e15647bf37
-+++ blueprint cd34e87b-d821-4768-b32f-f41c28db835e
-  sled 1f34b5cc-6dac-40cc-9ea1-95f32c33e59b
-      zone config generation 5
-         zone 0db88baf-911d-47bb-af71-3aebd7c96934 type nexus underlay IP fd00:1122:3344:103::4 (unchanged)
-         zone 213d7dd3-95a4-45a7-99ca-e4899991dab4 type crucible underlay IP fd00:1122:3344:103::6 (unchanged)
-         zone 218f36a5-9e51-4b88-ad1a-da4d85c246c4 type internal_dns underlay IP fd00:1122:3344:3::1 (unchanged)
-         zone 40c47ead-3be8-4724-a8ef-b66233fc5c65 type crucible underlay IP fd00:1122:3344:103::7 (unchanged)
-         zone 425d9d2f-72ea-4f53-80b6-04e002771cdb type crucible underlay IP fd00:1122:3344:103::9 (unchanged)
-         zone 46a050b6-37b8-47b9-b4ae-ccb736ecd6d5 type crucible underlay IP fd00:1122:3344:103::8 (unchanged)
-         zone 4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd type clickhouse underlay IP fd00:1122:3344:103::5 (unchanged)
-         zone 7611a549-effc-41b6-b6cb-c03e0c70a31f type internal_ntp underlay IP fd00:1122:3344:103::b (unchanged)
-         zone af7ccead-2bd8-4a8a-9fbf-3b788d180139 type cockroach_db underlay IP fd00:1122:3344:103::3 (unchanged)
-         zone d4428e41-ad7b-41f0-bde6-7c7c955f20c1 type crucible underlay IP fd00:1122:3344:103::a (unchanged)
-  sled 67a1268b-52d3-4159-b3ed-7a0777f1f340
-      zone config generation 5
-         zone 1ebea0c3-1bfd-472c-9f84-ac9e23b32707 type crucible underlay IP fd00:1122:3344:104::b (unchanged)
-         zone 2e9dfa8f-5371-48e5-b7f7-62a6af8283c1 type cockroach_db underlay IP fd00:1122:3344:104::4 (unchanged)
-         zone 57c712a6-6429-44b7-a9b2-9957cb9054d4 type crucible underlay IP fd00:1122:3344:104::a (unchanged)
-         zone 59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9 type crucible underlay IP fd00:1122:3344:104::7 (unchanged)
-         zone 73916098-788f-48d8-b745-e13fdcfb771b type internal_ntp underlay IP fd00:1122:3344:104::c (unchanged)
-         zone 9bb513e3-7505-416c-aa70-b26f1152bbdc type crucible underlay IP fd00:1122:3344:104::9 (unchanged)
-         zone 9c14c879-137b-4f5b-8068-1292bffc83ed type cockroach_db underlay IP fd00:1122:3344:104::3 (unchanged)
-         zone 9cf90589-5da1-4b42-b09b-d784e4134637 type nexus underlay IP fd00:1122:3344:104::5 (unchanged)
-         zone d48b2dfa-c749-47dd-98b6-3f478cd63934 type crucible_pantry underlay IP fd00:1122:3344:104::6 (unchanged)
-         zone fd003bee-818a-4c95-aae5-e378ac601522 type crucible underlay IP fd00:1122:3344:104::8 (unchanged)
-  sled 77851655-10a5-4e80-88b0-a001473cab7e
-      zone config generation 5
-         zone 084cac6e-44b5-4333-a2ff-b0993b534898 type crucible underlay IP fd00:1122:3344:102::8 (unchanged)
-         zone 11c527c5-0598-428e-9270-be5b74d3461b type cockroach_db underlay IP fd00:1122:3344:102::3 (unchanged)
-         zone 20be9299-495d-428d-b315-694c2c4b64af type crucible_pantry underlay IP fd00:1122:3344:102::6 (unchanged)
-         zone 2b29cb95-f8c6-4aff-acc0-1c0a627be392 type crucible underlay IP fd00:1122:3344:102::b (unchanged)
-         zone 8d452bdd-e8b8-4108-a690-46fc59197db0 type crucible underlay IP fd00:1122:3344:102::a (unchanged)
-         zone 953019c0-8e85-4417-b231-d42820470125 type oximeter underlay IP fd00:1122:3344:102::5 (unchanged)
-         zone 99c635b9-8e97-414a-8316-8261d2d767d0 type crucible underlay IP fd00:1122:3344:102::7 (unchanged)
-         zone cbe1abfa-0063-44b2-9ce7-7ad76b403a94 type internal_dns underlay IP fd00:1122:3344:2::1 (unchanged)
-         zone d52e797a-8fff-4e1e-a5ff-0ab8c8293ded type boundary_ntp underlay IP fd00:1122:3344:102::c (unchanged)
-         zone dc7f703a-27d1-4802-b85d-5629ddb2d67c type crucible underlay IP fd00:1122:3344:102::9 (unchanged)
-         zone f0ed9838-6f1b-4646-a478-9f74f2e6e802 type external_dns underlay IP fd00:1122:3344:102::4 (unchanged)
-  sled f122b3a2-e383-4e42-b2a5-74a69a9d110a
-      zone config generation 5
-         zone 0d459323-e414-4f32-9944-76c7331da622 type nexus underlay IP fd00:1122:3344:101::5 (unchanged)
-         zone 2902972b-3ae4-489d-b937-6480352e214a type crucible underlay IP fd00:1122:3344:101::8 (unchanged)
-         zone 2ee832de-af75-47ac-abac-f91304857ca4 type external_dns underlay IP fd00:1122:3344:101::4 (unchanged)
-         zone 30050a7e-7fc3-4965-b07b-1635212393eb type crucible_pantry underlay IP fd00:1122:3344:101::6 (unchanged)
-         zone 8494befe-6df6-42d1-9baa-599ba4228430 type boundary_ntp underlay IP fd00:1122:3344:101::c (unchanged)
-         zone 92cba955-3a5a-4d82-b0f2-30e89d4556d1 type crucible underlay IP fd00:1122:3344:101::7 (unchanged)
-         zone 9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b type cockroach_db underlay IP fd00:1122:3344:101::3 (unchanged)
-         zone e668e3f5-5457-4689-97da-b2af537787ac type crucible underlay IP fd00:1122:3344:101::b (unchanged)
-         zone f32fd972-7f40-4c58-9e00-7fc5c44273f4 type crucible underlay IP fd00:1122:3344:101::9 (unchanged)
-         zone f8c79ba4-194a-4cfe-8090-5315e297b780 type internal_dns underlay IP fd00:1122:3344:1::1 (unchanged)
-         zone fa5e0176-b4a8-4598-aa31-40fd562d246c type crucible underlay IP fd00:1122:3344:101::a (unchanged)
-
-
-> 
-
-> blueprint-diff cd34e87b-d821-4768-b32f-f41c28db835e 649e444d-d00f-4067-a915-12e15647bf37
-diff blueprint cd34e87b-d821-4768-b32f-f41c28db835e blueprint 649e444d-d00f-4067-a915-12e15647bf37
---- blueprint cd34e87b-d821-4768-b32f-f41c28db835e
-+++ blueprint 649e444d-d00f-4067-a915-12e15647bf37
-  sled 1f34b5cc-6dac-40cc-9ea1-95f32c33e59b
-      zone config generation 5
-         zone 0db88baf-911d-47bb-af71-3aebd7c96934 type nexus underlay IP fd00:1122:3344:103::4 (unchanged)
-         zone 213d7dd3-95a4-45a7-99ca-e4899991dab4 type crucible underlay IP fd00:1122:3344:103::6 (unchanged)
-         zone 218f36a5-9e51-4b88-ad1a-da4d85c246c4 type internal_dns underlay IP fd00:1122:3344:3::1 (unchanged)
-         zone 40c47ead-3be8-4724-a8ef-b66233fc5c65 type crucible underlay IP fd00:1122:3344:103::7 (unchanged)
-         zone 425d9d2f-72ea-4f53-80b6-04e002771cdb type crucible underlay IP fd00:1122:3344:103::9 (unchanged)
-         zone 46a050b6-37b8-47b9-b4ae-ccb736ecd6d5 type crucible underlay IP fd00:1122:3344:103::8 (unchanged)
-         zone 4ea8104f-2cfc-4599-9d2c-bcfd8f0b71fd type clickhouse underlay IP fd00:1122:3344:103::5 (unchanged)
-         zone 7611a549-effc-41b6-b6cb-c03e0c70a31f type internal_ntp underlay IP fd00:1122:3344:103::b (unchanged)
-         zone af7ccead-2bd8-4a8a-9fbf-3b788d180139 type cockroach_db underlay IP fd00:1122:3344:103::3 (unchanged)
-         zone d4428e41-ad7b-41f0-bde6-7c7c955f20c1 type crucible underlay IP fd00:1122:3344:103::a (unchanged)
-  sled 67a1268b-52d3-4159-b3ed-7a0777f1f340
-      zone config generation 5
-         zone 1ebea0c3-1bfd-472c-9f84-ac9e23b32707 type crucible underlay IP fd00:1122:3344:104::b (unchanged)
-         zone 2e9dfa8f-5371-48e5-b7f7-62a6af8283c1 type cockroach_db underlay IP fd00:1122:3344:104::4 (unchanged)
-         zone 57c712a6-6429-44b7-a9b2-9957cb9054d4 type crucible underlay IP fd00:1122:3344:104::a (unchanged)
-         zone 59c4c9f0-bea9-46b7-ae9d-fb6dc1d658b9 type crucible underlay IP fd00:1122:3344:104::7 (unchanged)
-         zone 73916098-788f-48d8-b745-e13fdcfb771b type internal_ntp underlay IP fd00:1122:3344:104::c (unchanged)
-         zone 9bb513e3-7505-416c-aa70-b26f1152bbdc type crucible underlay IP fd00:1122:3344:104::9 (unchanged)
-         zone 9c14c879-137b-4f5b-8068-1292bffc83ed type cockroach_db underlay IP fd00:1122:3344:104::3 (unchanged)
-         zone 9cf90589-5da1-4b42-b09b-d784e4134637 type nexus underlay IP fd00:1122:3344:104::5 (unchanged)
-         zone d48b2dfa-c749-47dd-98b6-3f478cd63934 type crucible_pantry underlay IP fd00:1122:3344:104::6 (unchanged)
-         zone fd003bee-818a-4c95-aae5-e378ac601522 type crucible underlay IP fd00:1122:3344:104::8 (unchanged)
-  sled 77851655-10a5-4e80-88b0-a001473cab7e
-      zone config generation 5
-         zone 084cac6e-44b5-4333-a2ff-b0993b534898 type crucible underlay IP fd00:1122:3344:102::8 (unchanged)
-         zone 11c527c5-0598-428e-9270-be5b74d3461b type cockroach_db underlay IP fd00:1122:3344:102::3 (unchanged)
-         zone 20be9299-495d-428d-b315-694c2c4b64af type crucible_pantry underlay IP fd00:1122:3344:102::6 (unchanged)
-         zone 2b29cb95-f8c6-4aff-acc0-1c0a627be392 type crucible underlay IP fd00:1122:3344:102::b (unchanged)
-         zone 8d452bdd-e8b8-4108-a690-46fc59197db0 type crucible underlay IP fd00:1122:3344:102::a (unchanged)
-         zone 953019c0-8e85-4417-b231-d42820470125 type oximeter underlay IP fd00:1122:3344:102::5 (unchanged)
-         zone 99c635b9-8e97-414a-8316-8261d2d767d0 type crucible underlay IP fd00:1122:3344:102::7 (unchanged)
-         zone cbe1abfa-0063-44b2-9ce7-7ad76b403a94 type internal_dns underlay IP fd00:1122:3344:2::1 (unchanged)
-         zone d52e797a-8fff-4e1e-a5ff-0ab8c8293ded type boundary_ntp underlay IP fd00:1122:3344:102::c (unchanged)
-         zone dc7f703a-27d1-4802-b85d-5629ddb2d67c type crucible underlay IP fd00:1122:3344:102::9 (unchanged)
-         zone f0ed9838-6f1b-4646-a478-9f74f2e6e802 type external_dns underlay IP fd00:1122:3344:102::4 (unchanged)
-  sled f122b3a2-e383-4e42-b2a5-74a69a9d110a
-      zone config generation 5
-         zone 0d459323-e414-4f32-9944-76c7331da622 type nexus underlay IP fd00:1122:3344:101::5 (unchanged)
-         zone 2902972b-3ae4-489d-b937-6480352e214a type crucible underlay IP fd00:1122:3344:101::8 (unchanged)
-         zone 2ee832de-af75-47ac-abac-f91304857ca4 type external_dns underlay IP fd00:1122:3344:101::4 (unchanged)
-         zone 30050a7e-7fc3-4965-b07b-1635212393eb type crucible_pantry underlay IP fd00:1122:3344:101::6 (unchanged)
-         zone 8494befe-6df6-42d1-9baa-599ba4228430 type boundary_ntp underlay IP fd00:1122:3344:101::c (unchanged)
-         zone 92cba955-3a5a-4d82-b0f2-30e89d4556d1 type crucible underlay IP fd00:1122:3344:101::7 (unchanged)
-         zone 9ba4cda7-d2e4-4c44-b1a6-8ddfe676098b type cockroach_db underlay IP fd00:1122:3344:101::3 (unchanged)
-         zone e668e3f5-5457-4689-97da-b2af537787ac type crucible underlay IP fd00:1122:3344:101::b (unchanged)
-         zone f32fd972-7f40-4c58-9e00-7fc5c44273f4 type crucible underlay IP fd00:1122:3344:101::9 (unchanged)
-         zone f8c79ba4-194a-4cfe-8090-5315e297b780 type internal_dns underlay IP fd00:1122:3344:1::1 (unchanged)
-         zone fa5e0176-b4a8-4598-aa31-40fd562d246c type crucible underlay IP fd00:1122:3344:101::a (unchanged)
-
-
-> 
-
-> sled-add dde1c0e2-b10d-4621-b420-f179f7a7a00a
-added sled
-
-> 
-
-> blueprint-plan cd34e87b-d821-4768-b32f-f41c28db835e 3f61b723-cfe7-40ec-b22c-e3e1f35325f9
-generated blueprint REDACTED_UUID based on parent blueprint cd34e87b-d821-4768-b32f-f41c28db835e
-
diff --git a/nexus/db-model/src/deployment.rs b/nexus/db-model/src/deployment.rs
index e34148bede..e56c8bff54 100644
--- a/nexus/db-model/src/deployment.rs
+++ b/nexus/db-model/src/deployment.rs
@@ -15,6 +15,9 @@ use crate::{ipv6, Generation, MacAddr, Name, SqlU16, SqlU32, SqlU8};
 use chrono::{DateTime, Utc};
 use ipnetwork::IpNetwork;
 use nexus_types::deployment::BlueprintTarget;
+use nexus_types::deployment::BlueprintZoneConfig;
+use nexus_types::deployment::BlueprintZoneDisposition;
+use nexus_types::deployment::BlueprintZonesConfig;
 use omicron_common::api::internal::shared::NetworkInterface;
 use uuid::Uuid;
 
@@ -103,7 +106,7 @@ impl BpSledOmicronZones {
     pub fn new(
         blueprint_id: Uuid,
         sled_id: Uuid,
-        zones_config: &nexus_types::deployment::OmicronZonesConfig,
+        zones_config: &BlueprintZonesConfig,
     ) -> Self {
         Self {
             blueprint_id,
@@ -144,9 +147,9 @@ impl BpOmicronZone {
     pub fn new(
         blueprint_id: Uuid,
         sled_id: Uuid,
-        zone: &nexus_types::inventory::OmicronZoneConfig,
+        zone: &BlueprintZoneConfig,
     ) -> Result<Self, anyhow::Error> {
-        let zone = OmicronZone::new(sled_id, zone)?;
+        let zone = OmicronZone::new(sled_id, &zone.config)?;
         Ok(Self {
             blueprint_id,
             sled_id: zone.sled_id,
@@ -172,10 +175,11 @@ impl BpOmicronZone {
         })
     }
 
-    pub fn into_omicron_zone_config(
+    pub fn into_blueprint_zone_config(
         self,
         nic_row: Option<BpOmicronZoneNic>,
-    ) -> Result<nexus_types::inventory::OmicronZoneConfig, anyhow::Error> {
+        disposition: BlueprintZoneDisposition,
+    ) -> Result<BlueprintZoneConfig, anyhow::Error> {
         let zone = OmicronZone {
             sled_id: self.sled_id,
             id: self.id,
@@ -198,7 +202,9 @@ impl BpOmicronZone {
             snat_first_port: self.snat_first_port,
             snat_last_port: self.snat_last_port,
         };
-        zone.into_omicron_zone_config(nic_row.map(OmicronZoneNic::from))
+        let config =
+            zone.into_omicron_zone_config(nic_row.map(OmicronZoneNic::from))?;
+        Ok(BlueprintZoneConfig { config, disposition })
     }
 }
 
@@ -234,9 +240,9 @@ impl From<BpOmicronZoneNic> for OmicronZoneNic {
 impl BpOmicronZoneNic {
     pub fn new(
         blueprint_id: Uuid,
-        zone: &nexus_types::inventory::OmicronZoneConfig,
+        zone: &BlueprintZoneConfig,
     ) -> Result<Option<BpOmicronZoneNic>, anyhow::Error> {
-        let zone_nic = OmicronZoneNic::new(zone)?;
+        let zone_nic = OmicronZoneNic::new(&zone.config)?;
         Ok(zone_nic.map(|nic| Self {
             blueprint_id,
             id: nic.id,
diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs
index 88c0897be2..02645ca4f6 100644
--- a/nexus/db-queries/src/db/datastore/deployment.rs
+++ b/nexus/db-queries/src/db/datastore/deployment.rs
@@ -43,7 +43,9 @@ use nexus_db_model::BpTarget;
 use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::BlueprintMetadata;
 use nexus_types::deployment::BlueprintTarget;
-use nexus_types::deployment::OmicronZonesConfig;
+use nexus_types::deployment::BlueprintZoneDisposition;
+use nexus_types::deployment::BlueprintZoneFilter;
+use nexus_types::deployment::BlueprintZonesConfig;
 use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::ListResultVec;
@@ -105,52 +107,67 @@ impl DataStore {
         // so that we can produce the `Error` type that we want here.
         let row_blueprint = DbBlueprint::from(blueprint);
         let blueprint_id = row_blueprint.id;
+
+        // `Blueprint` stores the policy for each zone next to the zone itself.
+        // This would ideally be represented as a simple column in
+        // bp_omicron_zone.
+        //
+        // But historically, `Blueprint` used to store the set of zones in
+        // service in a BTreeSet. Since most zones are expected to be in
+        // service, we store the set of zones NOT in service (which we expect
+        // to be much smaller, often empty). Build that inverted set here.
+        //
+        // This will soon be replaced with an extra column in the
+        // `bp_omicron_zone` table, coupled with other data migrations.
+        let omicron_zones_not_in_service = blueprint
+            .all_blueprint_zones(BlueprintZoneFilter::All)
+            .filter_map(|(_, zone)| {
+                // This is going to go away soon when we change the database
+                // representation to store the zone disposition enum next to
+                // each zone. For now, do an exhaustive match so that this
+                // fails if we add a new variant.
+                match zone.disposition {
+                    BlueprintZoneDisposition::InService => None,
+                    BlueprintZoneDisposition::Quiesced => {
+                        Some(BpOmicronZoneNotInService {
+                            blueprint_id,
+                            bp_omicron_zone_id: zone.config.id,
+                        })
+                    }
+                }
+            })
+            .collect::<Vec<_>>();
+
         let sled_omicron_zones = blueprint
-            .omicron_zones
+            .blueprint_zones
             .iter()
             .map(|(sled_id, zones_config)| {
                 BpSledOmicronZones::new(blueprint_id, *sled_id, zones_config)
             })
             .collect::<Vec<_>>();
         let omicron_zones = blueprint
-            .omicron_zones
+            .blueprint_zones
             .iter()
             .flat_map(|(sled_id, zones_config)| {
-                zones_config.zones.iter().map(|zone| {
+                zones_config.zones.iter().map(move |zone| {
                     BpOmicronZone::new(blueprint_id, *sled_id, zone)
                         .map_err(|e| Error::internal_error(&format!("{:#}", e)))
                 })
             })
             .collect::<Result<Vec<_>, Error>>()?;
         let omicron_zone_nics = blueprint
-            .omicron_zones
+            .blueprint_zones
             .values()
             .flat_map(|zones_config| {
                 zones_config.zones.iter().filter_map(|zone| {
                     BpOmicronZoneNic::new(blueprint_id, zone)
-                        .with_context(|| format!("zone {:?}", zone.id))
+                        .with_context(|| format!("zone {:?}", zone.config.id))
                         .map_err(|e| Error::internal_error(&format!("{:#}", e)))
                         .transpose()
                 })
             })
             .collect::<Result<Vec<BpOmicronZoneNic>, _>>()?;
 
-        // `Blueprint` stores a set of zones in service, but in the database we
-        // store the set of zones NOT in service (which we expect to be much
-        // smaller, often empty). Build that inverted set here.
-        let omicron_zones_not_in_service = {
-            let mut zones_not_in_service = Vec::new();
-            for zone in &omicron_zones {
-                if !blueprint.zones_in_service.contains(&zone.id) {
-                    zones_not_in_service.push(BpOmicronZoneNotInService {
-                        blueprint_id,
-                        bp_omicron_zone_id: zone.id,
-                    });
-                }
-            }
-            zones_not_in_service
-        };
-
         // This implementation inserts all records associated with the
         // blueprint in one transaction.  This is required: we don't want
         // any planner or executor to see a half-inserted blueprint, nor do we
@@ -273,10 +290,10 @@ impl DataStore {
         // the `OmicronZonesConfig` generation number for each sled that is a
         // part of this blueprint. Construct the BTreeMap we ultimately need,
         // but all the `zones` vecs will be empty until our next query below.
-        let mut omicron_zones: BTreeMap<Uuid, OmicronZonesConfig> = {
+        let mut blueprint_zones: BTreeMap<Uuid, BlueprintZonesConfig> = {
             use db::schema::bp_sled_omicron_zones::dsl;
 
-            let mut omicron_zones = BTreeMap::new();
+            let mut blueprint_zones = BTreeMap::new();
             let mut paginator = Paginator::new(SQL_BATCH_SIZE);
             while let Some(p) = paginator.next() {
                 let batch = paginated(
@@ -295,9 +312,9 @@ impl DataStore {
                 paginator = p.found_batch(&batch, &|s| s.sled_id);
 
                 for s in batch {
-                    let old = omicron_zones.insert(
+                    let old = blueprint_zones.insert(
                         s.sled_id,
-                        OmicronZonesConfig {
+                        BlueprintZonesConfig {
                             generation: *s.generation,
                             zones: Vec::new(),
                         },
@@ -310,7 +327,7 @@ impl DataStore {
                 }
             }
 
-            omicron_zones
+            blueprint_zones
         };
 
         // Assemble a mutable map of all the NICs found, by NIC id.  As we
@@ -391,11 +408,6 @@ impl DataStore {
             omicron_zones_not_in_service
         };
 
-        // Create the in-memory list of zones _in_ service, which we'll
-        // calculate below as we load zones. (Any zone that isn't present in
-        // `omicron_zones_not_in_service` is considered in service.)
-        let mut zones_in_service = BTreeSet::new();
-
         // Load all the zones for each sled.
         {
             use db::schema::bp_omicron_zone::dsl;
@@ -438,8 +450,9 @@ impl DataStore {
                             })
                         })
                         .transpose()?;
-                    let sled_zones =
-                        omicron_zones.get_mut(&z.sled_id).ok_or_else(|| {
+                    let sled_zones = blueprint_zones
+                        .get_mut(&z.sled_id)
+                        .ok_or_else(|| {
                             // This error means that we found a row in
                             // bp_omicron_zone with no associated record in
                             // bp_sled_omicron_zones.  This should be
@@ -451,8 +464,14 @@ impl DataStore {
                             ))
                         })?;
                     let zone_id = z.id;
+                    let disposition =
+                        if omicron_zones_not_in_service.remove(&zone_id) {
+                            BlueprintZoneDisposition::Quiesced
+                        } else {
+                            BlueprintZoneDisposition::InService
+                        };
                     let zone = z
-                        .into_omicron_zone_config(nic_row)
+                        .into_blueprint_zone_config(nic_row, disposition)
                         .with_context(|| {
                             format!("zone {:?}: parse from database", zone_id)
                         })
@@ -463,14 +482,6 @@ impl DataStore {
                             ))
                         })?;
                     sled_zones.zones.push(zone);
-
-                    // If we can remove `zone_id` from
-                    // `omicron_zones_not_in_service`, then the zone is not in
-                    // service. Otherwise, add it to the list of in-service
-                    // zones.
-                    if !omicron_zones_not_in_service.remove(&zone_id) {
-                        zones_in_service.insert(zone_id);
-                    }
                 }
             }
         }
@@ -488,8 +499,7 @@ impl DataStore {
 
         Ok(Blueprint {
             id: blueprint_id,
-            omicron_zones,
-            zones_in_service,
+            blueprint_zones,
             parent_blueprint_id,
             internal_dns_version,
             external_dns_version,
@@ -1359,10 +1369,8 @@ mod tests {
             [blueprint1.id]
         );
 
-        // There ought to be no sleds or zones in service, and no parent
-        // blueprint.
-        assert_eq!(blueprint1.omicron_zones.len(), 0);
-        assert_eq!(blueprint1.zones_in_service.len(), 0);
+        // There ought to be no sleds or zones, and no parent blueprint.
+        assert_eq!(blueprint1.blueprint_zones.len(), 0);
         assert_eq!(blueprint1.parent_blueprint_id, None);
 
         // Trying to insert the same blueprint again should fail.
@@ -1407,9 +1415,9 @@ mod tests {
         );
 
         // Check the number of blueprint elements against our collection.
-        assert_eq!(blueprint1.omicron_zones.len(), policy.sleds.len());
+        assert_eq!(blueprint1.blueprint_zones.len(), policy.sleds.len());
         assert_eq!(
-            blueprint1.omicron_zones.len(),
+            blueprint1.blueprint_zones.len(),
             collection.omicron_zones.len()
         );
         assert_eq!(
@@ -1417,10 +1425,7 @@ mod tests {
             collection.all_omicron_zones().count()
         );
         // All zones should be in service.
-        assert_eq!(
-            blueprint1.zones_in_service.len(),
-            blueprint1.all_omicron_zones().count()
-        );
+        assert_all_zones_in_service(&blueprint1);
         assert_eq!(blueprint1.parent_blueprint_id, None);
 
         // Set blueprint1 as the current target, and ensure that we cannot
@@ -1489,8 +1494,8 @@ mod tests {
 
         // Check that we added the new sled and its zones.
         assert_eq!(
-            blueprint1.omicron_zones.len() + 1,
-            blueprint2.omicron_zones.len()
+            blueprint1.blueprint_zones.len() + 1,
+            blueprint2.blueprint_zones.len()
         );
         assert_eq!(
             blueprint1.all_omicron_zones().count() + num_new_sled_zones,
@@ -1498,10 +1503,7 @@ mod tests {
         );
 
         // All zones should be in service.
-        assert_eq!(
-            blueprint2.zones_in_service.len(),
-            blueprint2.all_omicron_zones().count()
-        );
+        assert_all_zones_in_service(&blueprint2);
         assert_eq!(blueprint2.parent_blueprint_id, Some(blueprint1.id));
 
         // Check that we can write it to the DB and read it back.
@@ -1869,4 +1871,18 @@ mod tests {
         db.cleanup().await.unwrap();
         logctx.cleanup_successful();
     }
+
+    fn assert_all_zones_in_service(blueprint: &Blueprint) {
+        let not_in_service = blueprint
+            .all_blueprint_zones(BlueprintZoneFilter::All)
+            .filter(|(_, z)| {
+                z.disposition != BlueprintZoneDisposition::InService
+            })
+            .collect::<Vec<_>>();
+        assert!(
+            not_in_service.is_empty(),
+            "expected all zones to be in service, \
+             found these zones not in service: {not_in_service:?}"
+        );
+    }
 }
diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs
index c86c002c33..94e033ec3c 100644
--- a/nexus/db-queries/src/db/datastore/rack.rs
+++ b/nexus/db-queries/src/db/datastore/rack.rs
@@ -952,7 +952,7 @@ mod test {
     };
     use omicron_common::api::internal::shared::SourceNatConfig;
     use omicron_test_utils::dev;
-    use std::collections::{BTreeMap, BTreeSet, HashMap};
+    use std::collections::{BTreeMap, HashMap};
     use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddrV6};
     use std::num::NonZeroU32;
 
@@ -965,8 +965,7 @@ mod test {
                 rack_subnet: nexus_test_utils::RACK_SUBNET.parse().unwrap(),
                 blueprint: Blueprint {
                     id: Uuid::new_v4(),
-                    omicron_zones: BTreeMap::new(),
-                    zones_in_service: BTreeSet::new(),
+                    blueprint_zones: BTreeMap::new(),
                     parent_blueprint_id: None,
                     internal_dns_version: *Generation::new(),
                     external_dns_version: *Generation::new(),
diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs
index 8be21348bf..dd05498038 100644
--- a/nexus/db-queries/src/db/datastore/vpc.rs
+++ b/nexus/db-queries/src/db/datastore/vpc.rs
@@ -43,6 +43,8 @@ use diesel::prelude::*;
 use diesel::result::DatabaseErrorKind;
 use diesel::result::Error as DieselError;
 use ipnetwork::IpNetwork;
+use nexus_types::deployment::BlueprintZoneDisposition;
+use nexus_types::deployment::BlueprintZoneFilter;
 use omicron_common::api::external::http_pagination::PaginatedBy;
 use omicron_common::api::external::CreateResult;
 use omicron_common::api::external::DeleteResult;
@@ -694,6 +696,44 @@ impl DataStore {
 
         // ... and we also need to query for the current target blueprint to
         // support systems that _are_ under Reconfigurator control.
+
+        {
+            // Ideally this would do something like:
+            //
+            // .filter(bp_omicron_zone::disposition.eq_any(
+            //     BlueprintZoneDisposition::all_matching(
+            //         BlueprintZoneFilter::VpcFirewall,
+            //     ),
+            // )
+            //
+            // But that doesn't quite work today because we currently don't
+            // store the disposition enum next to each zone. Instead, this code
+            // makes its decision to select which sleds to return by just
+            // ignoring the zones_in_service table today.
+            //
+            // The purpose of this otherwise pointless block is to ensure that
+            // it is correct to ensure that the expressed logic by
+            // `BlueprintZoneFilter::VpcFirewall` matches the actual
+            // implementation. It will hopefully soon be replaced with storing
+            // the disposition in the bp_omicron_zone table and using the
+            // filter directly.
+
+            let mut matching = BlueprintZoneDisposition::all_matching(
+                BlueprintZoneFilter::VpcFirewall,
+            )
+            .collect::<Vec<_>>();
+            matching.sort();
+            let mut all = BlueprintZoneDisposition::all_matching(
+                BlueprintZoneFilter::All,
+            )
+            .collect::<Vec<_>>();
+            all.sort();
+            debug_assert_eq!(
+                matching, all,
+                "vpc firewall dispositions should match all dispositions"
+            );
+        }
+
         let reconfig_service_query = service_network_interface::table
             .inner_join(bp_omicron_zone::table.on(
                 bp_omicron_zone::id.eq(service_network_interface::service_id),
@@ -1252,9 +1292,11 @@ mod tests {
     use nexus_test_utils::db::test_setup_database;
     use nexus_types::deployment::Blueprint;
     use nexus_types::deployment::BlueprintTarget;
+    use nexus_types::deployment::BlueprintZoneConfig;
+    use nexus_types::deployment::BlueprintZoneDisposition;
+    use nexus_types::deployment::BlueprintZonesConfig;
     use nexus_types::deployment::OmicronZoneConfig;
     use nexus_types::deployment::OmicronZoneType;
-    use nexus_types::deployment::OmicronZonesConfig;
     use nexus_types::external_api::params;
     use nexus_types::identity::Asset;
     use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET;
@@ -1268,7 +1310,6 @@ mod tests {
     use omicron_test_utils::dev;
     use slog::info;
     use std::collections::BTreeMap;
-    use std::collections::BTreeSet;
     use std::net::IpAddr;
 
     // Test that we detect the right error condition and return None when we
@@ -1570,11 +1611,11 @@ mod tests {
             })
         }
 
-        fn omicron_zone_configs(
+        fn blueprint_zone_configs(
             &self,
-        ) -> impl Iterator<Item = (Uuid, OmicronZoneConfig)> + '_ {
+        ) -> impl Iterator<Item = (Uuid, BlueprintZoneConfig)> + '_ {
             self.db_services().map(|(service, nic)| {
-                let zone_config = OmicronZoneConfig {
+                let config = OmicronZoneConfig {
                     id: service.id(),
                     underlay_address: "::1".parse().unwrap(),
                     zone_type: OmicronZoneType::Nexus {
@@ -1599,6 +1640,10 @@ mod tests {
                         external_dns_servers: Vec::new(),
                     },
                 };
+                let zone_config = BlueprintZoneConfig {
+                    config,
+                    disposition: BlueprintZoneDisposition::InService,
+                };
                 (service.sled_id, zone_config)
             })
         }
@@ -1652,15 +1697,15 @@ mod tests {
         // Create a blueprint that has a Nexus on our third sled. (This
         // blueprint is completely invalid in many ways, but all we care about
         // here is inserting relevant records in `bp_omicron_zone`.)
-        let bp1_omicron_zones = {
+        let bp1_zones = {
             let (sled_id, zone_config) = harness
-                .omicron_zone_configs()
+                .blueprint_zone_configs()
                 .nth(2)
                 .expect("fewer than 3 services in test harness");
             let mut zones = BTreeMap::new();
             zones.insert(
                 sled_id,
-                OmicronZonesConfig {
+                BlueprintZonesConfig {
                     generation: Generation::new(),
                     zones: vec![zone_config],
                 },
@@ -1670,8 +1715,7 @@ mod tests {
         let bp1_id = Uuid::new_v4();
         let bp1 = Blueprint {
             id: bp1_id,
-            omicron_zones: bp1_omicron_zones,
-            zones_in_service: BTreeSet::new(),
+            blueprint_zones: bp1_zones,
             parent_blueprint_id: None,
             internal_dns_version: Generation::new(),
             external_dns_version: Generation::new(),
@@ -1722,8 +1766,7 @@ mod tests {
         let bp2_id = Uuid::new_v4();
         let bp2 = Blueprint {
             id: bp2_id,
-            omicron_zones: BTreeMap::new(),
-            zones_in_service: BTreeSet::new(),
+            blueprint_zones: BTreeMap::new(),
             parent_blueprint_id: Some(bp1_id),
             internal_dns_version: Generation::new(),
             external_dns_version: Generation::new(),
@@ -1765,15 +1808,15 @@ mod tests {
 
         // Create a blueprint that has a Nexus on our fourth sled. This
         // shouldn't change our VPC resolution.
-        let bp3_omicron_zones = {
+        let bp3_zones = {
             let (sled_id, zone_config) = harness
-                .omicron_zone_configs()
+                .blueprint_zone_configs()
                 .nth(3)
                 .expect("fewer than 3 services in test harness");
             let mut zones = BTreeMap::new();
             zones.insert(
                 sled_id,
-                OmicronZonesConfig {
+                BlueprintZonesConfig {
                     generation: Generation::new(),
                     zones: vec![zone_config],
                 },
@@ -1783,8 +1826,7 @@ mod tests {
         let bp3_id = Uuid::new_v4();
         let bp3 = Blueprint {
             id: bp3_id,
-            omicron_zones: bp3_omicron_zones,
-            zones_in_service: BTreeSet::new(),
+            blueprint_zones: bp3_zones,
             parent_blueprint_id: Some(bp2_id),
             internal_dns_version: Generation::new(),
             external_dns_version: Generation::new(),
@@ -1819,13 +1861,14 @@ mod tests {
 
         // Finally, create a blueprint that includes our third and fourth sleds,
         // make it the target, and ensure we resolve to all four sleds.
-        let bp4_omicron_zones = {
+        let bp4_zones = {
             let mut zones = BTreeMap::new();
-            for (sled_id, zone_config) in harness.omicron_zone_configs().skip(2)
+            for (sled_id, zone_config) in
+                harness.blueprint_zone_configs().skip(2)
             {
                 zones.insert(
                     sled_id,
-                    OmicronZonesConfig {
+                    BlueprintZonesConfig {
                         generation: Generation::new(),
                         zones: vec![zone_config],
                     },
@@ -1836,8 +1879,7 @@ mod tests {
         let bp4_id = Uuid::new_v4();
         let bp4 = Blueprint {
             id: bp4_id,
-            omicron_zones: bp4_omicron_zones,
-            zones_in_service: BTreeSet::new(),
+            blueprint_zones: bp4_zones,
             parent_blueprint_id: Some(bp3_id),
             internal_dns_version: Generation::new(),
             external_dns_version: Generation::new(),
diff --git a/nexus/reconfigurator/execution/src/datasets.rs b/nexus/reconfigurator/execution/src/datasets.rs
index 6da24e4279..1d08f3b294 100644
--- a/nexus/reconfigurator/execution/src/datasets.rs
+++ b/nexus/reconfigurator/execution/src/datasets.rs
@@ -19,11 +19,11 @@ use slog_error_chain::InlineErrorChain;
 use std::collections::BTreeSet;
 use std::net::SocketAddrV6;
 
-/// For each crucible zone in `blueprint`, ensure that a corresponding dataset
-/// record exists in `datastore`
+/// For each crucible zone in `all_omicron_zones`, ensure that a corresponding
+/// dataset record exists in `datastore`
 ///
-/// Does not modify any existing dataset records. Returns the number of datasets
-/// inserted.
+/// Does not modify any existing dataset records. Returns the number of
+/// datasets inserted.
 pub(crate) async fn ensure_crucible_dataset_records_exist(
     opctx: &OpContext,
     datastore: &DataStore,
diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs
index c06db2c23e..7d7e24b6cf 100644
--- a/nexus/reconfigurator/execution/src/dns.rs
+++ b/nexus/reconfigurator/execution/src/dns.rs
@@ -18,6 +18,7 @@ use nexus_db_queries::db::datastore::DnsVersionUpdateBuilder;
 use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID;
 use nexus_db_queries::db::DataStore;
 use nexus_types::deployment::Blueprint;
+use nexus_types::deployment::BlueprintZoneFilter;
 use nexus_types::deployment::OmicronZoneType;
 use nexus_types::identity::Resource;
 use nexus_types::internal_api::params::DnsConfigParams;
@@ -248,19 +249,18 @@ pub fn blueprint_internal_dns_config(
             .map(|addr| addr.port())
     }
 
-    for (_, omicron_zone) in blueprint.all_omicron_zones() {
-        if !blueprint.zones_in_service.contains(&omicron_zone.id) {
-            continue;
-        }
-
+    for (_, zone) in
+        blueprint.all_blueprint_zones(BlueprintZoneFilter::InternalDns)
+    {
         let context = || {
             format!(
                 "parsing {} zone with id {}",
-                omicron_zone.zone_type.label(),
-                omicron_zone.id
+                zone.config.zone_type.label(),
+                zone.config.id
             )
         };
-        let (service_name, port) = match &omicron_zone.zone_type {
+
+        let (service_name, port) = match &zone.config.zone_type {
             OmicronZoneType::BoundaryNtp { address, .. } => {
                 let port = parse_port(&address).with_context(context)?;
                 (ServiceName::BoundaryNtp, port)
@@ -288,7 +288,7 @@ pub fn blueprint_internal_dns_config(
             }
             OmicronZoneType::Crucible { address, .. } => {
                 let port = parse_port(address).with_context(context)?;
-                (ServiceName::Crucible(omicron_zone.id), port)
+                (ServiceName::Crucible(zone.config.id), port)
             }
             OmicronZoneType::CruciblePantry { address } => {
                 let port = parse_port(address).with_context(context)?;
@@ -312,8 +312,8 @@ pub fn blueprint_internal_dns_config(
         // the same zone id twice, which should not be possible here.
         dns_builder
             .host_zone_with_one_backend(
-                omicron_zone.id,
-                omicron_zone.underlay_address,
+                zone.config.id,
+                zone.config.underlay_address,
                 service_name,
                 port,
             )
@@ -469,6 +469,8 @@ mod test {
     use nexus_test_utils_macros::nexus_test;
     use nexus_types::deployment::Blueprint;
     use nexus_types::deployment::BlueprintTarget;
+    use nexus_types::deployment::BlueprintZoneConfig;
+    use nexus_types::deployment::BlueprintZoneDisposition;
     use nexus_types::deployment::OmicronZoneConfig;
     use nexus_types::deployment::OmicronZoneType;
     use nexus_types::deployment::Policy;
@@ -608,24 +610,27 @@ mod test {
         // not currently in service.
         let out_of_service_id = Uuid::new_v4();
         let out_of_service_addr = Ipv6Addr::LOCALHOST;
-        blueprint.omicron_zones.values_mut().next().unwrap().zones.push(
-            OmicronZoneConfig {
-                id: out_of_service_id,
-                underlay_address: out_of_service_addr,
-                zone_type: OmicronZoneType::Oximeter {
-                    address: SocketAddrV6::new(
-                        out_of_service_addr,
-                        12345,
-                        0,
-                        0,
-                    )
-                    .to_string(),
+        blueprint.blueprint_zones.values_mut().next().unwrap().zones.push(
+            BlueprintZoneConfig {
+                config: OmicronZoneConfig {
+                    id: out_of_service_id,
+                    underlay_address: out_of_service_addr,
+                    zone_type: OmicronZoneType::Oximeter {
+                        address: SocketAddrV6::new(
+                            out_of_service_addr,
+                            12345,
+                            0,
+                            0,
+                        )
+                        .to_string(),
+                    },
                 },
+                disposition: BlueprintZoneDisposition::Quiesced,
             },
         );
 
         // To generate the blueprint's DNS config, we need to make up a
-        // different set of information about the sleds in our fake system.
+        // different set of information about the Quiesced fake system.
         let sleds_by_id = policy
             .sleds
             .iter()
diff --git a/nexus/reconfigurator/execution/src/lib.rs b/nexus/reconfigurator/execution/src/lib.rs
index 8da4e7d44c..1373c9a31f 100644
--- a/nexus/reconfigurator/execution/src/lib.rs
+++ b/nexus/reconfigurator/execution/src/lib.rs
@@ -98,7 +98,7 @@ where
     resource_allocation::ensure_zone_resources_allocated(
         &opctx,
         datastore,
-        &blueprint.omicron_zones,
+        blueprint.all_omicron_zones().map(|(_sled_id, zone)| zone),
     )
     .await
     .map_err(|err| vec![err])?;
@@ -111,8 +111,12 @@ where
         .into_iter()
         .map(|db_sled| (db_sled.id(), Sled::from(db_sled)))
         .collect();
-    omicron_zones::deploy_zones(&opctx, &sleds_by_id, &blueprint.omicron_zones)
-        .await?;
+    omicron_zones::deploy_zones(
+        &opctx,
+        &sleds_by_id,
+        &blueprint.blueprint_zones,
+    )
+    .await?;
 
     // After deploying omicron zones, we may need to refresh OPTE service
     // firewall rules. This is an idempotent operation, so we don't attempt
diff --git a/nexus/reconfigurator/execution/src/omicron_zones.rs b/nexus/reconfigurator/execution/src/omicron_zones.rs
index 6054a40f8a..0150c40e9e 100644
--- a/nexus/reconfigurator/execution/src/omicron_zones.rs
+++ b/nexus/reconfigurator/execution/src/omicron_zones.rs
@@ -10,7 +10,8 @@ use anyhow::Context;
 use futures::stream;
 use futures::StreamExt;
 use nexus_db_queries::context::OpContext;
-use nexus_types::deployment::OmicronZonesConfig;
+use nexus_types::deployment::BlueprintZoneFilter;
+use nexus_types::deployment::BlueprintZonesConfig;
 use slog::info;
 use slog::warn;
 use std::collections::BTreeMap;
@@ -21,7 +22,7 @@ use uuid::Uuid;
 pub(crate) async fn deploy_zones(
     opctx: &OpContext,
     sleds_by_id: &BTreeMap<Uuid, Sled>,
-    zones: &BTreeMap<Uuid, OmicronZonesConfig>,
+    zones: &BTreeMap<Uuid, BlueprintZonesConfig>,
 ) -> Result<(), Vec<anyhow::Error>> {
     let errors: Vec<_> = stream::iter(zones)
         .filter_map(|(sled_id, config)| async move {
@@ -33,14 +34,21 @@ pub(crate) async fn deploy_zones(
                     return Some(err);
                 }
             };
+
             let client = nexus_networking::sled_client_from_address(
                 *sled_id,
                 db_sled.sled_agent_address,
                 &opctx.log,
             );
-            let result =
-                client.omicron_zones_put(&config).await.with_context(|| {
-                    format!("Failed to put {config:#?} to sled {sled_id}")
+            let omicron_zones = config
+                .to_omicron_zones_config(BlueprintZoneFilter::SledAgentPut);
+            let result = client
+                .omicron_zones_put(&omicron_zones)
+                .await
+                .with_context(|| {
+                    format!(
+                        "Failed to put {omicron_zones:#?} to sled {sled_id}"
+                    )
                 });
             match result {
                 Err(error) => {
@@ -78,22 +86,23 @@ mod test {
     use nexus_db_queries::context::OpContext;
     use nexus_test_utils_macros::nexus_test;
     use nexus_types::deployment::OmicronZonesConfig;
-    use nexus_types::deployment::{Blueprint, BlueprintTarget};
+    use nexus_types::deployment::{
+        Blueprint, BlueprintTarget, BlueprintZoneConfig,
+        BlueprintZoneDisposition, BlueprintZonesConfig,
+    };
     use nexus_types::inventory::{
         OmicronZoneConfig, OmicronZoneDataset, OmicronZoneType,
     };
     use omicron_common::api::external::Generation;
     use std::collections::BTreeMap;
-    use std::collections::BTreeSet;
     use std::net::SocketAddr;
-    use std::sync::Arc;
     use uuid::Uuid;
 
     type ControlPlaneTestContext =
         nexus_test_utils::ControlPlaneTestContext<omicron_nexus::Server>;
 
     fn create_blueprint(
-        omicron_zones: BTreeMap<Uuid, OmicronZonesConfig>,
+        blueprint_zones: BTreeMap<Uuid, BlueprintZonesConfig>,
     ) -> (BlueprintTarget, Blueprint) {
         let id = Uuid::new_v4();
         (
@@ -104,8 +113,7 @@ mod test {
             },
             Blueprint {
                 id,
-                omicron_zones,
-                zones_in_service: BTreeSet::new(),
+                blueprint_zones,
                 parent_blueprint_id: None,
                 internal_dns_version: Generation::new(),
                 external_dns_version: Generation::new(),
@@ -149,31 +157,34 @@ mod test {
 
         // Get a success result back when the blueprint has an empty set of
         // zones.
-        let blueprint = Arc::new(create_blueprint(BTreeMap::new()));
-        deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones)
+        let (_, blueprint) = create_blueprint(BTreeMap::new());
+        deploy_zones(&opctx, &sleds_by_id, &blueprint.blueprint_zones)
             .await
             .expect("failed to deploy no zones");
 
         // Zones are updated in a particular order, but each request contains
         // the full set of zones that must be running.
         // See `rack_setup::service::ServiceInner::run` for more details.
-        fn make_zones() -> OmicronZonesConfig {
-            OmicronZonesConfig {
+        fn make_zones() -> BlueprintZonesConfig {
+            BlueprintZonesConfig {
                 generation: Generation::new(),
-                zones: vec![OmicronZoneConfig {
-                    id: Uuid::new_v4(),
-                    underlay_address: "::1".parse().unwrap(),
-                    zone_type: OmicronZoneType::InternalDns {
-                        dataset: OmicronZoneDataset {
-                            pool_name: format!("oxp_{}", Uuid::new_v4())
-                                .parse()
-                                .unwrap(),
+                zones: vec![BlueprintZoneConfig {
+                    config: OmicronZoneConfig {
+                        id: Uuid::new_v4(),
+                        underlay_address: "::1".parse().unwrap(),
+                        zone_type: OmicronZoneType::InternalDns {
+                            dataset: OmicronZoneDataset {
+                                pool_name: format!("oxp_{}", Uuid::new_v4())
+                                    .parse()
+                                    .unwrap(),
+                            },
+                            dns_address: "oh-hello-internal-dns".into(),
+                            gz_address: "::1".parse().unwrap(),
+                            gz_address_index: 0,
+                            http_address: "some-ipv6-address".into(),
                         },
-                        dns_address: "oh-hello-internal-dns".into(),
-                        gz_address: "::1".parse().unwrap(),
-                        gz_address_index: 0,
-                        http_address: "some-ipv6-address".into(),
                     },
+                    disposition: BlueprintZoneDisposition::InService,
                 }],
             }
         }
@@ -183,10 +194,10 @@ mod test {
         // matter for this test.
         let mut zones1 = make_zones();
         let mut zones2 = make_zones();
-        let blueprint = Arc::new(create_blueprint(BTreeMap::from([
+        let (_, blueprint) = create_blueprint(BTreeMap::from([
             (sled_id1, zones1.clone()),
             (sled_id2, zones2.clone()),
-        ])));
+        ]));
 
         // Set expectations for the initial requests sent to the fake
         // sled-agents.
@@ -205,7 +216,7 @@ mod test {
         }
 
         // Execute it.
-        deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones)
+        deploy_zones(&opctx, &sleds_by_id, &blueprint.blueprint_zones)
             .await
             .expect("failed to deploy initial zones");
 
@@ -222,7 +233,7 @@ mod test {
                 .respond_with(status_code(204)),
             );
         }
-        deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones)
+        deploy_zones(&opctx, &sleds_by_id, &blueprint.blueprint_zones)
             .await
             .expect("failed to deploy same zones");
         s1.verify_and_clear();
@@ -246,7 +257,7 @@ mod test {
         );
 
         let errors =
-            deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones)
+            deploy_zones(&opctx, &sleds_by_id, &blueprint.blueprint_zones)
                 .await
                 .expect_err("unexpectedly succeeded in deploying zones");
 
@@ -259,26 +270,35 @@ mod test {
         s2.verify_and_clear();
 
         // Add an `InternalNtp` zone for our next update
-        fn append_zone(zones: &mut OmicronZonesConfig) {
+        fn append_zone(
+            zones: &mut BlueprintZonesConfig,
+            disposition: BlueprintZoneDisposition,
+        ) {
             zones.generation = zones.generation.next();
-            zones.zones.push(OmicronZoneConfig {
-                id: Uuid::new_v4(),
-                underlay_address: "::1".parse().unwrap(),
-                zone_type: OmicronZoneType::InternalNtp {
-                    address: "::1".into(),
-                    dns_servers: vec!["::1".parse().unwrap()],
-                    domain: None,
-                    ntp_servers: vec!["some-ntp-server-addr".into()],
+            zones.zones.push(BlueprintZoneConfig {
+                config: OmicronZoneConfig {
+                    id: Uuid::new_v4(),
+                    underlay_address: "::1".parse().unwrap(),
+                    zone_type: OmicronZoneType::InternalNtp {
+                        address: "::1".into(),
+                        dns_servers: vec!["::1".parse().unwrap()],
+                        domain: None,
+                        ntp_servers: vec!["some-ntp-server-addr".into()],
+                    },
                 },
+                disposition,
             });
         }
 
-        append_zone(&mut zones1);
-        append_zone(&mut zones2);
-        let blueprint = Arc::new(create_blueprint(BTreeMap::from([
+        // Both in-service and quiesced zones should be deployed.
+        //
+        // TODO: add expunged zones to the test (should not be deployed).
+        append_zone(&mut zones1, BlueprintZoneDisposition::InService);
+        append_zone(&mut zones2, BlueprintZoneDisposition::Quiesced);
+        let (_, blueprint) = create_blueprint(BTreeMap::from([
             (sled_id1, zones1),
             (sled_id2, zones2),
-        ])));
+        ]));
 
         // Set our new expectations
         for s in [&mut s1, &mut s2] {
@@ -296,7 +316,7 @@ mod test {
         }
 
         // Activate the task
-        deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones)
+        deploy_zones(&opctx, &sleds_by_id, &blueprint.blueprint_zones)
             .await
             .expect("failed to deploy last round of zones");
         s1.verify_and_clear();
diff --git a/nexus/reconfigurator/execution/src/resource_allocation.rs b/nexus/reconfigurator/execution/src/resource_allocation.rs
index 5872cee0f9..92262ce133 100644
--- a/nexus/reconfigurator/execution/src/resource_allocation.rs
+++ b/nexus/reconfigurator/execution/src/resource_allocation.rs
@@ -16,14 +16,13 @@ use nexus_db_queries::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET;
 use nexus_db_queries::db::fixed_data::vpc_subnet::NTP_VPC_SUBNET;
 use nexus_db_queries::db::DataStore;
 use nexus_types::deployment::OmicronZoneType;
-use nexus_types::deployment::OmicronZonesConfig;
 use nexus_types::deployment::SourceNatConfig;
+use nexus_types::inventory::OmicronZoneConfig;
 use omicron_common::api::external::IdentityMetadataCreateParams;
 use omicron_common::api::internal::shared::NetworkInterface;
 use omicron_common::api::internal::shared::NetworkInterfaceKind;
 use slog::info;
 use slog::warn;
-use std::collections::BTreeMap;
 use std::net::IpAddr;
 use std::net::SocketAddr;
 use uuid::Uuid;
@@ -31,47 +30,45 @@ use uuid::Uuid;
 pub(crate) async fn ensure_zone_resources_allocated(
     opctx: &OpContext,
     datastore: &DataStore,
-    zones: &BTreeMap<Uuid, OmicronZonesConfig>,
+    all_omicron_zones: impl Iterator<Item = &OmicronZoneConfig>,
 ) -> anyhow::Result<()> {
     let allocator = ResourceAllocator { opctx, datastore };
 
-    for config in zones.values() {
-        for z in &config.zones {
-            match &z.zone_type {
-                OmicronZoneType::Nexus { external_ip, nic, .. } => {
-                    allocator
-                        .ensure_nexus_external_networking_allocated(
-                            z.id,
-                            *external_ip,
-                            nic,
-                        )
-                        .await?;
-                }
-                OmicronZoneType::ExternalDns { dns_address, nic, .. } => {
-                    allocator
-                        .ensure_external_dns_external_networking_allocated(
-                            z.id,
-                            dns_address,
-                            nic,
-                        )
-                        .await?;
-                }
-                OmicronZoneType::BoundaryNtp { snat_cfg, nic, .. } => {
-                    allocator
-                        .ensure_boundary_ntp_external_networking_allocated(
-                            z.id, snat_cfg, nic,
-                        )
-                        .await?;
-                }
-                OmicronZoneType::InternalNtp { .. }
-                | OmicronZoneType::Clickhouse { .. }
-                | OmicronZoneType::ClickhouseKeeper { .. }
-                | OmicronZoneType::CockroachDb { .. }
-                | OmicronZoneType::Crucible { .. }
-                | OmicronZoneType::CruciblePantry { .. }
-                | OmicronZoneType::InternalDns { .. }
-                | OmicronZoneType::Oximeter { .. } => (),
+    for z in all_omicron_zones {
+        match &z.zone_type {
+            OmicronZoneType::Nexus { external_ip, nic, .. } => {
+                allocator
+                    .ensure_nexus_external_networking_allocated(
+                        z.id,
+                        *external_ip,
+                        nic,
+                    )
+                    .await?;
+            }
+            OmicronZoneType::ExternalDns { dns_address, nic, .. } => {
+                allocator
+                    .ensure_external_dns_external_networking_allocated(
+                        z.id,
+                        dns_address,
+                        nic,
+                    )
+                    .await?;
             }
+            OmicronZoneType::BoundaryNtp { snat_cfg, nic, .. } => {
+                allocator
+                    .ensure_boundary_ntp_external_networking_allocated(
+                        z.id, snat_cfg, nic,
+                    )
+                    .await?;
+            }
+            OmicronZoneType::InternalNtp { .. }
+            | OmicronZoneType::Clickhouse { .. }
+            | OmicronZoneType::ClickhouseKeeper { .. }
+            | OmicronZoneType::CockroachDb { .. }
+            | OmicronZoneType::Crucible { .. }
+            | OmicronZoneType::CruciblePantry { .. }
+            | OmicronZoneType::InternalDns { .. }
+            | OmicronZoneType::Oximeter { .. } => (),
         }
     }
 
@@ -510,7 +507,6 @@ mod tests {
     use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET;
     use omicron_common::address::NTP_OPTE_IPV4_SUBNET;
     use omicron_common::address::NUM_SOURCE_NAT_PORTS;
-    use omicron_common::api::external::Generation;
     use omicron_common::api::external::IpNet;
     use omicron_common::api::external::MacAddr;
     use omicron_common::api::external::Vni;
@@ -619,65 +615,56 @@ mod tests {
 
         // Build the `zones` map needed by `ensure_zone_resources_allocated`,
         // with an arbitrary sled_id.
-        let mut zones = BTreeMap::new();
-        let sled_id = Uuid::new_v4();
-        zones.insert(
-            sled_id,
-            OmicronZonesConfig {
-                generation: Generation::new().next(),
-                zones: vec![
-                    OmicronZoneConfig {
-                        id: nexus_id,
-                        underlay_address: Ipv6Addr::LOCALHOST,
-                        zone_type: OmicronZoneType::Nexus {
-                            internal_address: Ipv6Addr::LOCALHOST.to_string(),
-                            external_ip: nexus_external_ip,
-                            nic: nexus_nic.clone(),
-                            external_tls: false,
-                            external_dns_servers: Vec::new(),
-                        },
-                    },
-                    OmicronZoneConfig {
-                        id: dns_id,
-                        underlay_address: Ipv6Addr::LOCALHOST,
-                        zone_type: OmicronZoneType::ExternalDns {
-                            dataset: OmicronZoneDataset {
-                                pool_name: format!("oxp_{}", Uuid::new_v4())
-                                    .parse()
-                                    .expect("bad name"),
-                            },
-                            http_address: SocketAddrV6::new(
-                                Ipv6Addr::LOCALHOST,
-                                0,
-                                0,
-                                0,
-                            )
-                            .to_string(),
-                            dns_address: SocketAddr::new(dns_external_ip, 0)
-                                .to_string(),
-                            nic: dns_nic.clone(),
-                        },
-                    },
-                    OmicronZoneConfig {
-                        id: ntp_id,
-                        underlay_address: Ipv6Addr::LOCALHOST,
-                        zone_type: OmicronZoneType::BoundaryNtp {
-                            address: SocketAddr::new(dns_external_ip, 0)
-                                .to_string(),
-                            ntp_servers: Vec::new(),
-                            dns_servers: Vec::new(),
-                            domain: None,
-                            nic: ntp_nic.clone(),
-                            snat_cfg: ntp_snat,
-                        },
+        let zones = vec![
+            OmicronZoneConfig {
+                id: nexus_id,
+                underlay_address: Ipv6Addr::LOCALHOST,
+                zone_type: OmicronZoneType::Nexus {
+                    internal_address: Ipv6Addr::LOCALHOST.to_string(),
+                    external_ip: nexus_external_ip,
+                    nic: nexus_nic.clone(),
+                    external_tls: false,
+                    external_dns_servers: Vec::new(),
+                },
+            },
+            OmicronZoneConfig {
+                id: dns_id,
+                underlay_address: Ipv6Addr::LOCALHOST,
+                zone_type: OmicronZoneType::ExternalDns {
+                    dataset: OmicronZoneDataset {
+                        pool_name: format!("oxp_{}", Uuid::new_v4())
+                            .parse()
+                            .expect("bad name"),
                     },
-                ],
+                    http_address: SocketAddrV6::new(
+                        Ipv6Addr::LOCALHOST,
+                        0,
+                        0,
+                        0,
+                    )
+                    .to_string(),
+                    dns_address: SocketAddr::new(dns_external_ip, 0)
+                        .to_string(),
+                    nic: dns_nic.clone(),
+                },
             },
-        );
+            OmicronZoneConfig {
+                id: ntp_id,
+                underlay_address: Ipv6Addr::LOCALHOST,
+                zone_type: OmicronZoneType::BoundaryNtp {
+                    address: SocketAddr::new(dns_external_ip, 0).to_string(),
+                    ntp_servers: Vec::new(),
+                    dns_servers: Vec::new(),
+                    domain: None,
+                    nic: ntp_nic.clone(),
+                    snat_cfg: ntp_snat,
+                },
+            },
+        ];
 
         // Initialize resource allocation: this should succeed and create all
         // the relevant db records.
-        ensure_zone_resources_allocated(&opctx, datastore, &zones)
+        ensure_zone_resources_allocated(&opctx, datastore, zones.iter())
             .await
             .with_context(|| format!("{zones:#?}"))
             .unwrap();
@@ -761,7 +748,7 @@ mod tests {
 
         // We should be able to run the function again with the same inputs, and
         // it should succeed without inserting any new records.
-        ensure_zone_resources_allocated(&opctx, datastore, &zones)
+        ensure_zone_resources_allocated(&opctx, datastore, zones.iter())
             .await
             .with_context(|| format!("{zones:#?}"))
             .unwrap();
@@ -814,8 +801,8 @@ mod tests {
         let bogus_ip = external_ips.next().expect("exhausted external_ips");
         for mutate_zones_fn in [
             // non-matching IP on Nexus
-            (&|config: &mut OmicronZonesConfig| {
-                for zone in &mut config.zones {
+            (&|zones: &mut [OmicronZoneConfig]| {
+                for zone in zones {
                     if let OmicronZoneType::Nexus {
                         ref mut external_ip, ..
                     } = &mut zone.zone_type
@@ -827,11 +814,12 @@ mod tests {
                         );
                     }
                 }
+
                 panic!("didn't find expected zone");
-            }) as &dyn Fn(&mut OmicronZonesConfig) -> String,
+            }) as &dyn Fn(&mut [OmicronZoneConfig]) -> String,
             // non-matching IP on External DNS
-            &|config| {
-                for zone in &mut config.zones {
+            &|zones| {
+                for zone in zones {
                     if let OmicronZoneType::ExternalDns {
                         ref mut dns_address,
                         ..
@@ -847,8 +835,8 @@ mod tests {
                 panic!("didn't find expected zone");
             },
             // non-matching SNAT port range on Boundary NTP
-            &|config| {
-                for zone in &mut config.zones {
+            &|zones| {
+                for zone in zones {
                     if let OmicronZoneType::BoundaryNtp {
                         ref mut snat_cfg,
                         ..
@@ -866,24 +854,24 @@ mod tests {
             },
         ] {
             // Run `mutate_zones_fn` on our config...
-            let mut config =
-                zones.remove(&sled_id).expect("missing zone config");
-            let orig_config = config.clone();
-            let expected_error = mutate_zones_fn(&mut config);
-            zones.insert(sled_id, config);
-
-            // ... check that we get the error we expect
-            let err =
-                ensure_zone_resources_allocated(&opctx, datastore, &zones)
-                    .await
-                    .expect_err("unexpected success");
+            let (mutated_zones, expected_error) = {
+                let mut zones = zones.clone();
+                let expected_error = mutate_zones_fn(&mut zones);
+                (zones, expected_error)
+            };
+
+            // and check that we get the error we expect.
+            let err = ensure_zone_resources_allocated(
+                &opctx,
+                datastore,
+                mutated_zones.iter(),
+            )
+            .await
+            .expect_err("unexpected success");
             assert!(
                 err.to_string().contains(&expected_error),
                 "expected {expected_error:?}, got {err:#}"
             );
-
-            // ... and restore the original, valid config before iterating.
-            zones.insert(sled_id, orig_config);
         }
 
         // Also try some requests that ought to fail because the request
@@ -922,11 +910,7 @@ mod tests {
         ] {
             // Try this NIC mutation on Nexus...
             let mut mutated_zones = zones.clone();
-            for zone in &mut mutated_zones
-                .get_mut(&sled_id)
-                .expect("missing sled")
-                .zones
-            {
+            for zone in &mut mutated_zones {
                 if let OmicronZoneType::Nexus { ref mut nic, .. } =
                     &mut zone.zone_type
                 {
@@ -935,7 +919,7 @@ mod tests {
                     let err = ensure_zone_resources_allocated(
                         &opctx,
                         datastore,
-                        &mutated_zones,
+                        mutated_zones.iter(),
                     )
                     .await
                     .expect_err("unexpected success");
@@ -951,11 +935,7 @@ mod tests {
 
             // ... and again on ExternalDns
             let mut mutated_zones = zones.clone();
-            for zone in &mut mutated_zones
-                .get_mut(&sled_id)
-                .expect("missing sled")
-                .zones
-            {
+            for zone in &mut mutated_zones {
                 if let OmicronZoneType::ExternalDns { ref mut nic, .. } =
                     &mut zone.zone_type
                 {
@@ -964,7 +944,7 @@ mod tests {
                     let err = ensure_zone_resources_allocated(
                         &opctx,
                         datastore,
-                        &mutated_zones,
+                        mutated_zones.iter(),
                     )
                     .await
                     .expect_err("unexpected success");
@@ -980,11 +960,7 @@ mod tests {
 
             // ... and again on BoundaryNtp
             let mut mutated_zones = zones.clone();
-            for zone in &mut mutated_zones
-                .get_mut(&sled_id)
-                .expect("missing sled")
-                .zones
-            {
+            for zone in &mut mutated_zones {
                 if let OmicronZoneType::BoundaryNtp { ref mut nic, .. } =
                     &mut zone.zone_type
                 {
@@ -993,7 +969,7 @@ mod tests {
                     let err = ensure_zone_resources_allocated(
                         &opctx,
                         datastore,
-                        &mutated_zones,
+                        mutated_zones.iter(),
                     )
                     .await
                     .expect_err("unexpected success");
diff --git a/nexus/reconfigurator/planning/src/blueprint_builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder.rs
index c9cd79ea14..0b0d422916 100644
--- a/nexus/reconfigurator/planning/src/blueprint_builder.rs
+++ b/nexus/reconfigurator/planning/src/blueprint_builder.rs
@@ -13,10 +13,12 @@ use ipnet::IpAdd;
 use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use nexus_inventory::now_db_precision;
 use nexus_types::deployment::Blueprint;
+use nexus_types::deployment::BlueprintZoneConfig;
+use nexus_types::deployment::BlueprintZoneDisposition;
+use nexus_types::deployment::BlueprintZonesConfig;
 use nexus_types::deployment::OmicronZoneConfig;
 use nexus_types::deployment::OmicronZoneDataset;
 use nexus_types::deployment::OmicronZoneType;
-use nexus_types::deployment::OmicronZonesConfig;
 use nexus_types::deployment::Policy;
 use nexus_types::deployment::SledResources;
 use nexus_types::deployment::ZpoolName;
@@ -41,7 +43,6 @@ use rand::SeedableRng;
 use slog::o;
 use slog::Logger;
 use std::collections::BTreeMap;
-use std::collections::BTreeSet;
 use std::collections::HashSet;
 use std::hash::Hash;
 use std::net::IpAddr;
@@ -118,8 +119,7 @@ pub struct BlueprintBuilder<'a> {
 
     // These fields will become part of the final blueprint.  See the
     // corresponding fields in `Blueprint`.
-    zones: BlueprintZones<'a>,
-    zones_in_service: BTreeSet<Uuid>,
+    zones: BlueprintZonesBuilder<'a>,
     creator: String,
     comments: Vec<String>,
 
@@ -141,10 +141,10 @@ impl<'a> BlueprintBuilder<'a> {
     /// Directly construct a `Blueprint` from the contents of a particular
     /// collection (representing no changes from the collection state)
     pub fn build_initial_from_collection(
-        collection: &'a Collection,
+        collection: &Collection,
         internal_dns_version: Generation,
         external_dns_version: Generation,
-        policy: &'a Policy,
+        policy: &Policy,
         creator: &str,
     ) -> Result<Blueprint, Error> {
         Self::build_initial_impl(
@@ -160,10 +160,10 @@ impl<'a> BlueprintBuilder<'a> {
     /// A version of [`Self::build_initial_from_collection`] that allows the
     /// blueprint ID to be generated from a random seed.
     pub fn build_initial_from_collection_seeded<H: Hash>(
-        collection: &'a Collection,
+        collection: &Collection,
         internal_dns_version: Generation,
         external_dns_version: Generation,
-        policy: &'a Policy,
+        policy: &Policy,
         creator: &str,
         seed: H,
     ) -> Result<Blueprint, Error> {
@@ -180,21 +180,21 @@ impl<'a> BlueprintBuilder<'a> {
     }
 
     fn build_initial_impl(
-        collection: &'a Collection,
+        collection: &Collection,
         internal_dns_version: Generation,
         external_dns_version: Generation,
-        policy: &'a Policy,
+        policy: &Policy,
         creator: &str,
         mut rng: BlueprintBuilderRng,
     ) -> Result<Blueprint, Error> {
-        let omicron_zones = policy
+        let blueprint_zones = policy
             .sleds
             .keys()
             .map(|sled_id| {
-                let mut zones = collection
+                let zones = collection
                     .omicron_zones
                     .get(sled_id)
-                    .map(|z| z.zones.clone())
+                    .map(|z| &z.zones)
                     .ok_or_else(|| {
                         // We should not find a sled that's supposed to be
                         // in-service but is not part of the inventory.  It's
@@ -216,19 +216,15 @@ impl<'a> BlueprintBuilder<'a> {
                         ))
                     })?;
 
-                // This is not strictly necessary.  But for testing, it's
-                // helpful for things to be in sorted order.
-                zones.zones.sort_by_key(|zone| zone.id);
-
-                Ok((*sled_id, zones))
+                Ok((
+                    *sled_id,
+                    BlueprintZonesConfig::initial_from_collection(&zones),
+                ))
             })
             .collect::<Result<_, Error>>()?;
-        let zones_in_service =
-            collection.all_omicron_zones().map(|z| z.id).collect();
         Ok(Blueprint {
             id: rng.blueprint_rng.next_uuid(),
-            omicron_zones,
-            zones_in_service,
+            blueprint_zones,
             parent_blueprint_id: None,
             internal_dns_version,
             external_dns_version,
@@ -291,7 +287,8 @@ impl<'a> BlueprintBuilder<'a> {
         let mut used_macs: HashSet<MacAddr> = HashSet::new();
 
         for (_, z) in parent_blueprint.all_omicron_zones() {
-            if let OmicronZoneType::Nexus { nic, .. } = &z.zone_type {
+            let zone_type = &z.zone_type;
+            if let OmicronZoneType::Nexus { nic, .. } = zone_type {
                 match nic.ip {
                     IpAddr::V4(ip) => {
                         if !existing_nexus_v4_ips.insert(ip) {
@@ -305,7 +302,8 @@ impl<'a> BlueprintBuilder<'a> {
                     }
                 }
             }
-            if let Some(external_ip) = z.zone_type.external_ip()? {
+
+            if let Some(external_ip) = zone_type.external_ip()? {
                 // For the test suite, ignore localhost.  It gets reused many
                 // times and that's okay.  We don't expect to see localhost
                 // outside the test suite.
@@ -315,7 +313,7 @@ impl<'a> BlueprintBuilder<'a> {
                     bail!("duplicate external IP: {external_ip}");
                 }
             }
-            if let Some(nic) = z.zone_type.service_vnic() {
+            if let Some(nic) = zone_type.service_vnic() {
                 if !used_macs.insert(nic.mac) {
                     bail!("duplicate service vNIC MAC: {}", nic.mac);
                 }
@@ -360,8 +358,7 @@ impl<'a> BlueprintBuilder<'a> {
             external_dns_version,
             policy,
             sled_ip_allocators: BTreeMap::new(),
-            zones: BlueprintZones::new(parent_blueprint),
-            zones_in_service: parent_blueprint.zones_in_service.clone(),
+            zones: BlueprintZonesBuilder::new(parent_blueprint),
             creator: creator.to_owned(),
             comments: Vec::new(),
             nexus_v4_ips,
@@ -375,12 +372,11 @@ impl<'a> BlueprintBuilder<'a> {
     /// Assemble a final [`Blueprint`] based on the contents of the builder
     pub fn build(mut self) -> Blueprint {
         // Collect the Omicron zones config for each in-service sled.
-        let omicron_zones =
-            self.zones.into_omicron_zones(self.policy.sleds.keys().copied());
+        let blueprint_zones =
+            self.zones.into_zones_map(self.policy.sleds.keys().copied());
         Blueprint {
             id: self.rng.blueprint_rng.next_uuid(),
-            omicron_zones,
-            zones_in_service: self.zones_in_service,
+            blueprint_zones,
             parent_blueprint_id: Some(self.parent_blueprint.id),
             internal_dns_version: self.internal_dns_version,
             external_dns_version: self.external_dns_version,
@@ -418,7 +414,7 @@ impl<'a> BlueprintBuilder<'a> {
         let has_ntp = self
             .zones
             .current_sled_zones(sled_id)
-            .any(|z| z.zone_type.is_ntp());
+            .any(|z| z.config.zone_type.is_ntp());
         if has_ntp {
             return Ok(Ensure::NotNeeded);
         }
@@ -465,6 +461,10 @@ impl<'a> BlueprintBuilder<'a> {
                 domain: None,
             },
         };
+        let zone = BlueprintZoneConfig {
+            config: zone,
+            disposition: BlueprintZoneDisposition::InService,
+        };
 
         self.sled_add_zone(sled_id, zone)?;
         Ok(Ensure::Added)
@@ -479,7 +479,7 @@ impl<'a> BlueprintBuilder<'a> {
         let has_crucible_on_this_pool =
             self.zones.current_sled_zones(sled_id).any(|z| {
                 matches!(
-                    &z.zone_type,
+                    &z.config.zone_type,
                     OmicronZoneType::Crucible { dataset, .. }
                     if dataset.pool_name == pool_name
                 )
@@ -509,6 +509,11 @@ impl<'a> BlueprintBuilder<'a> {
                 dataset: OmicronZoneDataset { pool_name },
             },
         };
+
+        let zone = BlueprintZoneConfig {
+            config: zone,
+            disposition: BlueprintZoneDisposition::InService,
+        };
         self.sled_add_zone(sled_id, zone)?;
         Ok(Ensure::Added)
     }
@@ -521,7 +526,7 @@ impl<'a> BlueprintBuilder<'a> {
     pub fn sled_num_nexus_zones(&self, sled_id: Uuid) -> usize {
         self.zones
             .current_sled_zones(sled_id)
-            .filter(|z| z.zone_type.is_nexus())
+            .filter(|z| z.config.zone_type.is_nexus())
             .count()
     }
 
@@ -543,17 +548,14 @@ impl<'a> BlueprintBuilder<'a> {
         // settings should be part of `Policy` instead?
         let (external_tls, external_dns_servers) = self
             .parent_blueprint
-            .omicron_zones
-            .values()
-            .find_map(|sled_zones| {
-                sled_zones.zones.iter().find_map(|z| match &z.zone_type {
-                    OmicronZoneType::Nexus {
-                        external_tls,
-                        external_dns_servers,
-                        ..
-                    } => Some((*external_tls, external_dns_servers.clone())),
-                    _ => None,
-                })
+            .all_omicron_zones()
+            .find_map(|(_, z)| match &z.zone_type {
+                OmicronZoneType::Nexus {
+                    external_tls,
+                    external_dns_servers,
+                    ..
+                } => Some((*external_tls, external_dns_servers.clone())),
+                _ => None,
             })
             .ok_or(Error::NoNexusZonesInParentBlueprint)?;
         self.sled_ensure_zone_multiple_nexus_with_config(
@@ -642,6 +644,10 @@ impl<'a> BlueprintBuilder<'a> {
                     external_dns_servers: external_dns_servers.clone(),
                 },
             };
+            let zone = BlueprintZoneConfig {
+                config: zone,
+                disposition: BlueprintZoneDisposition::InService,
+            };
             self.sled_add_zone(sled_id, zone)?;
         }
 
@@ -651,19 +657,20 @@ impl<'a> BlueprintBuilder<'a> {
     fn sled_add_zone(
         &mut self,
         sled_id: Uuid,
-        zone: OmicronZoneConfig,
+        zone: BlueprintZoneConfig,
     ) -> Result<(), Error> {
         // Check the sled id and return an appropriate error if it's invalid.
         let _ = self.sled_resources(sled_id)?;
 
-        if !self.zones_in_service.insert(zone.id) {
+        let sled_zones = self.zones.change_sled_zones(sled_id);
+        // A sled should have a small number (< 20) of zones so a linear search
+        // should be very fast.
+        if sled_zones.zones.iter().any(|z| z.config.id == zone.config.id) {
             return Err(Error::Planner(anyhow!(
                 "attempted to add zone that already exists: {}",
-                zone.id
+                zone.config.id
             )));
         }
-
-        let sled_zones = self.zones.change_sled_zones(sled_id);
         sled_zones.zones.push(zone);
         Ok(())
     }
@@ -698,7 +705,7 @@ impl<'a> BlueprintBuilder<'a> {
                 // Record each of the sled's zones' underlay addresses as
                 // allocated.
                 for z in self.zones.current_sled_zones(sled_id) {
-                    allocator.reserve(z.underlay_address);
+                    allocator.reserve(z.config.underlay_address);
                 }
 
                 allocator
@@ -790,19 +797,19 @@ impl UuidRng {
 /// Tracking the set of zones is slightly non-trivial because we need to bump
 /// the per-sled generation number iff the zones are changed.  So we need to
 /// keep track of whether we've changed the zones relative to the parent
-/// blueprint.  We do this by keeping a copy of any `OmicronZonesConfig` that
-/// we've changed and a _reference_ to the parent blueprint's zones.  This
+/// blueprint.  We do this by keeping a copy of any [`BlueprintZonesConfig`]
+/// that we've changed and a _reference_ to the parent blueprint's zones.  This
 /// struct makes it easy for callers iterate over the right set of zones.
-struct BlueprintZones<'a> {
-    changed_zones: BTreeMap<Uuid, OmicronZonesConfig>,
-    parent_zones: &'a BTreeMap<Uuid, OmicronZonesConfig>,
+struct BlueprintZonesBuilder<'a> {
+    changed_zones: BTreeMap<Uuid, BlueprintZonesConfig>,
+    parent_zones: &'a BTreeMap<Uuid, BlueprintZonesConfig>,
 }
 
-impl<'a> BlueprintZones<'a> {
-    pub fn new(parent_blueprint: &'a Blueprint) -> BlueprintZones {
-        BlueprintZones {
+impl<'a> BlueprintZonesBuilder<'a> {
+    pub fn new(parent_blueprint: &'a Blueprint) -> BlueprintZonesBuilder {
+        BlueprintZonesBuilder {
             changed_zones: BTreeMap::new(),
-            parent_zones: &parent_blueprint.omicron_zones,
+            parent_zones: &parent_blueprint.blueprint_zones,
         }
     }
 
@@ -813,19 +820,18 @@ impl<'a> BlueprintZones<'a> {
     pub fn change_sled_zones(
         &mut self,
         sled_id: Uuid,
-    ) -> &mut OmicronZonesConfig {
+    ) -> &mut BlueprintZonesConfig {
         self.changed_zones.entry(sled_id).or_insert_with(|| {
             if let Some(old_sled_zones) = self.parent_zones.get(&sled_id) {
-                OmicronZonesConfig {
+                BlueprintZonesConfig {
                     generation: old_sled_zones.generation.next(),
                     zones: old_sled_zones.zones.clone(),
                 }
             } else {
-                // The first generation is reserved to mean the one
-                // containing no zones.  See
-                // OmicronZonesConfig::INITIAL_GENERATION.  So we start with the
-                // next one.
-                OmicronZonesConfig {
+                // The first generation is reserved to mean the one containing
+                // no zones.  See OmicronZonesConfig::INITIAL_GENERATION.  So
+                // we start with the next one.
+                BlueprintZonesConfig {
                     generation: Generation::new().next(),
                     zones: vec![],
                 }
@@ -838,7 +844,7 @@ impl<'a> BlueprintZones<'a> {
     pub fn current_sled_zones(
         &self,
         sled_id: Uuid,
-    ) -> Box<dyn Iterator<Item = &OmicronZoneConfig> + '_> {
+    ) -> Box<dyn Iterator<Item = &BlueprintZoneConfig> + '_> {
         if let Some(sled_zones) = self
             .changed_zones
             .get(&sled_id)
@@ -851,10 +857,10 @@ impl<'a> BlueprintZones<'a> {
     }
 
     /// Produces an owned map of zones for the requested sleds
-    pub fn into_omicron_zones(
+    pub fn into_zones_map(
         mut self,
         sled_ids: impl Iterator<Item = Uuid>,
-    ) -> BTreeMap<Uuid, OmicronZonesConfig> {
+    ) -> BTreeMap<Uuid, BlueprintZonesConfig> {
         sled_ids
             .map(|sled_id| {
                 // Start with self.changed_zones, which contains entries for any
@@ -868,14 +874,12 @@ impl<'a> BlueprintZones<'a> {
                     // If it's not there either, then this must be a new sled
                     // and we haven't added any zones to it yet.  Use the
                     // standard initial config.
-                    .unwrap_or_else(|| OmicronZonesConfig {
+                    .unwrap_or_else(|| BlueprintZonesConfig {
                         generation: Generation::new(),
                         zones: vec![],
                     });
 
-                // This is not strictly necessary.  But for testing, it's
-                // helpful for things to be in sorted order.
-                zones.zones.sort_by_key(|zone| zone.id);
+                zones.sort();
 
                 (sled_id, zones)
             })
@@ -892,6 +896,7 @@ pub mod test {
     use omicron_common::address::IpRange;
     use omicron_test_utils::dev::test_setup_log;
     use sled_agent_client::types::{OmicronZoneConfig, OmicronZoneType};
+    use std::collections::BTreeSet;
 
     pub const DEFAULT_N_SLEDS: usize = 3;
 
@@ -899,17 +904,15 @@ pub mod test {
     pub fn verify_blueprint(blueprint: &Blueprint) {
         let mut underlay_ips: BTreeMap<Ipv6Addr, &OmicronZoneConfig> =
             BTreeMap::new();
-        for sled_zones in blueprint.omicron_zones.values() {
-            for zone in &sled_zones.zones {
-                if let Some(previous) =
-                    underlay_ips.insert(zone.underlay_address, zone)
-                {
-                    panic!(
-                        "found duplicate underlay IP {} in zones {} and \
+        for (_, zone) in blueprint.all_omicron_zones() {
+            if let Some(previous) =
+                underlay_ips.insert(zone.underlay_address, zone)
+            {
+                panic!(
+                    "found duplicate underlay IP {} in zones {} and \
                         {}\n\nblueprint: {:#?}",
-                        zone.underlay_address, zone.id, previous.id, blueprint
-                    );
-                }
+                    zone.underlay_address, zone.id, previous.id, blueprint
+                );
             }
         }
     }
@@ -934,11 +937,7 @@ pub mod test {
             .expect("failed to create initial blueprint");
         verify_blueprint(&blueprint_initial);
 
-        // Since collections don't include what was in service, we have to
-        // provide that ourselves.  For our purposes though we don't care.
-        let zones_in_service = blueprint_initial.zones_in_service.clone();
-        let diff = blueprint_initial
-            .diff_sleds_from_collection(&collection, &zones_in_service);
+        let diff = blueprint_initial.diff_sleds_from_collection(&collection);
         println!(
             "collection -> initial blueprint (expected no changes):\n{}",
             diff.display()
@@ -1057,13 +1056,15 @@ pub mod test {
             assert!(new_sled_resources
                 .subnet
                 .net()
-                .contains(z.underlay_address));
+                .contains(z.config.underlay_address));
         }
 
         // Check for an NTP zone.  Its sockaddr's IP should also be on the
         // sled's subnet.
         assert!(new_sled_zones.zones.iter().any(|z| {
-            if let OmicronZoneType::InternalNtp { address, .. } = &z.zone_type {
+            if let OmicronZoneType::InternalNtp { address, .. } =
+                &z.config.zone_type
+            {
                 let sockaddr = address.parse::<SocketAddrV6>().unwrap();
                 assert!(new_sled_resources
                     .subnet
@@ -1079,7 +1080,7 @@ pub mod test {
             .iter()
             .filter_map(|z| {
                 if let OmicronZoneType::Crucible { address, dataset } =
-                    &z.zone_type
+                    &z.config.zone_type
                 {
                     let sockaddr = address.parse::<SocketAddrV6>().unwrap();
                     let ip = sockaddr.ip();
diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs
index c8c1bb380b..23df35e9ae 100644
--- a/nexus/reconfigurator/planning/src/example.rs
+++ b/nexus/reconfigurator/planning/src/example.rs
@@ -9,6 +9,7 @@ use crate::blueprint_builder::UuidRng;
 use crate::system::SledBuilder;
 use crate::system::SystemDescription;
 use nexus_types::deployment::Blueprint;
+use nexus_types::deployment::BlueprintZoneFilter;
 use nexus_types::deployment::Policy;
 use nexus_types::inventory::Collection;
 use omicron_common::api::external::Generation;
@@ -108,14 +109,16 @@ impl ExampleSystem {
             system.to_collection_builder().expect("failed to build collection");
 
         for sled_id in blueprint.sleds() {
-            let Some(zones) = blueprint.omicron_zones.get(&sled_id) else {
+            let Some(zones) = blueprint.blueprint_zones.get(&sled_id) else {
                 continue;
             };
             builder
                 .found_sled_omicron_zones(
                     "fake sled agent",
                     sled_id,
-                    zones.clone(),
+                    zones.to_omicron_zones_config(
+                        BlueprintZoneFilter::SledAgentPut,
+                    ),
                 )
                 .unwrap();
         }
diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs
index 819e56b5d0..60eef225d3 100644
--- a/nexus/reconfigurator/planning/src/planner.rs
+++ b/nexus/reconfigurator/planning/src/planner.rs
@@ -340,6 +340,7 @@ mod test {
     use crate::system::SledBuilder;
     use expectorate::assert_contents;
     use nexus_inventory::now_db_precision;
+    use nexus_types::deployment::BlueprintZoneFilter;
     use nexus_types::external_api::views::SledPolicy;
     use nexus_types::external_api::views::SledProvisionPolicy;
     use nexus_types::external_api::views::SledState;
@@ -439,7 +440,7 @@ mod test {
         assert_eq!(sled_id, new_sled_id);
         assert_eq!(sled_zones.zones.len(), 1);
         assert!(matches!(
-            sled_zones.zones[0].zone_type,
+            sled_zones.zones[0].config.zone_type,
             OmicronZoneType::InternalNtp { .. }
         ));
         assert_eq!(diff.sleds_removed().count(), 0);
@@ -480,10 +481,12 @@ mod test {
                     source: String::from("test suite"),
                     sled_id: new_sled_id,
                     zones: blueprint4
-                        .omicron_zones
+                        .blueprint_zones
                         .get(&new_sled_id)
-                        .cloned()
-                        .expect("blueprint should contain zones for new sled"),
+                        .expect("blueprint should contain zones for new sled")
+                        .to_omicron_zones_config(
+                            BlueprintZoneFilter::SledAgentPut
+                        )
                 }
             )
             .is_none());
@@ -524,9 +527,9 @@ mod test {
         let zones = sled_changes.zones_added().collect::<Vec<_>>();
         assert_eq!(zones.len(), 10);
         for zone in &zones {
-            let OmicronZoneType::Crucible { .. } = zone.zone_type else {
+            if !zone.config.zone_type.is_crucible() {
                 panic!("unexpectedly added a non-Crucible zone: {zone:?}");
-            };
+            }
         }
         verify_blueprint(&blueprint5);
 
@@ -599,15 +602,15 @@ mod test {
 
         // This blueprint should only have 1 Nexus instance on the one sled we
         // kept.
-        assert_eq!(blueprint1.omicron_zones.len(), 1);
+        assert_eq!(blueprint1.blueprint_zones.len(), 1);
         assert_eq!(
             blueprint1
-                .omicron_zones
+                .blueprint_zones
                 .get(&sled_id)
                 .expect("missing kept sled")
                 .zones
                 .iter()
-                .filter(|z| z.zone_type.is_nexus())
+                .filter(|z| z.config.zone_type.is_nexus())
                 .count(),
             1
         );
@@ -642,9 +645,9 @@ mod test {
         let zones = sled_changes.zones_added().collect::<Vec<_>>();
         assert_eq!(zones.len(), policy.target_nexus_zone_count - 1);
         for zone in &zones {
-            let OmicronZoneType::Nexus { .. } = zone.zone_type else {
+            if !zone.config.zone_type.is_nexus() {
                 panic!("unexpectedly added a non-Nexus zone: {zone:?}");
-            };
+            }
         }
 
         logctx.cleanup_successful();
@@ -675,13 +678,13 @@ mod test {
             .expect("failed to create initial blueprint");
 
         // This blueprint should only have 3 Nexus zones: one on each sled.
-        assert_eq!(blueprint1.omicron_zones.len(), 3);
-        for sled_config in blueprint1.omicron_zones.values() {
+        assert_eq!(blueprint1.blueprint_zones.len(), 3);
+        for sled_config in blueprint1.blueprint_zones.values() {
             assert_eq!(
                 sled_config
                     .zones
                     .iter()
-                    .filter(|z| z.zone_type.is_nexus())
+                    .filter(|z| z.config.zone_type.is_nexus())
                     .count(),
                 1
             );
@@ -727,9 +730,9 @@ mod test {
                 }
             }
             for zone in &zones {
-                let OmicronZoneType::Nexus { .. } = zone.zone_type else {
-                    panic!("unexpectedly added a non-Crucible zone: {zone:?}");
-                };
+                if !zone.config.zone_type.is_nexus() {
+                    panic!("unexpectedly added a non-Nexus zone: {zone:?}");
+                }
             }
         }
         assert_eq!(total_new_nexus_zones, 11);
@@ -766,13 +769,13 @@ mod test {
             .expect("failed to create initial blueprint");
 
         // This blueprint should only have 5 Nexus zones: one on each sled.
-        assert_eq!(blueprint1.omicron_zones.len(), 5);
-        for sled_config in blueprint1.omicron_zones.values() {
+        assert_eq!(blueprint1.blueprint_zones.len(), 5);
+        for sled_config in blueprint1.blueprint_zones.values() {
             assert_eq!(
                 sled_config
                     .zones
                     .iter()
-                    .filter(|z| z.zone_type.is_nexus())
+                    .filter(|z| z.config.zone_type.is_nexus())
                     .count(),
                 1
             );
@@ -857,7 +860,8 @@ mod test {
                 }
             }
             for zone in &zones {
-                let OmicronZoneType::Nexus { .. } = zone.zone_type else {
+                let OmicronZoneType::Nexus { .. } = zone.config.zone_type
+                else {
                     panic!("unexpectedly added a non-Crucible zone: {zone:?}");
                 };
             }
diff --git a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt
index edad186e0c..9f7cab737f 100644
--- a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt
+++ b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt
@@ -3,46 +3,46 @@ diff blueprint 979ef428-0bdd-4622-8a72-0719e942b415 blueprint 4171ad05-89dd-474b
 +++ blueprint 4171ad05-89dd-474b-846b-b007e4346366
   sled 41f45d9f-766e-4ca6-a881-61ee45c80f57
       zone config generation 2
-         zone 267ed614-92af-4b9d-bdba-c2881c2e43a2 type internal_ntp underlay IP fd00:1122:3344:103::21 (unchanged)
-         zone 322ee9f1-8903-4542-a0a8-a54cefabdeca type crucible underlay IP fd00:1122:3344:103::24 (unchanged)
-         zone 4ab1650f-32c5-447f-939d-64b8103a7645 type crucible underlay IP fd00:1122:3344:103::2a (unchanged)
-         zone 64aa65f8-1ccb-4cd6-9953-027aebdac8ff type crucible underlay IP fd00:1122:3344:103::27 (unchanged)
-         zone 6e811d86-8aa7-4660-935b-84b4b7721b10 type crucible underlay IP fd00:1122:3344:103::2b (unchanged)
-         zone 747d2426-68bf-4c22-8806-41d290b5d5f5 type crucible underlay IP fd00:1122:3344:103::25 (unchanged)
-         zone 7fbd2c38-5dc3-48c4-b061-558a2041d70f type crucible underlay IP fd00:1122:3344:103::2c (unchanged)
-         zone 8e9e923e-62b1-4cbc-9f59-d6397e338b6b type crucible underlay IP fd00:1122:3344:103::29 (unchanged)
-         zone b14d5478-1a0e-4b90-b526-36b06339dfc4 type crucible underlay IP fd00:1122:3344:103::28 (unchanged)
-         zone b40f7c7b-526c-46c8-ae33-67280c280eb7 type crucible underlay IP fd00:1122:3344:103::23 (unchanged)
-         zone be97b92b-38d6-422a-8c76-d37060f75bd2 type crucible underlay IP fd00:1122:3344:103::26 (unchanged)
-         zone cc816cfe-3869-4dde-b596-397d41198628 type nexus underlay IP fd00:1122:3344:103::22 (unchanged)
+          267ed614-92af-4b9d-bdba-c2881c2e43a2 in service internal_ntp [underlay IP fd00:1122:3344:103::21] (unchanged)
+          322ee9f1-8903-4542-a0a8-a54cefabdeca in service crucible [underlay IP fd00:1122:3344:103::24] (unchanged)
+          4ab1650f-32c5-447f-939d-64b8103a7645 in service crucible [underlay IP fd00:1122:3344:103::2a] (unchanged)
+          64aa65f8-1ccb-4cd6-9953-027aebdac8ff in service crucible [underlay IP fd00:1122:3344:103::27] (unchanged)
+          6e811d86-8aa7-4660-935b-84b4b7721b10 in service crucible [underlay IP fd00:1122:3344:103::2b] (unchanged)
+          747d2426-68bf-4c22-8806-41d290b5d5f5 in service crucible [underlay IP fd00:1122:3344:103::25] (unchanged)
+          7fbd2c38-5dc3-48c4-b061-558a2041d70f in service crucible [underlay IP fd00:1122:3344:103::2c] (unchanged)
+          8e9e923e-62b1-4cbc-9f59-d6397e338b6b in service crucible [underlay IP fd00:1122:3344:103::29] (unchanged)
+          b14d5478-1a0e-4b90-b526-36b06339dfc4 in service crucible [underlay IP fd00:1122:3344:103::28] (unchanged)
+          b40f7c7b-526c-46c8-ae33-67280c280eb7 in service crucible [underlay IP fd00:1122:3344:103::23] (unchanged)
+          be97b92b-38d6-422a-8c76-d37060f75bd2 in service crucible [underlay IP fd00:1122:3344:103::26] (unchanged)
+          cc816cfe-3869-4dde-b596-397d41198628 in service nexus [underlay IP fd00:1122:3344:103::22] (unchanged)
   sled 43677374-8d2f-4deb-8a41-eeea506db8e0
       zone config generation 2
-         zone 02acbe6a-1c88-47e3-94c3-94084cbde098 type crucible underlay IP fd00:1122:3344:101::27 (unchanged)
-         zone 07c3c805-8888-4fe5-9543-3d2479dbe6f3 type crucible underlay IP fd00:1122:3344:101::26 (unchanged)
-         zone 08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb type internal_ntp underlay IP fd00:1122:3344:101::21 (unchanged)
-         zone 10d98a73-ec88-4aff-a7e8-7db6a87880e6 type crucible underlay IP fd00:1122:3344:101::24 (unchanged)
-         zone 2a455c35-eb3c-4c73-ab6c-d0a706e25316 type crucible underlay IP fd00:1122:3344:101::29 (unchanged)
-         zone 3eda924f-22a9-4f3e-9a1b-91d1c47601ab type crucible underlay IP fd00:1122:3344:101::23 (unchanged)
-         zone 587be699-a320-4c79-b320-128d9ecddc0b type crucible underlay IP fd00:1122:3344:101::2b (unchanged)
-         zone 6fa06115-4959-4913-8e7b-dd70d7651f07 type crucible underlay IP fd00:1122:3344:101::2c (unchanged)
-         zone 8f3a1cc5-9195-4a30-ad02-b804278fe639 type crucible underlay IP fd00:1122:3344:101::28 (unchanged)
-         zone a1696cd4-588c-484a-b95b-66e824c0ce05 type crucible underlay IP fd00:1122:3344:101::25 (unchanged)
-         zone a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 type crucible underlay IP fd00:1122:3344:101::2a (unchanged)
-         zone c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 type nexus underlay IP fd00:1122:3344:101::22 (unchanged)
+          02acbe6a-1c88-47e3-94c3-94084cbde098 in service crucible [underlay IP fd00:1122:3344:101::27] (unchanged)
+          07c3c805-8888-4fe5-9543-3d2479dbe6f3 in service crucible [underlay IP fd00:1122:3344:101::26] (unchanged)
+          08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb in service internal_ntp [underlay IP fd00:1122:3344:101::21] (unchanged)
+          10d98a73-ec88-4aff-a7e8-7db6a87880e6 in service crucible [underlay IP fd00:1122:3344:101::24] (unchanged)
+          2a455c35-eb3c-4c73-ab6c-d0a706e25316 in service crucible [underlay IP fd00:1122:3344:101::29] (unchanged)
+          3eda924f-22a9-4f3e-9a1b-91d1c47601ab in service crucible [underlay IP fd00:1122:3344:101::23] (unchanged)
+          587be699-a320-4c79-b320-128d9ecddc0b in service crucible [underlay IP fd00:1122:3344:101::2b] (unchanged)
+          6fa06115-4959-4913-8e7b-dd70d7651f07 in service crucible [underlay IP fd00:1122:3344:101::2c] (unchanged)
+          8f3a1cc5-9195-4a30-ad02-b804278fe639 in service crucible [underlay IP fd00:1122:3344:101::28] (unchanged)
+          a1696cd4-588c-484a-b95b-66e824c0ce05 in service crucible [underlay IP fd00:1122:3344:101::25] (unchanged)
+          a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 in service crucible [underlay IP fd00:1122:3344:101::2a] (unchanged)
+          c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 in service nexus [underlay IP fd00:1122:3344:101::22] (unchanged)
   sled 590e3034-d946-4166-b0e5-2d0034197a07
       zone config generation 2
-         zone 18f8fe40-646e-4962-b17a-20e201f3a6e5 type crucible underlay IP fd00:1122:3344:102::2a (unchanged)
-         zone 47199d48-534c-4267-a654-d2d90e64b498 type internal_ntp underlay IP fd00:1122:3344:102::21 (unchanged)
-         zone 56d5d7cf-db2c-40a3-a775-003241ad4820 type crucible underlay IP fd00:1122:3344:102::29 (unchanged)
-         zone 6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 type crucible underlay IP fd00:1122:3344:102::2b (unchanged)
-         zone 704e1fed-f8d6-4cfa-a470-bad27fdc06d1 type nexus underlay IP fd00:1122:3344:102::22 (unchanged)
-         zone 7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 type crucible underlay IP fd00:1122:3344:102::26 (unchanged)
-         zone 93f2f40c-5616-4d8d-8519-ec6debdcede0 type crucible underlay IP fd00:1122:3344:102::2c (unchanged)
-         zone ab7ba6df-d401-40bd-940e-faf57c57aa2a type crucible underlay IP fd00:1122:3344:102::28 (unchanged)
-         zone af322036-371f-437c-8c08-7f40f3f1403b type crucible underlay IP fd00:1122:3344:102::23 (unchanged)
-         zone d637264f-6f40-44c2-8b7e-a179430210d2 type crucible underlay IP fd00:1122:3344:102::25 (unchanged)
-         zone dce226c9-7373-4bfa-8a94-79dc472857a6 type crucible underlay IP fd00:1122:3344:102::27 (unchanged)
-         zone edabedf3-839c-488d-ad6f-508ffa864674 type crucible underlay IP fd00:1122:3344:102::24 (unchanged)
+          18f8fe40-646e-4962-b17a-20e201f3a6e5 in service crucible [underlay IP fd00:1122:3344:102::2a] (unchanged)
+          47199d48-534c-4267-a654-d2d90e64b498 in service internal_ntp [underlay IP fd00:1122:3344:102::21] (unchanged)
+          56d5d7cf-db2c-40a3-a775-003241ad4820 in service crucible [underlay IP fd00:1122:3344:102::29] (unchanged)
+          6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 in service crucible [underlay IP fd00:1122:3344:102::2b] (unchanged)
+          704e1fed-f8d6-4cfa-a470-bad27fdc06d1 in service nexus [underlay IP fd00:1122:3344:102::22] (unchanged)
+          7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 in service crucible [underlay IP fd00:1122:3344:102::26] (unchanged)
+          93f2f40c-5616-4d8d-8519-ec6debdcede0 in service crucible [underlay IP fd00:1122:3344:102::2c] (unchanged)
+          ab7ba6df-d401-40bd-940e-faf57c57aa2a in service crucible [underlay IP fd00:1122:3344:102::28] (unchanged)
+          af322036-371f-437c-8c08-7f40f3f1403b in service crucible [underlay IP fd00:1122:3344:102::23] (unchanged)
+          d637264f-6f40-44c2-8b7e-a179430210d2 in service crucible [underlay IP fd00:1122:3344:102::25] (unchanged)
+          dce226c9-7373-4bfa-8a94-79dc472857a6 in service crucible [underlay IP fd00:1122:3344:102::27] (unchanged)
+          edabedf3-839c-488d-ad6f-508ffa864674 in service crucible [underlay IP fd00:1122:3344:102::24] (unchanged)
 + sled b59ec570-2abb-4017-80ce-129d94e7a025 (added)
 +     zone config generation 2
-+         zone 2d73d30e-ca47-46a8-9c12-917d4ab824b6 type internal_ntp underlay IP fd00:1122:3344:104::21 (added)
++         2d73d30e-ca47-46a8-9c12-917d4ab824b6 in service internal_ntp [underlay IP fd00:1122:3344:104::21] (added)
diff --git a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt
index 6ff8080165..9d98daac36 100644
--- a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt
+++ b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt
@@ -3,57 +3,57 @@ diff blueprint 4171ad05-89dd-474b-846b-b007e4346366 blueprint f432fcd5-1284-4058
 +++ blueprint f432fcd5-1284-4058-8b4a-9286a3de6163
   sled 41f45d9f-766e-4ca6-a881-61ee45c80f57
       zone config generation 2
-         zone 267ed614-92af-4b9d-bdba-c2881c2e43a2 type internal_ntp underlay IP fd00:1122:3344:103::21 (unchanged)
-         zone 322ee9f1-8903-4542-a0a8-a54cefabdeca type crucible underlay IP fd00:1122:3344:103::24 (unchanged)
-         zone 4ab1650f-32c5-447f-939d-64b8103a7645 type crucible underlay IP fd00:1122:3344:103::2a (unchanged)
-         zone 64aa65f8-1ccb-4cd6-9953-027aebdac8ff type crucible underlay IP fd00:1122:3344:103::27 (unchanged)
-         zone 6e811d86-8aa7-4660-935b-84b4b7721b10 type crucible underlay IP fd00:1122:3344:103::2b (unchanged)
-         zone 747d2426-68bf-4c22-8806-41d290b5d5f5 type crucible underlay IP fd00:1122:3344:103::25 (unchanged)
-         zone 7fbd2c38-5dc3-48c4-b061-558a2041d70f type crucible underlay IP fd00:1122:3344:103::2c (unchanged)
-         zone 8e9e923e-62b1-4cbc-9f59-d6397e338b6b type crucible underlay IP fd00:1122:3344:103::29 (unchanged)
-         zone b14d5478-1a0e-4b90-b526-36b06339dfc4 type crucible underlay IP fd00:1122:3344:103::28 (unchanged)
-         zone b40f7c7b-526c-46c8-ae33-67280c280eb7 type crucible underlay IP fd00:1122:3344:103::23 (unchanged)
-         zone be97b92b-38d6-422a-8c76-d37060f75bd2 type crucible underlay IP fd00:1122:3344:103::26 (unchanged)
-         zone cc816cfe-3869-4dde-b596-397d41198628 type nexus underlay IP fd00:1122:3344:103::22 (unchanged)
+          267ed614-92af-4b9d-bdba-c2881c2e43a2 in service internal_ntp [underlay IP fd00:1122:3344:103::21] (unchanged)
+          322ee9f1-8903-4542-a0a8-a54cefabdeca in service crucible [underlay IP fd00:1122:3344:103::24] (unchanged)
+          4ab1650f-32c5-447f-939d-64b8103a7645 in service crucible [underlay IP fd00:1122:3344:103::2a] (unchanged)
+          64aa65f8-1ccb-4cd6-9953-027aebdac8ff in service crucible [underlay IP fd00:1122:3344:103::27] (unchanged)
+          6e811d86-8aa7-4660-935b-84b4b7721b10 in service crucible [underlay IP fd00:1122:3344:103::2b] (unchanged)
+          747d2426-68bf-4c22-8806-41d290b5d5f5 in service crucible [underlay IP fd00:1122:3344:103::25] (unchanged)
+          7fbd2c38-5dc3-48c4-b061-558a2041d70f in service crucible [underlay IP fd00:1122:3344:103::2c] (unchanged)
+          8e9e923e-62b1-4cbc-9f59-d6397e338b6b in service crucible [underlay IP fd00:1122:3344:103::29] (unchanged)
+          b14d5478-1a0e-4b90-b526-36b06339dfc4 in service crucible [underlay IP fd00:1122:3344:103::28] (unchanged)
+          b40f7c7b-526c-46c8-ae33-67280c280eb7 in service crucible [underlay IP fd00:1122:3344:103::23] (unchanged)
+          be97b92b-38d6-422a-8c76-d37060f75bd2 in service crucible [underlay IP fd00:1122:3344:103::26] (unchanged)
+          cc816cfe-3869-4dde-b596-397d41198628 in service nexus [underlay IP fd00:1122:3344:103::22] (unchanged)
   sled 43677374-8d2f-4deb-8a41-eeea506db8e0
       zone config generation 2
-         zone 02acbe6a-1c88-47e3-94c3-94084cbde098 type crucible underlay IP fd00:1122:3344:101::27 (unchanged)
-         zone 07c3c805-8888-4fe5-9543-3d2479dbe6f3 type crucible underlay IP fd00:1122:3344:101::26 (unchanged)
-         zone 08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb type internal_ntp underlay IP fd00:1122:3344:101::21 (unchanged)
-         zone 10d98a73-ec88-4aff-a7e8-7db6a87880e6 type crucible underlay IP fd00:1122:3344:101::24 (unchanged)
-         zone 2a455c35-eb3c-4c73-ab6c-d0a706e25316 type crucible underlay IP fd00:1122:3344:101::29 (unchanged)
-         zone 3eda924f-22a9-4f3e-9a1b-91d1c47601ab type crucible underlay IP fd00:1122:3344:101::23 (unchanged)
-         zone 587be699-a320-4c79-b320-128d9ecddc0b type crucible underlay IP fd00:1122:3344:101::2b (unchanged)
-         zone 6fa06115-4959-4913-8e7b-dd70d7651f07 type crucible underlay IP fd00:1122:3344:101::2c (unchanged)
-         zone 8f3a1cc5-9195-4a30-ad02-b804278fe639 type crucible underlay IP fd00:1122:3344:101::28 (unchanged)
-         zone a1696cd4-588c-484a-b95b-66e824c0ce05 type crucible underlay IP fd00:1122:3344:101::25 (unchanged)
-         zone a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 type crucible underlay IP fd00:1122:3344:101::2a (unchanged)
-         zone c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 type nexus underlay IP fd00:1122:3344:101::22 (unchanged)
+          02acbe6a-1c88-47e3-94c3-94084cbde098 in service crucible [underlay IP fd00:1122:3344:101::27] (unchanged)
+          07c3c805-8888-4fe5-9543-3d2479dbe6f3 in service crucible [underlay IP fd00:1122:3344:101::26] (unchanged)
+          08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb in service internal_ntp [underlay IP fd00:1122:3344:101::21] (unchanged)
+          10d98a73-ec88-4aff-a7e8-7db6a87880e6 in service crucible [underlay IP fd00:1122:3344:101::24] (unchanged)
+          2a455c35-eb3c-4c73-ab6c-d0a706e25316 in service crucible [underlay IP fd00:1122:3344:101::29] (unchanged)
+          3eda924f-22a9-4f3e-9a1b-91d1c47601ab in service crucible [underlay IP fd00:1122:3344:101::23] (unchanged)
+          587be699-a320-4c79-b320-128d9ecddc0b in service crucible [underlay IP fd00:1122:3344:101::2b] (unchanged)
+          6fa06115-4959-4913-8e7b-dd70d7651f07 in service crucible [underlay IP fd00:1122:3344:101::2c] (unchanged)
+          8f3a1cc5-9195-4a30-ad02-b804278fe639 in service crucible [underlay IP fd00:1122:3344:101::28] (unchanged)
+          a1696cd4-588c-484a-b95b-66e824c0ce05 in service crucible [underlay IP fd00:1122:3344:101::25] (unchanged)
+          a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 in service crucible [underlay IP fd00:1122:3344:101::2a] (unchanged)
+          c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 in service nexus [underlay IP fd00:1122:3344:101::22] (unchanged)
   sled 590e3034-d946-4166-b0e5-2d0034197a07
       zone config generation 2
-         zone 18f8fe40-646e-4962-b17a-20e201f3a6e5 type crucible underlay IP fd00:1122:3344:102::2a (unchanged)
-         zone 47199d48-534c-4267-a654-d2d90e64b498 type internal_ntp underlay IP fd00:1122:3344:102::21 (unchanged)
-         zone 56d5d7cf-db2c-40a3-a775-003241ad4820 type crucible underlay IP fd00:1122:3344:102::29 (unchanged)
-         zone 6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 type crucible underlay IP fd00:1122:3344:102::2b (unchanged)
-         zone 704e1fed-f8d6-4cfa-a470-bad27fdc06d1 type nexus underlay IP fd00:1122:3344:102::22 (unchanged)
-         zone 7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 type crucible underlay IP fd00:1122:3344:102::26 (unchanged)
-         zone 93f2f40c-5616-4d8d-8519-ec6debdcede0 type crucible underlay IP fd00:1122:3344:102::2c (unchanged)
-         zone ab7ba6df-d401-40bd-940e-faf57c57aa2a type crucible underlay IP fd00:1122:3344:102::28 (unchanged)
-         zone af322036-371f-437c-8c08-7f40f3f1403b type crucible underlay IP fd00:1122:3344:102::23 (unchanged)
-         zone d637264f-6f40-44c2-8b7e-a179430210d2 type crucible underlay IP fd00:1122:3344:102::25 (unchanged)
-         zone dce226c9-7373-4bfa-8a94-79dc472857a6 type crucible underlay IP fd00:1122:3344:102::27 (unchanged)
-         zone edabedf3-839c-488d-ad6f-508ffa864674 type crucible underlay IP fd00:1122:3344:102::24 (unchanged)
+          18f8fe40-646e-4962-b17a-20e201f3a6e5 in service crucible [underlay IP fd00:1122:3344:102::2a] (unchanged)
+          47199d48-534c-4267-a654-d2d90e64b498 in service internal_ntp [underlay IP fd00:1122:3344:102::21] (unchanged)
+          56d5d7cf-db2c-40a3-a775-003241ad4820 in service crucible [underlay IP fd00:1122:3344:102::29] (unchanged)
+          6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 in service crucible [underlay IP fd00:1122:3344:102::2b] (unchanged)
+          704e1fed-f8d6-4cfa-a470-bad27fdc06d1 in service nexus [underlay IP fd00:1122:3344:102::22] (unchanged)
+          7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 in service crucible [underlay IP fd00:1122:3344:102::26] (unchanged)
+          93f2f40c-5616-4d8d-8519-ec6debdcede0 in service crucible [underlay IP fd00:1122:3344:102::2c] (unchanged)
+          ab7ba6df-d401-40bd-940e-faf57c57aa2a in service crucible [underlay IP fd00:1122:3344:102::28] (unchanged)
+          af322036-371f-437c-8c08-7f40f3f1403b in service crucible [underlay IP fd00:1122:3344:102::23] (unchanged)
+          d637264f-6f40-44c2-8b7e-a179430210d2 in service crucible [underlay IP fd00:1122:3344:102::25] (unchanged)
+          dce226c9-7373-4bfa-8a94-79dc472857a6 in service crucible [underlay IP fd00:1122:3344:102::27] (unchanged)
+          edabedf3-839c-488d-ad6f-508ffa864674 in service crucible [underlay IP fd00:1122:3344:102::24] (unchanged)
   sled b59ec570-2abb-4017-80ce-129d94e7a025
 -     zone config generation 2
 +     zone config generation 3
-         zone 2d73d30e-ca47-46a8-9c12-917d4ab824b6 type internal_ntp underlay IP fd00:1122:3344:104::21 (unchanged)
-+        zone 1a20ee3c-f66e-4fca-ab85-2a248aa3d79d type crucible underlay IP fd00:1122:3344:104::2b (added)
-+        zone 28852beb-d0e5-4cba-9adb-e7f0cd4bb864 type crucible underlay IP fd00:1122:3344:104::29 (added)
-+        zone 45556184-7092-4a3d-873f-637976bb133b type crucible underlay IP fd00:1122:3344:104::22 (added)
-+        zone 8215bf7a-10d6-4f40-aeb7-27a196307c37 type crucible underlay IP fd00:1122:3344:104::25 (added)
-+        zone 9d75abfe-47ab-434a-93dd-af50dc0dddde type crucible underlay IP fd00:1122:3344:104::23 (added)
-+        zone a36d291c-7f68-462f-830e-bc29e5841ce2 type crucible underlay IP fd00:1122:3344:104::27 (added)
-+        zone b3a4d434-aaee-4752-8c99-69d88fbcb8c5 type crucible underlay IP fd00:1122:3344:104::2a (added)
-+        zone cf5b636b-a505-4db6-bc32-baf9f53f4371 type crucible underlay IP fd00:1122:3344:104::28 (added)
-+        zone f6125d45-b9cc-4721-ba60-ed4dbb177e41 type crucible underlay IP fd00:1122:3344:104::26 (added)
-+        zone f86e19d2-9145-41cf-be89-6aaa34a73873 type crucible underlay IP fd00:1122:3344:104::24 (added)
+          2d73d30e-ca47-46a8-9c12-917d4ab824b6 in service internal_ntp [underlay IP fd00:1122:3344:104::21] (unchanged)
++         1a20ee3c-f66e-4fca-ab85-2a248aa3d79d in service crucible [underlay IP fd00:1122:3344:104::2b] (added)
++         28852beb-d0e5-4cba-9adb-e7f0cd4bb864 in service crucible [underlay IP fd00:1122:3344:104::29] (added)
++         45556184-7092-4a3d-873f-637976bb133b in service crucible [underlay IP fd00:1122:3344:104::22] (added)
++         8215bf7a-10d6-4f40-aeb7-27a196307c37 in service crucible [underlay IP fd00:1122:3344:104::25] (added)
++         9d75abfe-47ab-434a-93dd-af50dc0dddde in service crucible [underlay IP fd00:1122:3344:104::23] (added)
++         a36d291c-7f68-462f-830e-bc29e5841ce2 in service crucible [underlay IP fd00:1122:3344:104::27] (added)
++         b3a4d434-aaee-4752-8c99-69d88fbcb8c5 in service crucible [underlay IP fd00:1122:3344:104::2a] (added)
++         cf5b636b-a505-4db6-bc32-baf9f53f4371 in service crucible [underlay IP fd00:1122:3344:104::28] (added)
++         f6125d45-b9cc-4721-ba60-ed4dbb177e41 in service crucible [underlay IP fd00:1122:3344:104::26] (added)
++         f86e19d2-9145-41cf-be89-6aaa34a73873 in service crucible [underlay IP fd00:1122:3344:104::24] (added)
diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt
index b63a92ad94..17d3db6228 100644
--- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt
+++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt
@@ -3,84 +3,84 @@ diff blueprint 55502b1b-e255-438b-a16a-2680a4b5f962 blueprint 9f71f5d3-a272-4382
 +++ blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6
   sled 2d1cb4f2-cf44-40fc-b118-85036eb732a9
       zone config generation 2
-         zone 19fbc4f8-a683-4f22-8f5a-e74782b935be type crucible underlay IP fd00:1122:3344:105::26 (unchanged)
-         zone 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 type crucible underlay IP fd00:1122:3344:105::2c (unchanged)
-         zone 6b53ab2e-d98c-485f-87a3-4d5df595390f type crucible underlay IP fd00:1122:3344:105::27 (unchanged)
-         zone 6dff7633-66bb-4924-a6ff-2c896e66964b type nexus underlay IP fd00:1122:3344:105::22 (unchanged)
-         zone 7f4e9f9f-08f8-4d14-885d-e977c05525ad type internal_ntp underlay IP fd00:1122:3344:105::21 (unchanged)
-         zone 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 type crucible underlay IP fd00:1122:3344:105::23 (unchanged)
-         zone 9f0abbad-dbd3-4d43-9675-78092217ffd9 type crucible underlay IP fd00:1122:3344:105::25 (unchanged)
-         zone b0c63f48-01ea-4aae-bb26-fb0dd59d1662 type crucible underlay IP fd00:1122:3344:105::28 (unchanged)
-         zone c406da50-34b9-4bb4-a460-8f49875d2a6a type crucible underlay IP fd00:1122:3344:105::24 (unchanged)
-         zone d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 type crucible underlay IP fd00:1122:3344:105::2a (unchanged)
-         zone e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 type crucible underlay IP fd00:1122:3344:105::2b (unchanged)
-         zone f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 type crucible underlay IP fd00:1122:3344:105::29 (unchanged)
+          19fbc4f8-a683-4f22-8f5a-e74782b935be in service crucible [underlay IP fd00:1122:3344:105::26] (unchanged)
+          4f1ce8a2-d3a5-4a38-be4c-9817de52db37 in service crucible [underlay IP fd00:1122:3344:105::2c] (unchanged)
+          6b53ab2e-d98c-485f-87a3-4d5df595390f in service crucible [underlay IP fd00:1122:3344:105::27] (unchanged)
+          6dff7633-66bb-4924-a6ff-2c896e66964b in service nexus [underlay IP fd00:1122:3344:105::22] (unchanged)
+          7f4e9f9f-08f8-4d14-885d-e977c05525ad in service internal_ntp [underlay IP fd00:1122:3344:105::21] (unchanged)
+          93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 in service crucible [underlay IP fd00:1122:3344:105::23] (unchanged)
+          9f0abbad-dbd3-4d43-9675-78092217ffd9 in service crucible [underlay IP fd00:1122:3344:105::25] (unchanged)
+          b0c63f48-01ea-4aae-bb26-fb0dd59d1662 in service crucible [underlay IP fd00:1122:3344:105::28] (unchanged)
+          c406da50-34b9-4bb4-a460-8f49875d2a6a in service crucible [underlay IP fd00:1122:3344:105::24] (unchanged)
+          d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 in service crucible [underlay IP fd00:1122:3344:105::2a] (unchanged)
+          e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 in service crucible [underlay IP fd00:1122:3344:105::2b] (unchanged)
+          f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 in service crucible [underlay IP fd00:1122:3344:105::29] (unchanged)
   sled 48d95fef-bc9f-4f50-9a53-1e075836291d
       zone config generation 2
-         zone 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 type crucible underlay IP fd00:1122:3344:103::2c (unchanged)
-         zone 0dcfdfc5-481e-4153-b97c-11cf02b648ea type crucible underlay IP fd00:1122:3344:103::25 (unchanged)
-         zone 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb type nexus underlay IP fd00:1122:3344:103::22 (unchanged)
-         zone 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f type crucible underlay IP fd00:1122:3344:103::27 (unchanged)
-         zone 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 type crucible underlay IP fd00:1122:3344:103::28 (unchanged)
-         zone 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb type crucible underlay IP fd00:1122:3344:103::24 (unchanged)
-         zone 67622d61-2df4-414d-aa0e-d1277265f405 type crucible underlay IP fd00:1122:3344:103::23 (unchanged)
-         zone 67d913e0-0005-4599-9b28-0abbf6cc2916 type internal_ntp underlay IP fd00:1122:3344:103::21 (unchanged)
-         zone b91b271d-8d80-4f49-99a0-34006ae86063 type crucible underlay IP fd00:1122:3344:103::2a (unchanged)
-         zone d6ee1338-3127-43ec-9aaa-b973ccf05496 type crucible underlay IP fd00:1122:3344:103::26 (unchanged)
-         zone e39d7c9e-182b-48af-af87-58079d723583 type crucible underlay IP fd00:1122:3344:103::29 (unchanged)
-         zone f69f92a1-5007-4bb0-a85b-604dc217154b type crucible underlay IP fd00:1122:3344:103::2b (unchanged)
+          094f27af-1acb-4d1e-ba97-1fc1377d4bf2 in service crucible [underlay IP fd00:1122:3344:103::2c] (unchanged)
+          0dcfdfc5-481e-4153-b97c-11cf02b648ea in service crucible [underlay IP fd00:1122:3344:103::25] (unchanged)
+          2aa0ea4f-3561-4989-a98c-9ab7d9a240fb in service nexus [underlay IP fd00:1122:3344:103::22] (unchanged)
+          2f5e8010-a94d-43a4-9c5c-3f52832f5f7f in service crucible [underlay IP fd00:1122:3344:103::27] (unchanged)
+          4a9a0a9d-87f0-4f1d-9181-27f6b435e637 in service crucible [underlay IP fd00:1122:3344:103::28] (unchanged)
+          56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb in service crucible [underlay IP fd00:1122:3344:103::24] (unchanged)
+          67622d61-2df4-414d-aa0e-d1277265f405 in service crucible [underlay IP fd00:1122:3344:103::23] (unchanged)
+          67d913e0-0005-4599-9b28-0abbf6cc2916 in service internal_ntp [underlay IP fd00:1122:3344:103::21] (unchanged)
+          b91b271d-8d80-4f49-99a0-34006ae86063 in service crucible [underlay IP fd00:1122:3344:103::2a] (unchanged)
+          d6ee1338-3127-43ec-9aaa-b973ccf05496 in service crucible [underlay IP fd00:1122:3344:103::26] (unchanged)
+          e39d7c9e-182b-48af-af87-58079d723583 in service crucible [underlay IP fd00:1122:3344:103::29] (unchanged)
+          f69f92a1-5007-4bb0-a85b-604dc217154b in service crucible [underlay IP fd00:1122:3344:103::2b] (unchanged)
   sled 68d24ac5-f341-49ea-a92a-0381b52ab387
       zone config generation 2
-         zone 01d58626-e1b0-480f-96be-ac784863c7dc type nexus underlay IP fd00:1122:3344:102::22 (unchanged)
-         zone 3b3c14b6-a8e2-4054-a577-8d96cb576230 type crucible underlay IP fd00:1122:3344:102::2c (unchanged)
-         zone 47a87c6e-ef45-4d52-9a3e-69cdd96737cc type crucible underlay IP fd00:1122:3344:102::23 (unchanged)
-         zone 6464d025-4652-4948-919e-740bec5699b1 type crucible underlay IP fd00:1122:3344:102::24 (unchanged)
-         zone 6939ce48-b17c-4616-b176-8a419a7697be type crucible underlay IP fd00:1122:3344:102::29 (unchanged)
-         zone 878dfddd-3113-4197-a3ea-e0d4dbe9b476 type crucible underlay IP fd00:1122:3344:102::25 (unchanged)
-         zone 8d4d2b28-82bb-4e36-80da-1408d8c35d82 type crucible underlay IP fd00:1122:3344:102::2b (unchanged)
-         zone 9fd52961-426f-4e62-a644-b70871103fca type crucible underlay IP fd00:1122:3344:102::26 (unchanged)
-         zone b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 type crucible underlay IP fd00:1122:3344:102::27 (unchanged)
-         zone b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 type crucible underlay IP fd00:1122:3344:102::28 (unchanged)
-         zone c407795c-6c8b-428e-8ab8-b962913c447f type crucible underlay IP fd00:1122:3344:102::2a (unchanged)
-         zone f3f2e4f3-0985-4ef6-8336-ce479382d05d type internal_ntp underlay IP fd00:1122:3344:102::21 (unchanged)
+          01d58626-e1b0-480f-96be-ac784863c7dc in service nexus [underlay IP fd00:1122:3344:102::22] (unchanged)
+          3b3c14b6-a8e2-4054-a577-8d96cb576230 in service crucible [underlay IP fd00:1122:3344:102::2c] (unchanged)
+          47a87c6e-ef45-4d52-9a3e-69cdd96737cc in service crucible [underlay IP fd00:1122:3344:102::23] (unchanged)
+          6464d025-4652-4948-919e-740bec5699b1 in service crucible [underlay IP fd00:1122:3344:102::24] (unchanged)
+          6939ce48-b17c-4616-b176-8a419a7697be in service crucible [underlay IP fd00:1122:3344:102::29] (unchanged)
+          878dfddd-3113-4197-a3ea-e0d4dbe9b476 in service crucible [underlay IP fd00:1122:3344:102::25] (unchanged)
+          8d4d2b28-82bb-4e36-80da-1408d8c35d82 in service crucible [underlay IP fd00:1122:3344:102::2b] (unchanged)
+          9fd52961-426f-4e62-a644-b70871103fca in service crucible [underlay IP fd00:1122:3344:102::26] (unchanged)
+          b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 in service crucible [underlay IP fd00:1122:3344:102::27] (unchanged)
+          b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 in service crucible [underlay IP fd00:1122:3344:102::28] (unchanged)
+          c407795c-6c8b-428e-8ab8-b962913c447f in service crucible [underlay IP fd00:1122:3344:102::2a] (unchanged)
+          f3f2e4f3-0985-4ef6-8336-ce479382d05d in service internal_ntp [underlay IP fd00:1122:3344:102::21] (unchanged)
   sled 75bc286f-2b4b-482c-9431-59272af529da
 -     zone config generation 2
 +     zone config generation 3
-         zone 15bb9def-69b8-4d2e-b04f-9fee1143387c type crucible underlay IP fd00:1122:3344:104::25 (unchanged)
-         zone 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c type crucible underlay IP fd00:1122:3344:104::2c (unchanged)
-         zone 57b96d5c-b71e-43e4-8869-7d514003d00d type internal_ntp underlay IP fd00:1122:3344:104::21 (unchanged)
-         zone 621509d6-3772-4009-aca1-35eefd1098fb type crucible underlay IP fd00:1122:3344:104::28 (unchanged)
-         zone 85b8c68a-160d-461d-94dd-1baf175fa75c type crucible underlay IP fd00:1122:3344:104::2a (unchanged)
-         zone 996d7570-b0df-46d5-aaa4-0c97697cf484 type crucible underlay IP fd00:1122:3344:104::26 (unchanged)
-         zone a732c489-d29a-4f75-b900-5966385943af type crucible underlay IP fd00:1122:3344:104::29 (unchanged)
-         zone b1783e95-9598-451d-b6ba-c50b52b428c3 type crucible underlay IP fd00:1122:3344:104::24 (unchanged)
-         zone b4947d31-f70e-4ee0-8817-0ca6cea9b16b type nexus underlay IP fd00:1122:3344:104::22 (unchanged)
-         zone c6dd531e-2d1d-423b-acc8-358533dab78c type crucible underlay IP fd00:1122:3344:104::27 (unchanged)
-         zone e4b3e159-3dbe-48cb-8497-e3da92a90e5a type crucible underlay IP fd00:1122:3344:104::23 (unchanged)
-         zone f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 type crucible underlay IP fd00:1122:3344:104::2b (unchanged)
-+        zone 2ec75441-3d7d-4b4b-9614-af03de5a3666 type nexus underlay IP fd00:1122:3344:104::2d (added)
-+        zone 3ca5292f-8a59-4475-bb72-0f43714d0fff type nexus underlay IP fd00:1122:3344:104::31 (added)
-+        zone 508abd03-cbfe-4654-9a6d-7f15a1ad32e5 type nexus underlay IP fd00:1122:3344:104::2e (added)
-+        zone 59950bc8-1497-44dd-8cbf-b6502ba921b2 type nexus underlay IP fd00:1122:3344:104::2f (added)
-+        zone 99f6d544-8599-4e2b-a55a-82d9e0034662 type nexus underlay IP fd00:1122:3344:104::30 (added)
-+        zone c26b3bda-5561-44a1-a69f-22103fe209a1 type nexus underlay IP fd00:1122:3344:104::32 (added)
+          15bb9def-69b8-4d2e-b04f-9fee1143387c in service crucible [underlay IP fd00:1122:3344:104::25] (unchanged)
+          23a8fa2b-ef3e-4017-a43f-f7a83953bd7c in service crucible [underlay IP fd00:1122:3344:104::2c] (unchanged)
+          57b96d5c-b71e-43e4-8869-7d514003d00d in service internal_ntp [underlay IP fd00:1122:3344:104::21] (unchanged)
+          621509d6-3772-4009-aca1-35eefd1098fb in service crucible [underlay IP fd00:1122:3344:104::28] (unchanged)
+          85b8c68a-160d-461d-94dd-1baf175fa75c in service crucible [underlay IP fd00:1122:3344:104::2a] (unchanged)
+          996d7570-b0df-46d5-aaa4-0c97697cf484 in service crucible [underlay IP fd00:1122:3344:104::26] (unchanged)
+          a732c489-d29a-4f75-b900-5966385943af in service crucible [underlay IP fd00:1122:3344:104::29] (unchanged)
+          b1783e95-9598-451d-b6ba-c50b52b428c3 in service crucible [underlay IP fd00:1122:3344:104::24] (unchanged)
+          b4947d31-f70e-4ee0-8817-0ca6cea9b16b in service nexus [underlay IP fd00:1122:3344:104::22] (unchanged)
+          c6dd531e-2d1d-423b-acc8-358533dab78c in service crucible [underlay IP fd00:1122:3344:104::27] (unchanged)
+          e4b3e159-3dbe-48cb-8497-e3da92a90e5a in service crucible [underlay IP fd00:1122:3344:104::23] (unchanged)
+          f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 in service crucible [underlay IP fd00:1122:3344:104::2b] (unchanged)
++         2ec75441-3d7d-4b4b-9614-af03de5a3666 in service nexus [underlay IP fd00:1122:3344:104::2d] (added)
++         3ca5292f-8a59-4475-bb72-0f43714d0fff in service nexus [underlay IP fd00:1122:3344:104::31] (added)
++         508abd03-cbfe-4654-9a6d-7f15a1ad32e5 in service nexus [underlay IP fd00:1122:3344:104::2e] (added)
++         59950bc8-1497-44dd-8cbf-b6502ba921b2 in service nexus [underlay IP fd00:1122:3344:104::2f] (added)
++         99f6d544-8599-4e2b-a55a-82d9e0034662 in service nexus [underlay IP fd00:1122:3344:104::30] (added)
++         c26b3bda-5561-44a1-a69f-22103fe209a1 in service nexus [underlay IP fd00:1122:3344:104::32] (added)
   sled affab35f-600a-4109-8ea0-34a067a4e0bc
 -     zone config generation 2
 +     zone config generation 3
-         zone 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 type crucible underlay IP fd00:1122:3344:101::27 (unchanged)
-         zone 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 type nexus underlay IP fd00:1122:3344:101::22 (unchanged)
-         zone 3aa07966-5899-4789-ace5-f8eeb375c6c3 type crucible underlay IP fd00:1122:3344:101::24 (unchanged)
-         zone 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf type crucible underlay IP fd00:1122:3344:101::29 (unchanged)
-         zone 72c5a909-077d-4ec1-a9d5-ae64ef9d716e type crucible underlay IP fd00:1122:3344:101::26 (unchanged)
-         zone 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 type crucible underlay IP fd00:1122:3344:101::23 (unchanged)
-         zone a1c03689-fc62-4ea5-bb72-4d01f5138614 type crucible underlay IP fd00:1122:3344:101::2a (unchanged)
-         zone a568e92e-4fbd-4b69-acd8-f16277073031 type crucible underlay IP fd00:1122:3344:101::2c (unchanged)
-         zone bf79a56a-97af-4cc4-94a5-8b20d64c2cda type crucible underlay IP fd00:1122:3344:101::28 (unchanged)
-         zone c60379ba-4e30-4628-a79a-0ae509aef4c5 type crucible underlay IP fd00:1122:3344:101::25 (unchanged)
-         zone d47f4996-fac0-4657-bcea-01b1fee6404d type crucible underlay IP fd00:1122:3344:101::2b (unchanged)
-         zone f1a7b9a7-fc6a-4b23-b829-045ff33117ff type internal_ntp underlay IP fd00:1122:3344:101::21 (unchanged)
-+        zone 6f86d5cb-17d7-424b-9d4c-39f670532cbe type nexus underlay IP fd00:1122:3344:101::2e (added)
-+        zone 87c299eb-470e-4b6d-b8c7-6759694e66b6 type nexus underlay IP fd00:1122:3344:101::30 (added)
-+        zone c72b7930-0580-4f00-93b9-8cba2c8d344e type nexus underlay IP fd00:1122:3344:101::2d (added)
-+        zone d0095508-bdb8-4faf-b091-964276a20b15 type nexus underlay IP fd00:1122:3344:101::31 (added)
-+        zone ff422442-4b31-4ade-a11a-9e5a25f0404c type nexus underlay IP fd00:1122:3344:101::2f (added)
+          0dfbf374-9ef9-430f-b06d-f271bf7f84c4 in service crucible [underlay IP fd00:1122:3344:101::27] (unchanged)
+          15c103f0-ac63-423b-ba5d-1b5fcd563ba3 in service nexus [underlay IP fd00:1122:3344:101::22] (unchanged)
+          3aa07966-5899-4789-ace5-f8eeb375c6c3 in service crucible [underlay IP fd00:1122:3344:101::24] (unchanged)
+          4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf in service crucible [underlay IP fd00:1122:3344:101::29] (unchanged)
+          72c5a909-077d-4ec1-a9d5-ae64ef9d716e in service crucible [underlay IP fd00:1122:3344:101::26] (unchanged)
+          95482c25-1e7f-43e8-adf1-e3548a1b3ae0 in service crucible [underlay IP fd00:1122:3344:101::23] (unchanged)
+          a1c03689-fc62-4ea5-bb72-4d01f5138614 in service crucible [underlay IP fd00:1122:3344:101::2a] (unchanged)
+          a568e92e-4fbd-4b69-acd8-f16277073031 in service crucible [underlay IP fd00:1122:3344:101::2c] (unchanged)
+          bf79a56a-97af-4cc4-94a5-8b20d64c2cda in service crucible [underlay IP fd00:1122:3344:101::28] (unchanged)
+          c60379ba-4e30-4628-a79a-0ae509aef4c5 in service crucible [underlay IP fd00:1122:3344:101::25] (unchanged)
+          d47f4996-fac0-4657-bcea-01b1fee6404d in service crucible [underlay IP fd00:1122:3344:101::2b] (unchanged)
+          f1a7b9a7-fc6a-4b23-b829-045ff33117ff in service internal_ntp [underlay IP fd00:1122:3344:101::21] (unchanged)
++         6f86d5cb-17d7-424b-9d4c-39f670532cbe in service nexus [underlay IP fd00:1122:3344:101::2e] (added)
++         87c299eb-470e-4b6d-b8c7-6759694e66b6 in service nexus [underlay IP fd00:1122:3344:101::30] (added)
++         c72b7930-0580-4f00-93b9-8cba2c8d344e in service nexus [underlay IP fd00:1122:3344:101::2d] (added)
++         d0095508-bdb8-4faf-b091-964276a20b15 in service nexus [underlay IP fd00:1122:3344:101::31] (added)
++         ff422442-4b31-4ade-a11a-9e5a25f0404c in service nexus [underlay IP fd00:1122:3344:101::2f] (added)
diff --git a/nexus/src/app/background/blueprint_execution.rs b/nexus/src/app/background/blueprint_execution.rs
index c65a49ec24..7db59bc966 100644
--- a/nexus/src/app/background/blueprint_execution.rs
+++ b/nexus/src/app/background/blueprint_execution.rs
@@ -38,64 +38,72 @@ impl BlueprintExecutor {
     pub fn watcher(&self) -> watch::Receiver<usize> {
         self.tx.subscribe()
     }
-}
 
-impl BackgroundTask for BlueprintExecutor {
-    fn activate<'a>(
-        &'a mut self,
-        opctx: &'a OpContext,
-    ) -> BoxFuture<'a, serde_json::Value> {
-        async {
-            // Get the latest blueprint, cloning to prevent holding a read lock
-            // on the watch.
-            let update = self.rx_blueprint.borrow_and_update().clone();
+    /// Implementation for `BackgroundTask::activate` for `BlueprintExecutor`,
+    /// added here to produce better compile errors.
+    ///
+    /// The presence of `boxed()` in `BackgroundTask::activate` has caused some
+    /// confusion with compilation errors in the past. So separate this method
+    /// out.
+    async fn activate_impl<'a>(
+        &mut self,
+        opctx: &OpContext,
+    ) -> serde_json::Value {
+        // Get the latest blueprint, cloning to prevent holding a read lock
+        // on the watch.
+        let update = self.rx_blueprint.borrow_and_update().clone();
 
-            let Some(update) = update else {
-                warn!(&opctx.log,
+        let Some(update) = update else {
+            warn!(&opctx.log,
                       "Blueprint execution: skipped";
                       "reason" => "no blueprint");
-                return json!({"error": "no blueprint" });
-            };
+            return json!({"error": "no blueprint" });
+        };
 
-            let (bp_target, blueprint) = &*update;
-            if !bp_target.enabled {
-                warn!(&opctx.log,
+        let (bp_target, blueprint) = &*update;
+        if !bp_target.enabled {
+            warn!(&opctx.log,
                       "Blueprint execution: skipped";
                       "reason" => "blueprint disabled",
                       "target_id" => %blueprint.id);
-                return json!({
-                    "target_id": blueprint.id.to_string(),
-                    "error": "blueprint disabled"
-                });
-            }
+            return json!({
+                "target_id": blueprint.id.to_string(),
+                "error": "blueprint disabled"
+            });
+        }
 
-            let result = nexus_reconfigurator_execution::realize_blueprint(
-                opctx,
-                &self.datastore,
-                blueprint,
-                &self.nexus_label,
-            )
-            .await;
+        let result = nexus_reconfigurator_execution::realize_blueprint(
+            opctx,
+            &self.datastore,
+            blueprint,
+            &self.nexus_label,
+        )
+        .await;
 
-            // Trigger anybody waiting for this to finish.
-            self.tx.send_modify(|count| *count = *count + 1);
+        // Trigger anybody waiting for this to finish.
+        self.tx.send_modify(|count| *count = *count + 1);
 
-            // Return the result as a `serde_json::Value`
-            match result {
-                Ok(()) => json!({}),
-                Err(errors) => {
-                    let errors: Vec<_> = errors
-                        .into_iter()
-                        .map(|e| format!("{:#}", e))
-                        .collect();
-                    json!({
-                        "target_id": blueprint.id.to_string(),
-                        "errors": errors
-                    })
-                }
+        // Return the result as a `serde_json::Value`
+        match result {
+            Ok(()) => json!({}),
+            Err(errors) => {
+                let errors: Vec<_> =
+                    errors.into_iter().map(|e| format!("{:#}", e)).collect();
+                json!({
+                    "target_id": blueprint.id.to_string(),
+                    "errors": errors
+                })
             }
         }
-        .boxed()
+    }
+}
+
+impl BackgroundTask for BlueprintExecutor {
+    fn activate<'a>(
+        &'a mut self,
+        opctx: &'a OpContext,
+    ) -> BoxFuture<'a, serde_json::Value> {
+        self.activate_impl(opctx).boxed()
     }
 }
 
@@ -112,8 +120,10 @@ mod test {
     use nexus_db_queries::authn;
     use nexus_db_queries::context::OpContext;
     use nexus_test_utils_macros::nexus_test;
-    use nexus_types::deployment::OmicronZonesConfig;
-    use nexus_types::deployment::{Blueprint, BlueprintTarget};
+    use nexus_types::deployment::{
+        Blueprint, BlueprintTarget, BlueprintZoneConfig,
+        BlueprintZoneDisposition, BlueprintZonesConfig,
+    };
     use nexus_types::inventory::{
         OmicronZoneConfig, OmicronZoneDataset, OmicronZoneType,
     };
@@ -121,7 +131,6 @@ mod test {
     use serde::Deserialize;
     use serde_json::json;
     use std::collections::BTreeMap;
-    use std::collections::BTreeSet;
     use std::net::SocketAddr;
     use std::sync::Arc;
     use tokio::sync::watch;
@@ -131,7 +140,7 @@ mod test {
         nexus_test_utils::ControlPlaneTestContext<crate::Server>;
 
     fn create_blueprint(
-        omicron_zones: BTreeMap<Uuid, OmicronZonesConfig>,
+        blueprint_zones: BTreeMap<Uuid, BlueprintZonesConfig>,
         dns_version: Generation,
     ) -> (BlueprintTarget, Blueprint) {
         let id = Uuid::new_v4();
@@ -143,8 +152,7 @@ mod test {
             },
             Blueprint {
                 id,
-                omicron_zones,
-                zones_in_service: BTreeSet::new(),
+                blueprint_zones,
                 parent_blueprint_id: None,
                 internal_dns_version: dns_version,
                 external_dns_version: dns_version,
@@ -229,31 +237,41 @@ mod test {
         // Create a non-empty blueprint describing two servers and verify that
         // the task correctly winds up making requests to both of them and
         // reporting success.
-        fn make_zones() -> OmicronZonesConfig {
-            OmicronZonesConfig {
+        fn make_zones(
+            disposition: BlueprintZoneDisposition,
+        ) -> BlueprintZonesConfig {
+            BlueprintZonesConfig {
                 generation: Generation::new(),
-                zones: vec![OmicronZoneConfig {
-                    id: Uuid::new_v4(),
-                    underlay_address: "::1".parse().unwrap(),
-                    zone_type: OmicronZoneType::InternalDns {
-                        dataset: OmicronZoneDataset {
-                            pool_name: format!("oxp_{}", Uuid::new_v4())
-                                .parse()
-                                .unwrap(),
+                zones: vec![BlueprintZoneConfig {
+                    config: OmicronZoneConfig {
+                        id: Uuid::new_v4(),
+                        underlay_address: "::1".parse().unwrap(),
+                        zone_type: OmicronZoneType::InternalDns {
+                            dataset: OmicronZoneDataset {
+                                pool_name: format!("oxp_{}", Uuid::new_v4())
+                                    .parse()
+                                    .unwrap(),
+                            },
+                            dns_address: "oh-hello-internal-dns".into(),
+                            gz_address: "::1".parse().unwrap(),
+                            gz_address_index: 0,
+                            http_address: "[::1]:12345".into(),
                         },
-                        dns_address: "oh-hello-internal-dns".into(),
-                        gz_address: "::1".parse().unwrap(),
-                        gz_address_index: 0,
-                        http_address: "some-ipv6-address".into(),
                     },
+                    disposition,
                 }],
             }
         }
+
         let generation = generation.next();
+
+        // Both in-service and quiesced zones should be deployed.
+        //
+        // TODO: add expunged zones to the test (should not be deployed).
         let mut blueprint = create_blueprint(
             BTreeMap::from([
-                (sled_id1, make_zones()),
-                (sled_id2, make_zones()),
+                (sled_id1, make_zones(BlueprintZoneDisposition::InService)),
+                (sled_id2, make_zones(BlueprintZoneDisposition::Quiesced)),
             ]),
             generation,
         );
diff --git a/nexus/src/app/background/blueprint_load.rs b/nexus/src/app/background/blueprint_load.rs
index 614c59ff9b..2afe2d2f97 100644
--- a/nexus/src/app/background/blueprint_load.rs
+++ b/nexus/src/app/background/blueprint_load.rs
@@ -210,7 +210,7 @@ mod test {
     use nexus_types::deployment::{Blueprint, BlueprintTarget};
     use omicron_common::api::external::Generation;
     use serde::Deserialize;
-    use std::collections::{BTreeMap, BTreeSet};
+    use std::collections::BTreeMap;
     use uuid::Uuid;
 
     type ControlPlaneTestContext =
@@ -228,8 +228,7 @@ mod test {
             },
             Blueprint {
                 id,
-                omicron_zones: BTreeMap::new(),
-                zones_in_service: BTreeSet::new(),
+                blueprint_zones: BTreeMap::new(),
                 parent_blueprint_id: Some(parent_blueprint_id),
                 internal_dns_version: Generation::new(),
                 external_dns_version: Generation::new(),
diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs
index 0358cf523c..cc9c8c43df 100644
--- a/nexus/test-utils/src/lib.rs
+++ b/nexus/test-utils/src/lib.rs
@@ -26,6 +26,9 @@ use nexus_config::NexusConfig;
 use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use nexus_test_interface::NexusServer;
 use nexus_types::deployment::Blueprint;
+use nexus_types::deployment::BlueprintZoneConfig;
+use nexus_types::deployment::BlueprintZoneDisposition;
+use nexus_types::deployment::BlueprintZonesConfig;
 use nexus_types::external_api::params::UserId;
 use nexus_types::internal_api::params::Certificate;
 use nexus_types::internal_api::params::DatasetCreateRequest;
@@ -57,7 +60,6 @@ use oximeter_producer::LogConfig;
 use oximeter_producer::Server as ProducerServer;
 use slog::{debug, error, o, Logger};
 use std::collections::BTreeMap;
-use std::collections::BTreeSet;
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6};
@@ -790,29 +792,34 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
         };
 
         let blueprint = {
-            let mut omicron_zones = BTreeMap::new();
-            let mut zones_in_service = BTreeSet::new();
+            let mut blueprint_zones = BTreeMap::new();
             for (maybe_sled_agent, zones) in [
                 (self.sled_agent.as_ref(), &self.omicron_zones),
                 (self.sled_agent2.as_ref(), &self.omicron_zones2),
             ] {
                 if let Some(sa) = maybe_sled_agent {
-                    omicron_zones.insert(
+                    blueprint_zones.insert(
                         sa.sled_agent.id,
-                        OmicronZonesConfig {
+                        BlueprintZonesConfig {
                             generation: Generation::new().next(),
-                            zones: zones.clone(),
+                            zones: zones
+                                .iter()
+                                .map(|z| {
+                                    BlueprintZoneConfig {
+                                        config: z.clone(),
+                                        // All initial zones are in-service
+                                        disposition:
+                                            BlueprintZoneDisposition::InService,
+                                    }
+                                })
+                                .collect(),
                         },
                     );
-                    for z in zones {
-                        zones_in_service.insert(z.id);
-                    }
                 }
             }
             Blueprint {
                 id: Uuid::new_v4(),
-                omicron_zones,
-                zones_in_service,
+                blueprint_zones,
                 parent_blueprint_id: None,
                 internal_dns_version: dns_config
                     .generation
diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs
index a6b53272eb..b435964b53 100644
--- a/nexus/types/src/deployment.rs
+++ b/nexus/types/src/deployment.rs
@@ -31,6 +31,8 @@ use serde::Serialize;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::fmt;
+use strum::EnumIter;
+use strum::IntoEnumIterator;
 use uuid::Uuid;
 
 /// Fleet-wide deployment policy
@@ -138,14 +140,12 @@ pub struct Blueprint {
     /// unique identifier for this blueprint
     pub id: Uuid,
 
-    /// mapping: sled id -> zones deployed on each sled
+    /// A map of sled id -> zones deployed on each sled, along with the
+    /// [`BlueprintZoneDisposition`] for each zone.
+    ///
     /// A sled is considered part of the control plane cluster iff it has an
     /// entry in this map.
-    pub omicron_zones: BTreeMap<Uuid, OmicronZonesConfig>,
-
-    /// Omicron zones considered in-service (which generally means that they
-    /// should appear in DNS)
-    pub zones_in_service: BTreeSet<Uuid>,
+    pub blueprint_zones: BTreeMap<Uuid, BlueprintZonesConfig>,
 
     /// which blueprint this blueprint is based on
     pub parent_blueprint_id: Option<Uuid>,
@@ -169,19 +169,32 @@ pub struct Blueprint {
 }
 
 impl Blueprint {
-    /// Iterate over all the Omicron zones in the blueprint, along with
-    /// associated sled id
+    /// Iterate over the [`BlueprintZoneConfig`] instances in the blueprint
+    /// that match the provided filter, along with the associated sled id.
+    pub fn all_blueprint_zones(
+        &self,
+        filter: BlueprintZoneFilter,
+    ) -> impl Iterator<Item = (Uuid, &BlueprintZoneConfig)> {
+        self.blueprint_zones.iter().flat_map(move |(sled_id, z)| {
+            z.zones.iter().filter_map(move |z| {
+                z.disposition.matches(filter).then_some((*sled_id, z))
+            })
+        })
+    }
+
+    /// Iterate over all the [`OmicronZoneConfig`] instances in the blueprint,
+    /// along with the associated sled id.
     pub fn all_omicron_zones(
         &self,
     ) -> impl Iterator<Item = (Uuid, &OmicronZoneConfig)> {
-        self.omicron_zones
-            .iter()
-            .flat_map(|(sled_id, z)| z.zones.iter().map(|z| (*sled_id, z)))
+        self.blueprint_zones.iter().flat_map(|(sled_id, z)| {
+            z.zones.iter().map(|z| (*sled_id, &z.config))
+        })
     }
 
     /// Iterate over the ids of all sleds in the blueprint
     pub fn sleds(&self) -> impl Iterator<Item = Uuid> + '_ {
-        self.omicron_zones.keys().copied()
+        self.blueprint_zones.keys().copied()
     }
 
     /// Summarize the difference between sleds and zones between two blueprints
@@ -191,43 +204,58 @@ impl Blueprint {
     ) -> OmicronZonesDiff<'a> {
         OmicronZonesDiff {
             before_label: format!("blueprint {}", self.id),
-            before_zones: self.omicron_zones.clone(),
-            before_zones_in_service: &self.zones_in_service,
+            before_zones: self.blueprint_zones.clone(),
             after_label: format!("blueprint {}", other.id),
-            after_zones: &other.omicron_zones,
-            after_zones_in_service: &other.zones_in_service,
+            after_zones: &other.blueprint_zones,
         }
     }
 
     /// Summarize the differences in sleds and zones between a collection and a
     /// blueprint
     ///
-    /// This gives an idea about what would change about a running system if one
-    /// were to execute the blueprint.
+    /// This gives an idea about what would change about a running system if
+    /// one were to execute the blueprint.
     ///
     /// Note that collections do not currently include information about what
-    /// zones are in-service, so the caller must provide that information.
-    pub fn diff_sleds_from_collection<'a>(
-        &'a self,
-        collection: &'a Collection,
-        before_zones_in_service: &'a BTreeSet<Uuid>,
-    ) -> OmicronZonesDiff<'a> {
+    /// zones are in-service, so it is assumed that all zones in the collection
+    /// are in-service. (This is the same assumption made by
+    /// [`BlueprintZonesConfig::initial_from_collection`]. The logic here may
+    /// also be expanded to handle cases where not all zones in the collection
+    /// are in-service.)
+    pub fn diff_sleds_from_collection(
+        &self,
+        collection: &Collection,
+    ) -> OmicronZonesDiff<'_> {
         let before_zones = collection
             .omicron_zones
             .iter()
-            .map(|(sled_id, zones_found)| (*sled_id, zones_found.zones.clone()))
+            .map(|(sled_id, zones_found)| {
+                let zones = zones_found
+                    .zones
+                    .zones
+                    .iter()
+                    .map(|z| BlueprintZoneConfig {
+                        config: z.clone(),
+                        disposition: BlueprintZoneDisposition::InService,
+                    })
+                    .collect();
+                let zones = BlueprintZonesConfig {
+                    generation: zones_found.zones.generation,
+                    zones,
+                };
+                (*sled_id, zones)
+            })
             .collect();
         OmicronZonesDiff {
             before_label: format!("collection {}", collection.id),
             before_zones,
-            before_zones_in_service,
             after_label: format!("blueprint {}", self.id),
-            after_zones: &self.omicron_zones,
-            after_zones_in_service: &self.zones_in_service,
+            after_zones: &self.blueprint_zones,
         }
     }
 
-    /// Return a struct that can be displayed.
+    /// Return a struct that can be displayed to present information about the
+    /// blueprint.
     pub fn display(&self) -> BlueprintDisplay<'_> {
         BlueprintDisplay { blueprint: self }
     }
@@ -237,6 +265,7 @@ impl Blueprint {
 ///
 /// Returned by [`Blueprint::display()`].
 #[derive(Clone, Debug)]
+#[must_use = "this struct does nothing unless displayed"]
 pub struct BlueprintDisplay<'a> {
     blueprint: &'a Blueprint,
     // TODO: add colorization with a stylesheet
@@ -271,24 +300,15 @@ impl<'a> fmt::Display for BlueprintDisplay<'a> {
         writeln!(f, "internal DNS version: {}", b.internal_dns_version)?;
         writeln!(f, "comment: {}", b.comment)?;
         writeln!(f, "zones:\n")?;
-        for (sled_id, sled_zones) in &b.omicron_zones {
+
+        for (sled_id, sled_zones) in &b.blueprint_zones {
             writeln!(
                 f,
                 "  sled {}: Omicron zones at generation {}",
                 sled_id, sled_zones.generation
             )?;
             for z in &sled_zones.zones {
-                writeln!(
-                    f,
-                    "    {} {} {}",
-                    z.id,
-                    if b.zones_in_service.contains(&z.id) {
-                        "in service    "
-                    } else {
-                        "not in service"
-                    },
-                    z.zone_type.label(),
-                )?;
+                writeln!(f, "    {}", z.display())?;
             }
         }
 
@@ -296,6 +316,236 @@ impl<'a> fmt::Display for BlueprintDisplay<'a> {
     }
 }
 
+/// Information about an Omicron zone as recorded in a blueprint.
+///
+/// Currently, this is similar to [`OmicronZonesConfig`], but also contains a
+/// per-zone [`BlueprintZoneDisposition`].
+///
+/// Part of [`Blueprint`].
+#[derive(Debug, Clone, Eq, PartialEq, JsonSchema, Deserialize, Serialize)]
+pub struct BlueprintZonesConfig {
+    /// Generation number of this configuration.
+    ///
+    /// This generation number is owned by the control plane. See
+    /// [`OmicronZonesConfig::generation`] for more details.
+    pub generation: Generation,
+
+    /// The list of running zones.
+    pub zones: Vec<BlueprintZoneConfig>,
+}
+
+impl BlueprintZonesConfig {
+    /// Constructs a new [`BlueprintZonesConfig`] from a collection's zones.
+    ///
+    /// For the initial blueprint, all zones within a collection are assumed to
+    /// be in-service.
+    pub fn initial_from_collection(collection: &OmicronZonesConfig) -> Self {
+        let zones = collection
+            .zones
+            .iter()
+            .map(|z| BlueprintZoneConfig {
+                config: z.clone(),
+                disposition: BlueprintZoneDisposition::InService,
+            })
+            .collect();
+
+        let mut ret = Self {
+            // An initial `BlueprintZonesConfig` reuses the generation from
+            // `OmicronZonesConfig`.
+            generation: collection.generation,
+            zones,
+        };
+        // For testing, it's helpful for zones to be in sorted order.
+        ret.sort();
+
+        ret
+    }
+
+    /// Sorts the list of zones stored in this configuration.
+    ///
+    /// This is not strictly necessary. But for testing, it's helpful for
+    /// zones to be in sorted order.
+    pub fn sort(&mut self) {
+        self.zones.sort_unstable_by_key(|z| z.config.id);
+    }
+
+    /// Converts self to an [`OmicronZonesConfig`], applying the provided
+    /// [`BlueprintZoneFilter`].
+    ///
+    /// The filter controls which zones should be exported into the resulting
+    /// [`OmicronZonesConfig`].
+    pub fn to_omicron_zones_config(
+        &self,
+        filter: BlueprintZoneFilter,
+    ) -> OmicronZonesConfig {
+        OmicronZonesConfig {
+            generation: self.generation,
+            zones: self
+                .zones
+                .iter()
+                .filter_map(|z| {
+                    z.disposition.matches(filter).then(|| z.config.clone())
+                })
+                .collect(),
+        }
+    }
+}
+
+/// Describes one Omicron-managed zone in a blueprint.
+///
+/// This is a wrapper around an [`OmicronZoneConfig`] that also includes a
+/// [`BlueprintZoneDisposition`].
+///
+/// Part of [`BlueprintZonesConfig`].
+#[derive(Debug, Clone, Eq, PartialEq, JsonSchema, Deserialize, Serialize)]
+pub struct BlueprintZoneConfig {
+    /// The underlying zone configuration.
+    pub config: OmicronZoneConfig,
+
+    /// The disposition (desired state) of this zone recorded in the blueprint.
+    pub disposition: BlueprintZoneDisposition,
+}
+
+impl BlueprintZoneConfig {
+    /// Return a struct that can be displayed to present information about the
+    /// zone.
+    pub fn display(&self) -> BlueprintZoneConfigDisplay<'_> {
+        BlueprintZoneConfigDisplay { zone: self }
+    }
+}
+
+/// A wrapper to allow a [`BlueprintZoneConfig`] to be displayed with
+/// information.
+///
+/// Returned by [`BlueprintZoneConfig::display()`].
+#[derive(Clone, Debug)]
+#[must_use = "this struct does nothing unless displayed"]
+pub struct BlueprintZoneConfigDisplay<'a> {
+    zone: &'a BlueprintZoneConfig,
+}
+
+impl<'a> fmt::Display for BlueprintZoneConfigDisplay<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let z = self.zone;
+        write!(
+            f,
+            "{} {:<width$} {} [underlay IP {}]",
+            z.config.id,
+            z.disposition,
+            z.config.zone_type.label(),
+            z.config.underlay_address,
+            width = BlueprintZoneDisposition::DISPLAY_WIDTH,
+        )
+    }
+}
+
+/// The desired state of an Omicron-managed zone in a blueprint.
+///
+/// Part of [`BlueprintZoneConfig`].
+#[derive(
+    Debug,
+    Copy,
+    Clone,
+    PartialEq,
+    Eq,
+    Hash,
+    PartialOrd,
+    Ord,
+    JsonSchema,
+    Deserialize,
+    Serialize,
+    EnumIter,
+)]
+#[serde(rename_all = "snake_case")]
+pub enum BlueprintZoneDisposition {
+    /// The zone is in-service.
+    InService,
+
+    /// The zone is not in service.
+    Quiesced,
+}
+
+impl BlueprintZoneDisposition {
+    /// The maximum width of `Display` output.
+    const DISPLAY_WIDTH: usize = 10;
+
+    /// Returns true if the zone disposition matches this filter.
+    pub fn matches(self, filter: BlueprintZoneFilter) -> bool {
+        // This code could be written in three ways:
+        //
+        // 1. match self { match filter { ... } }
+        // 2. match filter { match self { ... } }
+        // 3. match (self, filter) { ... }
+        //
+        // We choose 1 here because we expect many filters and just a few
+        // dispositions, and 1 is the easiest form to represent that.
+        match self {
+            Self::InService => match filter {
+                BlueprintZoneFilter::All => true,
+                BlueprintZoneFilter::SledAgentPut => true,
+                BlueprintZoneFilter::InternalDns => true,
+                BlueprintZoneFilter::VpcFirewall => true,
+            },
+            Self::Quiesced => match filter {
+                BlueprintZoneFilter::All => true,
+
+                // Quiesced zones should not be exposed in DNS.
+                BlueprintZoneFilter::InternalDns => false,
+
+                // Quiesced zones are expected to be deployed by sled-agent.
+                BlueprintZoneFilter::SledAgentPut => true,
+
+                // Quiesced zones should get firewall rules.
+                BlueprintZoneFilter::VpcFirewall => true,
+            },
+        }
+    }
+
+    /// Returns all zone dispositions that match the given filter.
+    pub fn all_matching(
+        filter: BlueprintZoneFilter,
+    ) -> impl Iterator<Item = Self> {
+        BlueprintZoneDisposition::iter().filter(move |&d| d.matches(filter))
+    }
+}
+
+impl fmt::Display for BlueprintZoneDisposition {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            // Neither `write!(f, "...")` nor `f.write_str("...")` obey fill
+            // and alignment (used above), but this does.
+            BlueprintZoneDisposition::InService => "in service".fmt(f),
+            BlueprintZoneDisposition::Quiesced => "quiesced".fmt(f),
+        }
+    }
+}
+
+/// Filters that apply to blueprint zones.
+///
+/// This logic lives here rather than within the individual components making
+/// decisions, so that this is easier to read.
+///
+/// The meaning of a particular filter should not be overloaded -- each time a
+/// new use case wants to make a decision based on the zone disposition, a new
+/// variant should be added to this enum.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub enum BlueprintZoneFilter {
+    // ---
+    // Prefer to keep this list in alphabetical order.
+    // ---
+    /// All zones.
+    All,
+
+    /// Filter by zones that should be in internal DNS.
+    InternalDns,
+
+    /// Filter by zones that we should tell sled-agent to deploy.
+    SledAgentPut,
+
+    /// Filter by zones that should be sent VPC firewall rules.
+    VpcFirewall,
+}
+
 /// Describe high-level metadata about a blueprint
 // These fields are a subset of [`Blueprint`], and include only the data we can
 // quickly fetch from the main blueprint table (e.g., when listing all
@@ -348,11 +598,9 @@ pub struct OmicronZonesDiff<'a> {
     before_label: String,
     // We store an owned copy of "before_zones" to make it easier to support
     // collections here, where we need to assemble this map ourselves.
-    before_zones: BTreeMap<Uuid, OmicronZonesConfig>,
-    before_zones_in_service: &'a BTreeSet<Uuid>,
+    before_zones: BTreeMap<Uuid, BlueprintZonesConfig>,
     after_label: String,
-    after_zones: &'a BTreeMap<Uuid, OmicronZonesConfig>,
-    after_zones_in_service: &'a BTreeSet<Uuid>,
+    after_zones: &'a BTreeMap<Uuid, BlueprintZonesConfig>,
 }
 
 /// Describes a sled that appeared on both sides of a diff (possibly changed)
@@ -364,8 +612,8 @@ pub struct DiffSledCommon<'a> {
     pub generation_before: Generation,
     /// generation of the "zones" configuration on the right side
     pub generation_after: Generation,
-    zones_added: Vec<&'a OmicronZoneConfig>,
-    zones_removed: Vec<&'a OmicronZoneConfig>,
+    zones_added: Vec<&'a BlueprintZoneConfig>,
+    zones_removed: Vec<&'a BlueprintZoneConfig>,
     zones_common: Vec<DiffZoneCommon<'a>>,
 }
 
@@ -373,14 +621,14 @@ impl<'a> DiffSledCommon<'a> {
     /// Iterate over zones added between the blueprints
     pub fn zones_added(
         &self,
-    ) -> impl Iterator<Item = &'a OmicronZoneConfig> + '_ {
+    ) -> impl Iterator<Item = &'a BlueprintZoneConfig> + '_ {
         self.zones_added.iter().copied()
     }
 
     /// Iterate over zones removed between the blueprints
     pub fn zones_removed(
         &self,
-    ) -> impl Iterator<Item = &'a OmicronZoneConfig> + '_ {
+    ) -> impl Iterator<Item = &'a BlueprintZoneConfig> + '_ {
         self.zones_removed.iter().copied()
     }
 
@@ -395,8 +643,7 @@ impl<'a> DiffSledCommon<'a> {
     pub fn zones_changed(
         &self,
     ) -> impl Iterator<Item = DiffZoneCommon<'a>> + '_ {
-        self.zones_in_common()
-            .filter(|z| z.changed_how != DiffZoneChangedHow::NoChanges)
+        self.zones_in_common().filter(|z| z.is_changed())
     }
 }
 
@@ -404,24 +651,34 @@ impl<'a> DiffSledCommon<'a> {
 #[derive(Debug, Copy, Clone)]
 pub struct DiffZoneCommon<'a> {
     /// full zone configuration before
-    pub zone_before: &'a OmicronZoneConfig,
+    pub zone_before: &'a BlueprintZoneConfig,
     /// full zone configuration after
-    pub zone_after: &'a OmicronZoneConfig,
-    /// summary of what changed, if anything
-    pub changed_how: DiffZoneChangedHow,
+    pub zone_after: &'a BlueprintZoneConfig,
 }
 
-/// Describes how a zone changed across two blueprints, if at all
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub enum DiffZoneChangedHow {
-    /// the zone did not change between these two blueprints
-    NoChanges,
-    /// the zone details are the same, but it was brought into service
-    AddedToService,
-    /// the zone details are the same, but it was removed from service
-    RemovedFromService,
-    /// the zone's details (i.e., configuration) changed
-    DetailsChanged,
+impl<'a> DiffZoneCommon<'a> {
+    /// Returns true if there are any differences between `zone_before` and
+    /// `zone_after`.
+    ///
+    /// This is equivalent to `config_changed() || disposition_changed()`.
+    #[inline]
+    pub fn is_changed(&self) -> bool {
+        // state is smaller and easier to compare than config.
+        self.disposition_changed() || self.config_changed()
+    }
+
+    /// Returns true if the zone configuration (excluding the disposition)
+    /// changed.
+    #[inline]
+    pub fn config_changed(&self) -> bool {
+        self.zone_before.config != self.zone_after.config
+    }
+
+    /// Returns true if the [`BlueprintZoneDisposition`] for the zone changed.
+    #[inline]
+    pub fn disposition_changed(&self) -> bool {
+        self.zone_before.disposition != self.zone_after.disposition
+    }
 }
 
 impl<'a> OmicronZonesDiff<'a> {
@@ -436,7 +693,7 @@ impl<'a> OmicronZonesDiff<'a> {
     /// Iterate over sleds only present in the second blueprint of a diff
     pub fn sleds_added(
         &self,
-    ) -> impl Iterator<Item = (Uuid, &OmicronZonesConfig)> + '_ {
+    ) -> impl Iterator<Item = (Uuid, &BlueprintZonesConfig)> + '_ {
         let sled_ids = self
             .sleds_after()
             .difference(&self.sleds_before())
@@ -451,7 +708,7 @@ impl<'a> OmicronZonesDiff<'a> {
     /// Iterate over sleds only present in the first blueprint of a diff
     pub fn sleds_removed(
         &self,
-    ) -> impl Iterator<Item = (Uuid, &OmicronZonesConfig)> + '_ {
+    ) -> impl Iterator<Item = (Uuid, &BlueprintZonesConfig)> + '_ {
         let sled_ids = self
             .sleds_before()
             .difference(&self.sleds_after())
@@ -476,71 +733,33 @@ impl<'a> OmicronZonesDiff<'a> {
             let b2sledzones = self.after_zones.get(&sled_id).unwrap();
 
             // Assemble separate summaries of the zones, indexed by zone id.
-            #[derive(Debug)]
-            struct ZoneInfo<'a> {
-                zone: &'a OmicronZoneConfig,
-                in_service: bool,
-            }
-
-            let b1zones: BTreeMap<Uuid, ZoneInfo> = b1sledzones
+            let b1_zones: BTreeMap<Uuid, &'a BlueprintZoneConfig> = b1sledzones
                 .zones
                 .iter()
-                .map(|zone| {
-                    (
-                        zone.id,
-                        ZoneInfo {
-                            zone,
-                            in_service: self
-                                .before_zones_in_service
-                                .contains(&zone.id),
-                        },
-                    )
-                })
-                .collect();
-            let mut b2zones: BTreeMap<Uuid, ZoneInfo> = b2sledzones
-                .zones
-                .iter()
-                .map(|zone| {
-                    (
-                        zone.id,
-                        ZoneInfo {
-                            zone,
-                            in_service: self
-                                .after_zones_in_service
-                                .contains(&zone.id),
-                        },
-                    )
-                })
+                .map(|zone| (zone.config.id, zone))
                 .collect();
+            let mut b2_zones: BTreeMap<Uuid, &'a BlueprintZoneConfig> =
+                b2sledzones
+                    .zones
+                    .iter()
+                    .map(|zone| (zone.config.id, zone))
+                    .collect();
             let mut zones_removed = vec![];
-            let mut zones_changed = vec![];
+            let mut zones_common = vec![];
 
             // Now go through each zone and compare them.
-            for (zone_id, b1z_info) in &b1zones {
-                if let Some(b2z_info) = b2zones.remove(zone_id) {
-                    let changed_how = if b1z_info.zone != b2z_info.zone {
-                        DiffZoneChangedHow::DetailsChanged
-                    } else if b1z_info.in_service && !b2z_info.in_service {
-                        DiffZoneChangedHow::RemovedFromService
-                    } else if !b1z_info.in_service && b2z_info.in_service {
-                        DiffZoneChangedHow::AddedToService
-                    } else {
-                        DiffZoneChangedHow::NoChanges
-                    };
-                    zones_changed.push(DiffZoneCommon {
-                        zone_before: b1z_info.zone,
-                        zone_after: b2z_info.zone,
-                        changed_how,
-                    });
+            for (zone_id, zone_before) in &b1_zones {
+                if let Some(zone_after) = b2_zones.remove(zone_id) {
+                    zones_common
+                        .push(DiffZoneCommon { zone_before, zone_after });
                 } else {
-                    zones_removed.push(b1z_info.zone);
+                    zones_removed.push(*zone_before);
                 }
             }
 
             // Since we removed common zones above, anything else exists only in
             // b2 and was therefore added.
-            let zones_added =
-                b2zones.into_values().map(|b2z_info| b2z_info.zone).collect();
+            let zones_added = b2_zones.into_values().collect();
 
             (
                 sled_id,
@@ -550,7 +769,7 @@ impl<'a> OmicronZonesDiff<'a> {
                     generation_after: b2sledzones.generation,
                     zones_added,
                     zones_removed,
-                    zones_common: zones_changed,
+                    zones_common,
                 },
             )
         })
@@ -573,7 +792,12 @@ impl<'a> OmicronZonesDiff<'a> {
     }
 }
 
+/// Wrapper to allow a [`OmicronZonesDiff`] to be displayed in a unified
+/// `diff(1)`-like format.
+///
+/// Returned by [`OmicronZonesDiff::display()`].
 #[derive(Clone, Debug)]
+#[must_use = "this struct does nothing unless displayed"]
 pub struct OmicronZonesDiffDisplay<'diff, 'a> {
     diff: &'diff OmicronZonesDiff<'a>,
     // TODO: add colorization with a stylesheet
@@ -590,7 +814,7 @@ impl<'diff, 'a> OmicronZonesDiffDisplay<'diff, 'a> {
         f: &mut fmt::Formatter<'_>,
         prefix: char,
         label: &str,
-        bbsledzones: &OmicronZonesConfig,
+        bbsledzones: &BlueprintZonesConfig,
         sled_id: Uuid,
     ) -> fmt::Result {
         writeln!(f, "{} sled {} ({})", prefix, sled_id, label)?;
@@ -600,15 +824,7 @@ impl<'diff, 'a> OmicronZonesDiffDisplay<'diff, 'a> {
             prefix, bbsledzones.generation
         )?;
         for z in &bbsledzones.zones {
-            writeln!(
-                f,
-                "{}         zone {} type {} underlay IP {} ({})",
-                prefix,
-                z.id,
-                z.zone_type.label(),
-                z.underlay_address,
-                label
-            )?;
+            writeln!(f, "{prefix}         {} ({label})", z.display())?;
         }
 
         Ok(())
@@ -650,94 +866,43 @@ impl<'diff, 'a> fmt::Display for OmicronZonesDiffDisplay<'diff, 'a> {
             }
 
             for zone in sled_changes.zones_removed() {
-                writeln!(
-                    f,
-                    "-        zone {} type {} (removed)",
-                    zone.id,
-                    zone.zone_type.label(),
-                )?;
+                writeln!(f, "-         {} (removed)", zone.display())?;
             }
 
             for zone_changes in sled_changes.zones_in_common() {
-                let zone_id = zone_changes.zone_before.id;
-                let zone_type = zone_changes.zone_before.zone_type.label();
-                let zone2_type = zone_changes.zone_after.zone_type.label();
-                match zone_changes.changed_how {
-                    DiffZoneChangedHow::DetailsChanged => {
-                        writeln!(
-                            f,
-                            "-         zone {} type {} underlay IP {} \
-                                (changed)",
-                            zone_id,
-                            zone_type,
-                            zone_changes.zone_before.underlay_address,
-                        )?;
-                        writeln!(
-                            f,
-                            "+         zone {} type {} underlay IP {} \
-                                (changed)",
-                            zone_id,
-                            zone2_type,
-                            zone_changes.zone_after.underlay_address,
-                        )?;
-                    }
-                    DiffZoneChangedHow::RemovedFromService => {
-                        writeln!(
-                            f,
-                            "-         zone {} type {} underlay IP {} \
-                                (in service)",
-                            zone_id,
-                            zone_type,
-                            zone_changes.zone_before.underlay_address,
-                        )?;
-                        writeln!(
-                            f,
-                            "+         zone {} type {} underlay IP {} \
-                                (removed from service)",
-                            zone_id,
-                            zone2_type,
-                            zone_changes.zone_after.underlay_address,
-                        )?;
-                    }
-                    DiffZoneChangedHow::AddedToService => {
-                        writeln!(
-                            f,
-                            "-         zone {} type {} underlay IP {} \
-                                (not in service)",
-                            zone_id,
-                            zone_type,
-                            zone_changes.zone_before.underlay_address,
-                        )?;
-                        writeln!(
-                            f,
-                            "+         zone {} type {} underlay IP {} \
-                                (added to service)",
-                            zone_id,
-                            zone2_type,
-                            zone_changes.zone_after.underlay_address,
-                        )?;
-                    }
-                    DiffZoneChangedHow::NoChanges => {
-                        writeln!(
-                            f,
-                            "         zone {} type {} underlay IP {} \
-                                (unchanged)",
-                            zone_id,
-                            zone_type,
-                            zone_changes.zone_before.underlay_address,
-                        )?;
-                    }
+                if zone_changes.config_changed() {
+                    writeln!(
+                        f,
+                        "-         {} (changed)",
+                        zone_changes.zone_before.display(),
+                    )?;
+                    writeln!(
+                        f,
+                        "+         {} (changed)",
+                        zone_changes.zone_after.display(),
+                    )?;
+                } else if zone_changes.disposition_changed() {
+                    writeln!(
+                        f,
+                        "-         {} (disposition changed)",
+                        zone_changes.zone_before.display(),
+                    )?;
+                    writeln!(
+                        f,
+                        "+         {} (disposition changed)",
+                        zone_changes.zone_after.display(),
+                    )?;
+                } else {
+                    writeln!(
+                        f,
+                        "          {} (unchanged)",
+                        zone_changes.zone_before.display(),
+                    )?;
                 }
             }
 
             for zone in sled_changes.zones_added() {
-                writeln!(
-                    f,
-                    "+        zone {} type {} underlay IP {} (added)",
-                    zone.id,
-                    zone.zone_type.label(),
-                    zone.underlay_address,
-                )?;
+                writeln!(f, "+         {} (added)", zone.display())?;
             }
         }
 
diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json
index 0beab820a1..cba8063b7e 100644
--- a/openapi/nexus-internal.json
+++ b/openapi/nexus-internal.json
@@ -2589,6 +2589,13 @@
         "description": "Describes a complete set of software and configuration for the system",
         "type": "object",
         "properties": {
+          "blueprint_zones": {
+            "description": "A map of sled id -> zones deployed on each sled, along with the [`BlueprintZoneDisposition`] for each zone.\n\nA sled is considered part of the control plane cluster iff it has an entry in this map.",
+            "type": "object",
+            "additionalProperties": {
+              "$ref": "#/components/schemas/BlueprintZonesConfig"
+            }
+          },
           "comment": {
             "description": "human-readable string describing why this blueprint was created (for debugging)",
             "type": "string"
@@ -2618,13 +2625,6 @@
               }
             ]
           },
-          "omicron_zones": {
-            "description": "mapping: sled id -> zones deployed on each sled A sled is considered part of the control plane cluster iff it has an entry in this map.",
-            "type": "object",
-            "additionalProperties": {
-              "$ref": "#/components/schemas/OmicronZonesConfig"
-            }
-          },
           "parent_blueprint_id": {
             "nullable": true,
             "description": "which blueprint this blueprint is based on",
@@ -2635,26 +2635,16 @@
             "description": "when this blueprint was generated (for debugging)",
             "type": "string",
             "format": "date-time"
-          },
-          "zones_in_service": {
-            "description": "Omicron zones considered in-service (which generally means that they should appear in DNS)",
-            "type": "array",
-            "items": {
-              "type": "string",
-              "format": "uuid"
-            },
-            "uniqueItems": true
           }
         },
         "required": [
+          "blueprint_zones",
           "comment",
           "creator",
           "external_dns_version",
           "id",
           "internal_dns_version",
-          "omicron_zones",
-          "time_created",
-          "zones_in_service"
+          "time_created"
         ]
       },
       "BlueprintMetadata": {
@@ -2774,6 +2764,76 @@
           "target_id"
         ]
       },
+      "BlueprintZoneConfig": {
+        "description": "Describes one Omicron-managed zone in a blueprint.\n\nThis is a wrapper around an [`OmicronZoneConfig`] that also includes a [`BlueprintZoneDisposition`].\n\nPart of [`BlueprintZonesConfig`].",
+        "type": "object",
+        "properties": {
+          "config": {
+            "description": "The underlying zone configuration.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/OmicronZoneConfig"
+              }
+            ]
+          },
+          "disposition": {
+            "description": "The disposition (desired state) of this zone recorded in the blueprint.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/BlueprintZoneDisposition"
+              }
+            ]
+          }
+        },
+        "required": [
+          "config",
+          "disposition"
+        ]
+      },
+      "BlueprintZoneDisposition": {
+        "description": "The desired state of an Omicron-managed zone in a blueprint.\n\nPart of [`BlueprintZoneConfig`].",
+        "oneOf": [
+          {
+            "description": "The zone is in-service.",
+            "type": "string",
+            "enum": [
+              "in_service"
+            ]
+          },
+          {
+            "description": "The zone is not in service.",
+            "type": "string",
+            "enum": [
+              "quiesced"
+            ]
+          }
+        ]
+      },
+      "BlueprintZonesConfig": {
+        "description": "Information about an Omicron zone as recorded in a blueprint.\n\nCurrently, this is similar to [`OmicronZonesConfig`], but also contains a per-zone [`BlueprintZoneDisposition`].\n\nPart of [`Blueprint`].",
+        "type": "object",
+        "properties": {
+          "generation": {
+            "description": "Generation number of this configuration.\n\nThis generation number is owned by the control plane. See [`OmicronZonesConfig::generation`] for more details.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Generation"
+              }
+            ]
+          },
+          "zones": {
+            "description": "The list of running zones.",
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/BlueprintZoneConfig"
+            }
+          }
+        },
+        "required": [
+          "generation",
+          "zones"
+        ]
+      },
       "ByteCount": {
         "description": "Byte count to express memory or storage capacity.",
         "type": "integer",
@@ -5835,31 +5895,6 @@
           }
         ]
       },
-      "OmicronZonesConfig": {
-        "description": "Describes the set of Omicron-managed zones running on a sled\n\n<details><summary>JSON schema</summary>\n\n```json { \"description\": \"Describes the set of Omicron-managed zones running on a sled\", \"type\": \"object\", \"required\": [ \"generation\", \"zones\" ], \"properties\": { \"generation\": { \"description\": \"generation number of this configuration\\n\\nThis generation number is owned by the control plane (i.e., RSS or Nexus, depending on whether RSS-to-Nexus handoff has happened).  It should not be bumped within Sled Agent.\\n\\nSled Agent rejects attempts to set the configuration to a generation older than the one it's currently running.\", \"allOf\": [ { \"$ref\": \"#/components/schemas/Generation\" } ] }, \"zones\": { \"description\": \"list of running zones\", \"type\": \"array\", \"items\": { \"$ref\": \"#/components/schemas/OmicronZoneConfig\" } } } } ``` </details>",
-        "type": "object",
-        "properties": {
-          "generation": {
-            "description": "generation number of this configuration\n\nThis generation number is owned by the control plane (i.e., RSS or Nexus, depending on whether RSS-to-Nexus handoff has happened).  It should not be bumped within Sled Agent.\n\nSled Agent rejects attempts to set the configuration to a generation older than the one it's currently running.",
-            "allOf": [
-              {
-                "$ref": "#/components/schemas/Generation"
-              }
-            ]
-          },
-          "zones": {
-            "description": "list of running zones",
-            "type": "array",
-            "items": {
-              "$ref": "#/components/schemas/OmicronZoneConfig"
-            }
-          }
-        },
-        "required": [
-          "generation",
-          "zones"
-        ]
-      },
       "OximeterInfo": {
         "description": "Message used to notify Nexus that this oximeter instance is up and running.",
         "type": "object",
diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs
index 8c678daf30..587625fe7b 100644
--- a/sled-agent/src/rack_setup/service.rs
+++ b/sled-agent/src/rack_setup/service.rs
@@ -91,7 +91,10 @@ use internal_dns::ServiceName;
 use nexus_client::{
     types as NexusTypes, Client as NexusClient, Error as NexusError,
 };
-use nexus_types::deployment::Blueprint;
+use nexus_types::deployment::{
+    Blueprint, BlueprintZoneConfig, BlueprintZoneDisposition,
+    BlueprintZonesConfig,
+};
 use omicron_common::address::get_sled_address;
 use omicron_common::api::external::Generation;
 use omicron_common::api::internal::shared::ExternalPortDiscovery;
@@ -1158,35 +1161,37 @@ pub(crate) fn build_initial_blueprint_from_sled_configs(
     sled_configs: BTreeMap<Uuid, SledConfig>,
     internal_dns_version: Generation,
 ) -> Blueprint {
-    let mut omicron_zones = BTreeMap::new();
-    let mut zones_in_service = BTreeSet::new();
+    let mut blueprint_zones = BTreeMap::new();
     for (sled_id, sled_config) in sled_configs {
-        for zone in &sled_config.zones {
-            zones_in_service.insert(zone.id);
-        }
-        let zones_config = sled_agent_client::types::OmicronZonesConfig::from(
-            OmicronZonesConfig {
-                // This is a bit of a hack. We only construct a blueprint after
-                // completing RSS, so we need to know the final generation value
-                // sent to all sleds. Arguably, we should record this in the
-                // serialized RSS plan; however, we have already deployed
-                // systems that did not. We know that every such system used
-                // `V5_EVERYTHING` as the final generation count, so we can just
-                // use that value here. If we ever change this, in particular in
-                // a way where newly-deployed systems will have a different
-                // value, we will need to revisit storing this in the serialized
-                // RSS plan.
-                generation: DeployStepVersion::V5_EVERYTHING,
-                zones: sled_config.zones,
-            },
-        );
-        omicron_zones.insert(sled_id, zones_config);
+        let zones_config = BlueprintZonesConfig {
+            // This is a bit of a hack. We only construct a blueprint after
+            // completing RSS, so we need to know the final generation value
+            // sent to all sleds. Arguably, we should record this in the
+            // serialized RSS plan; however, we have already deployed
+            // systems that did not. We know that every such system used
+            // `V5_EVERYTHING` as the final generation count, so we can just
+            // use that value here. If we ever change this, in particular in
+            // a way where newly-deployed systems will have a different
+            // value, we will need to revisit storing this in the serialized
+            // RSS plan.
+            generation: DeployStepVersion::V5_EVERYTHING,
+            zones: sled_config
+                .zones
+                .into_iter()
+                .map(|z| BlueprintZoneConfig {
+                    config: z.into(),
+                    // All initial zones are in-service.
+                    disposition: BlueprintZoneDisposition::InService,
+                })
+                .collect(),
+        };
+
+        blueprint_zones.insert(sled_id, zones_config);
     }
 
     Blueprint {
         id: Uuid::new_v4(),
-        omicron_zones,
-        zones_in_service,
+        blueprint_zones,
         parent_blueprint_id: None,
         internal_dns_version,
         // We don't configure external DNS during RSS, so set it to an initial

From ae980a1c7e82ea7f6ef537fbef849813151f8d8c Mon Sep 17 00:00:00 2001
From: John Gallagher <john@oxidecomputer.com>
Date: Mon, 25 Mar 2024 12:38:53 -0400
Subject: [PATCH 152/157] Move sled-agent contract-reaper thread creation
 earlier (#5322)

This should ensure the thread before we have any chance to interact with
zones:

1. `main` calls
[`BootstrapServer::start()`](https://github.com/oxidecomputer/omicron/blob/2349feb36d39804e38539e134a9a3a4462e7d516/sled-agent/src/bin/sled-agent.rs#L91-L93)
2. The first thing `BootstrapServer::start()` does is call
[`BoostrapAgentStartup::run()`](https://github.com/oxidecomputer/omicron/blob/2349feb36d39804e38539e134a9a3a4462e7d516/sled-agent/src/bootstrap/server.rs#L169-L183)
3. As of this PR, `BootstrapAgentStartup::run()` creates the reaper
thread immediately after building the original sled-agent `Logger`

Fixes #5319.
---
 sled-agent/src/bootstrap/pre_server.rs | 9 +++++++++
 sled-agent/src/sled_agent.rs           | 5 -----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/sled-agent/src/bootstrap/pre_server.rs b/sled-agent/src/bootstrap/pre_server.rs
index 394cde7ac2..38bedf921c 100644
--- a/sled-agent/src/bootstrap/pre_server.rs
+++ b/sled-agent/src/bootstrap/pre_server.rs
@@ -61,6 +61,15 @@ impl BootstrapAgentStartup {
     pub(super) async fn run(config: Config) -> Result<Self, StartError> {
         let base_log = build_logger(&config)?;
 
+        // Ensure we have a thread that automatically reaps process contracts
+        // when they become empty. See the comments in
+        // illumos-utils/src/running_zone.rs for more detail.
+        //
+        // We're going to start monitoring for hardware below, which could
+        // trigger launching the switch zone, and we need the contract reaper to
+        // exist before entering any zones.
+        illumos_utils::running_zone::ensure_contract_reaper(&base_log);
+
         let log = base_log.new(o!("component" => "BootstrapAgentStartup"));
 
         // Perform several blocking startup tasks first; we move `config` and
diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs
index aa051e6e5b..cbda32bbe1 100644
--- a/sled-agent/src/sled_agent.rs
+++ b/sled-agent/src/sled_agent.rs
@@ -383,11 +383,6 @@ impl SledAgent {
         info!(log, "Mounting backing filesystems");
         crate::backing_fs::ensure_backing_fs(&parent_log, &boot_disk.1)?;
 
-        // Ensure we have a thread that automatically reaps process contracts
-        // when they become empty. See the comments in
-        // illumos-utils/src/running_zone.rs for more detail.
-        illumos_utils::running_zone::ensure_contract_reaper(&parent_log);
-
         // TODO-correctness Bootstrap-agent already ensures the underlay
         // etherstub and etherstub VNIC exist on startup - could it pass them
         // through to us?

From 0ad5b9a38faf1511b06193fbc1d3e9f47ce17627 Mon Sep 17 00:00:00 2001
From: iliana etaoin <iliana@oxide.computer>
Date: Mon, 25 Mar 2024 09:44:12 -0700
Subject: [PATCH 153/157] stop building rot-prod-rel/rot-staging-dev repos
 (#5316)

---
 .github/buildomat/jobs/tuf-repo.sh | 140 ++++++-----------------------
 1 file changed, 28 insertions(+), 112 deletions(-)

diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh
index 9c70dce2ac..56cb41b51b 100755
--- a/.github/buildomat/jobs/tuf-repo.sh
+++ b/.github/buildomat/jobs/tuf-repo.sh
@@ -36,36 +36,6 @@
 #: name = "repo.zip.sha256.txt"
 #: from_output = "/work/repo-rot-all.zip.sha256.txt"
 #:
-#: [[publish]]
-#: series = "rot-prod-rel"
-#: name = "repo.zip.parta"
-#: from_output = "/work/repo-rot-prod-rel.zip.parta"
-#:
-#: [[publish]]
-#: series = "rot-prod-rel"
-#: name = "repo.zip.partb"
-#: from_output = "/work/repo-rot-prod-rel.zip.partb"
-#:
-#: [[publish]]
-#: series = "rot-prod-rel"
-#: name = "repo.zip.sha256.txt"
-#: from_output = "/work/repo-rot-prod-rel.zip.sha256.txt"
-#:
-#: [[publish]]
-#: series = "rot-staging-dev"
-#: name = "repo.zip.parta"
-#: from_output = "/work/repo-rot-staging-dev.zip.parta"
-#:
-#: [[publish]]
-#: series = "rot-staging-dev"
-#: name = "repo.zip.partb"
-#: from_output = "/work/repo-rot-staging-dev.zip.partb"
-#:
-#: [[publish]]
-#: series = "rot-staging-dev"
-#: name = "repo.zip.sha256.txt"
-#: from_output = "/work/repo-rot-staging-dev.zip.sha256.txt"
-#:
 
 set -o errexit
 set -o pipefail
@@ -156,7 +126,7 @@ for tag in "${TAGS[@]}"; do
 done
 popd
 
-# Fetch ROT images from dvt-dock.
+# Fetch signed ROT images from dvt-dock.
 source "$TOP/tools/dvt_dock_version"
 git init /work/dvt-dock
 (
@@ -179,28 +149,15 @@ caboose_util_rot() {
     echo "$output_a"
 }
 
-SERIES_LIST=()
-
-# Create an initial `manifest-rot-all.toml` containing the SP images for all
-# boards. While we still need to build multiple TUF repos,
-# `add_hubris_artifacts` below will append RoT images to this manifest (in
-# addition to the single-RoT manifest it creates).
-prep_rot_all_series() {
-    series="rot-all"
-
-    SERIES_LIST+=("$series")
-
-    manifest=/work/manifest-$series.toml
-    cp /work/manifest.toml "$manifest"
+# Add the SP images.
+for board_rev in "${ALL_BOARDS[@]}"; do
+    board=${board_rev%-?}
+    tufaceous_board=${board//sidecar/switch}
+    sp_image="/work/hubris/${board_rev}.zip"
+    sp_caboose_version=$(/work/caboose-util read-version "$sp_image")
+    sp_caboose_board=$(/work/caboose-util read-board "$sp_image")
 
-    for board_rev in "${ALL_BOARDS[@]}"; do
-        board=${board_rev%-?}
-        tufaceous_board=${board//sidecar/switch}
-        sp_image="/work/hubris/${board_rev}.zip"
-        sp_caboose_version=$(/work/caboose-util read-version "$sp_image")
-        sp_caboose_board=$(/work/caboose-util read-board "$sp_image")
-
-        cat >>"$manifest" <<EOF
+    cat >>/work/manifest.toml <<EOF
 [[artifact.${tufaceous_board}_sp]]
 name = "$sp_caboose_board"
 version = "$sp_caboose_version"
@@ -208,21 +165,12 @@ version = "$sp_caboose_version"
 kind = "file"
 path = "$sp_image"
 EOF
-    done
-}
-prep_rot_all_series
+done
 
+# Add the ROT images.
 add_hubris_artifacts() {
-    series="$1"
-    rot_dir="$2"
-    rot_version="$3"
-    shift 3
-
-    SERIES_LIST+=("$series")
-
-    manifest=/work/manifest-$series.toml
-    manifest_rot_all=/work/manifest-rot-all.toml
-    cp /work/manifest.toml "$manifest"
+    rot_dir="$1"
+    rot_version="$2"
 
     for board in gimlet psc sidecar; do
         tufaceous_board=${board//sidecar/switch}
@@ -231,21 +179,7 @@ add_hubris_artifacts() {
         rot_caboose_version=$(caboose_util_rot read-version "$rot_image_a" "$rot_image_b")
         rot_caboose_board=$(caboose_util_rot read-board "$rot_image_a" "$rot_image_b")
 
-        cat >>"$manifest" <<EOF
-[[artifact.${tufaceous_board}_rot]]
-name = "$rot_caboose_board"
-version = "$rot_caboose_version"
-[artifact.${tufaceous_board}_rot.source]
-kind = "composite-rot"
-[artifact.${tufaceous_board}_rot.source.archive_a]
-kind = "file"
-path = "$rot_image_a"
-[artifact.${tufaceous_board}_rot.source.archive_b]
-kind = "file"
-path = "$rot_image_b"
-EOF
-
-        cat >>"$manifest_rot_all" <<EOF
+        cat >>/work/manifest.toml <<EOF
 [[artifact.${tufaceous_board}_rot]]
 name = "$rot_caboose_board-${rot_dir//\//-}"
 version = "$rot_caboose_version"
@@ -257,40 +191,22 @@ path = "$rot_image_a"
 [artifact.${tufaceous_board}_rot.source.archive_b]
 kind = "file"
 path = "$rot_image_b"
-EOF
-    done
-
-    for board_rev in "$@"; do
-        board=${board_rev%-?}
-        tufaceous_board=${board//sidecar/switch}
-        sp_image="/work/hubris/${board_rev}.zip"
-        sp_caboose_version=$(/work/caboose-util read-version "$sp_image")
-        sp_caboose_board=$(/work/caboose-util read-board "$sp_image")
-
-        cat >>"$manifest" <<EOF
-[[artifact.${tufaceous_board}_sp]]
-name = "$sp_caboose_board"
-version = "$sp_caboose_version"
-[artifact.${tufaceous_board}_sp.source]
-kind = "file"
-path = "$sp_image"
 EOF
     done
 }
-# usage:              SERIES           ROT_DIR      ROT_VERSION              BOARDS...
-add_hubris_artifacts  rot-staging-dev  staging/dev  cert-staging-dev-v1.0.7  "${ALL_BOARDS[@]}"
-add_hubris_artifacts  rot-prod-rel     prod/rel     cert-prod-rel-v1.0.7     "${ALL_BOARDS[@]}"
+# usage:              ROT_DIR      ROT_VERSION
+add_hubris_artifacts  staging/dev  cert-staging-dev-v1.0.7
+add_hubris_artifacts  prod/rel     cert-prod-rel-v1.0.7
 
-for series in "${SERIES_LIST[@]}"; do
-    /work/tufaceous assemble --no-generate-key /work/manifest-"$series".toml /work/repo-"$series".zip
-    digest -a sha256 /work/repo-"$series".zip > /work/repo-"$series".zip.sha256.txt
+# Build the TUF ZIP.
+/work/tufaceous assemble --no-generate-key /work/manifest.toml /work/repo-rot-all.zip
+digest -a sha256 /work/repo-rot-all.zip > /work/repo-rot-all.zip.sha256.txt
 
-    #
-    # XXX: There are some issues downloading Buildomat artifacts > 1 GiB, see
-    # oxidecomputer/buildomat#36.
-    #
-    split -a 1 -b 1024m /work/repo-"$series".zip /work/repo-"$series".zip.part
-    rm /work/repo-"$series".zip
-    # Ensure the build doesn't fail if the repo gets smaller than 1 GiB.
-    touch /work/repo-"$series".zip.partb
-done
+#
+# XXX: There are some issues downloading Buildomat artifacts > 1 GiB, see
+# oxidecomputer/buildomat#36.
+#
+split -a 1 -b 1024m /work/repo-rot-all.zip /work/repo-rot-all.zip.part
+rm /work/repo-rot-all.zip
+# Ensure the build doesn't fail if the repo gets smaller than 1 GiB.
+touch /work/repo-rot-all.zip.partb

From 1244d5918dabf8804f5ab5570ef6cee1d3c02b9f Mon Sep 17 00:00:00 2001
From: James MacMahon <james@oxide.computer>
Date: Mon, 25 Mar 2024 19:59:04 -0400
Subject: [PATCH 154/157] Update crucible (#5327)

Bump crucible rev to pick up:

- Return *410 Gone* if volume is inactive
- Update Rust crate opentelemetry to 0.22.0
- Update Rust crate base64 to 0.22.0
- Update Rust crate async-recursion to 1.1.0
- Minor cleanups to extent implementations
- Update Rust crate http to 0.2.12
- Update Rust crate reedline to 0.30.0
- Update Rust crate rayon to 1.9.0
- Update Rust crate nix to 0.28
- Update Rust crate async-trait to 0.1.78
- Various buffer optimizations
- Add low-level test for message encoding
- Don't let df failures ruin the buildomat tests
- Activate the NBD server's psuedo file
---
 Cargo.lock            | 6 +++---
 Cargo.toml            | 6 +++---
 package-manifest.toml | 8 ++++----
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 16e684d422..a93912c9a3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1335,7 +1335,7 @@ dependencies = [
 [[package]]
 name = "crucible-agent-client"
 version = "0.0.1"
-source = "git+https://github.com/oxidecomputer/crucible?rev=952c7d60d22be5198b892bec8d92f4291b9160c2#952c7d60d22be5198b892bec8d92f4291b9160c2"
+source = "git+https://github.com/oxidecomputer/crucible?rev=16f16478f4af1502b25ddcd79d307b3f116f13f6#16f16478f4af1502b25ddcd79d307b3f116f13f6"
 dependencies = [
  "anyhow",
  "chrono",
@@ -1351,7 +1351,7 @@ dependencies = [
 [[package]]
 name = "crucible-pantry-client"
 version = "0.0.1"
-source = "git+https://github.com/oxidecomputer/crucible?rev=952c7d60d22be5198b892bec8d92f4291b9160c2#952c7d60d22be5198b892bec8d92f4291b9160c2"
+source = "git+https://github.com/oxidecomputer/crucible?rev=16f16478f4af1502b25ddcd79d307b3f116f13f6#16f16478f4af1502b25ddcd79d307b3f116f13f6"
 dependencies = [
  "anyhow",
  "chrono",
@@ -1368,7 +1368,7 @@ dependencies = [
 [[package]]
 name = "crucible-smf"
 version = "0.0.0"
-source = "git+https://github.com/oxidecomputer/crucible?rev=952c7d60d22be5198b892bec8d92f4291b9160c2#952c7d60d22be5198b892bec8d92f4291b9160c2"
+source = "git+https://github.com/oxidecomputer/crucible?rev=16f16478f4af1502b25ddcd79d307b3f116f13f6#16f16478f4af1502b25ddcd79d307b3f116f13f6"
 dependencies = [
  "crucible-workspace-hack",
  "libc",
diff --git a/Cargo.toml b/Cargo.toml
index a8fa57d5d6..6e5b23b6ac 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -192,9 +192,9 @@ cookie = "0.18"
 criterion = { version = "0.5.1", features = [ "async_tokio" ] }
 crossbeam = "0.8"
 crossterm = { version = "0.27.0", features = ["event-stream"] }
-crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "952c7d60d22be5198b892bec8d92f4291b9160c2" }
-crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "952c7d60d22be5198b892bec8d92f4291b9160c2" }
-crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "952c7d60d22be5198b892bec8d92f4291b9160c2" }
+crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "16f16478f4af1502b25ddcd79d307b3f116f13f6" }
+crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "16f16478f4af1502b25ddcd79d307b3f116f13f6" }
+crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "16f16478f4af1502b25ddcd79d307b3f116f13f6" }
 csv = "1.3.0"
 curve25519-dalek = "4"
 datatest-stable = "0.2.3"
diff --git a/package-manifest.toml b/package-manifest.toml
index 18cde0368b..e3198cef8b 100644
--- a/package-manifest.toml
+++ b/package-manifest.toml
@@ -492,10 +492,10 @@ only_for_targets.image = "standard"
 # 3. Use source.type = "manual" instead of "prebuilt"
 source.type = "prebuilt"
 source.repo = "crucible"
-source.commit = "952c7d60d22be5198b892bec8d92f4291b9160c2"
+source.commit = "16f16478f4af1502b25ddcd79d307b3f116f13f6"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image/<commit>/crucible.sha256.txt
-source.sha256 = "af200d930e6d658bdd1688a032747da7c2bafa0e6ab1b53bda7b7f571d8a19c6"
+source.sha256 = "ce186a1a1243ea618755ae341844795cff0ce2c1415c6bd770360b3330dc664b"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -504,10 +504,10 @@ service_name = "crucible_pantry_prebuilt"
 only_for_targets.image = "standard"
 source.type = "prebuilt"
 source.repo = "crucible"
-source.commit = "952c7d60d22be5198b892bec8d92f4291b9160c2"
+source.commit = "16f16478f4af1502b25ddcd79d307b3f116f13f6"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image/<commit>/crucible-pantry.sha256.txt
-source.sha256 = "1a40e24b176e8179cc7f74ef6922cd2ee51d5eca9bfa7057fc37994cf5c4678e"
+source.sha256 = "8920622255fa0317ce312d0127c94b8fef647a85be4c8abaf861be560fe43194"
 output.type = "zone"
 output.intermediate_only = true
 

From a17750d8c959682ac3da5fc138503d063fa00851 Mon Sep 17 00:00:00 2001
From: "Andrew J. Stone" <andrew@oxidecomputer.com>
Date: Tue, 26 Mar 2024 10:57:27 -0400
Subject: [PATCH 155/157] Add a target runner for running tests in VMs (#5285)

This PR allows us to run tests inside falcon VMs via nextest. Due to the
lack of [nextest per-test target
runners](https://github.com/nextest-rs/nextest/issues/1358), we enable
the target runner via environment variable and run a specific test at a
time.

In order to run a test, do the following, where the trailing
`--nocapture` is optional but encouraged at this moment.

```
cd omicron
CARGO_TARGET_X86_64_UNKNOWN_ILLUMOS_RUNNER=./test-utils/src/bin/falcon_runner.sh cargo nextest run -p <package> <test name> --nocapture
```


An example:
```
CARGO_TARGET_X86_64_UNKNOWN_ILLUMOS_RUNNER=./test-utils/src/bin/falcon_runner.sh cargo nextest run -p omicron-test-utils launch --nocapture
```

A few things remain to be done, although this is safe to merge without
adding them in this PR. See `test-utils/README.md` for details.
---
 .gitignore                              |   2 +
 Cargo.lock                              | 376 ++++++++++++++++++++++--
 Cargo.toml                              |   3 +-
 README.adoc                             |   2 +
 test-utils/Cargo.toml                   |  10 +
 test-utils/README.md                    | 110 +++++++
 test-utils/src/bin/falcon_runner.rs     | 103 +++++++
 test-utils/src/bin/falcon_runner.sh     |  11 +
 test-utils/src/bin/falcon_runner_cli.rs |  25 ++
 test-utils/src/dev/falcon.rs            |  22 ++
 test-utils/src/dev/mod.rs               |   1 +
 11 files changed, 645 insertions(+), 20 deletions(-)
 create mode 100644 test-utils/README.md
 create mode 100644 test-utils/src/bin/falcon_runner.rs
 create mode 100755 test-utils/src/bin/falcon_runner.sh
 create mode 100644 test-utils/src/bin/falcon_runner_cli.rs
 create mode 100644 test-utils/src/dev/falcon.rs

diff --git a/.gitignore b/.gitignore
index a13536aab7..39a8f36144 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,4 +14,6 @@ rusty-tags.vi
 *.sw*
 tags
 .direnv
+.falcon/*
+.img/*
 connectivity-report.json
diff --git a/Cargo.lock b/Cargo.lock
index a93912c9a3..f7b9f27b1f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -126,9 +126,9 @@ dependencies = [
 
 [[package]]
 name = "anstyle"
-version = "1.0.2"
+version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea"
+checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc"
 
 [[package]]
 name = "anstyle-parse"
@@ -225,6 +225,12 @@ version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
 
+[[package]]
+name = "ascii"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16"
+
 [[package]]
 name = "ascii-canvas"
 version = "3.0.0"
@@ -405,6 +411,12 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf"
 
+[[package]]
+name = "base64"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
+
 [[package]]
 name = "base64"
 version = "0.21.7"
@@ -466,12 +478,31 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "bhyve_api"
+version = "0.0.0"
+source = "git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af#6dceb9ef69c217cb78a2018bbedafbc19f6ec1af"
+dependencies = [
+ "bhyve_api_sys 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af)",
+ "libc",
+ "strum 0.26.1",
+]
+
 [[package]]
 name = "bhyve_api"
 version = "0.0.0"
 source = "git+https://github.com/oxidecomputer/propolis?rev=fdf0585c6a227a7cfbee4a61a36938c3d77e4712#fdf0585c6a227a7cfbee4a61a36938c3d77e4712"
 dependencies = [
- "bhyve_api_sys",
+ "bhyve_api_sys 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=fdf0585c6a227a7cfbee4a61a36938c3d77e4712)",
+ "libc",
+ "strum 0.26.1",
+]
+
+[[package]]
+name = "bhyve_api_sys"
+version = "0.0.0"
+source = "git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af#6dceb9ef69c217cb78a2018bbedafbc19f6ec1af"
+dependencies = [
  "libc",
  "strum 0.26.1",
 ]
@@ -544,6 +575,26 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "bitstruct"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1b10c3912af09af44ea1dafe307edb5ed374b2a32658eb610e372270c9017b4"
+dependencies = [
+ "bitstruct_derive",
+]
+
+[[package]]
+name = "bitstruct_derive"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35fd19022c2b750d14eb9724c204d08ab7544570105b3b466d8a9f2f3feded27"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "bitvec"
 version = "1.0.1"
@@ -1183,6 +1234,18 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "cpuid_profile_config"
+version = "0.0.0"
+source = "git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af#6dceb9ef69c217cb78a2018bbedafbc19f6ec1af"
+dependencies = [
+ "propolis",
+ "serde",
+ "serde_derive",
+ "thiserror",
+ "toml 0.7.8",
+]
+
 [[package]]
 name = "crc"
 version = "3.0.1"
@@ -1882,6 +1945,15 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f77af9e75578c1ab34f5f04545a8b05be0c36fbd7a9bb3cf2d2a971e435fdbb9"
 
+[[package]]
+name = "dladm"
+version = "0.0.0"
+source = "git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af#6dceb9ef69c217cb78a2018bbedafbc19f6ec1af"
+dependencies = [
+ "libc",
+ "strum 0.26.1",
+]
+
 [[package]]
 name = "dlpi"
 version = "0.2.0"
@@ -2253,6 +2325,19 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "env_logger"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7"
+dependencies = [
+ "atty",
+ "humantime",
+ "log",
+ "regex",
+ "termcolor",
+]
+
 [[package]]
 name = "env_logger"
 version = "0.10.0"
@@ -2270,6 +2355,15 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 
+[[package]]
+name = "erased-serde"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "errno"
 version = "0.3.8"
@@ -2631,7 +2725,7 @@ dependencies = [
  "slog-term",
  "termios",
  "tokio",
- "tokio-tungstenite",
+ "tokio-tungstenite 0.20.1",
  "uuid 1.7.0",
 ]
 
@@ -3424,7 +3518,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "async-trait",
- "bhyve_api",
+ "bhyve_api 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=fdf0585c6a227a7cfbee4a61a36938c3d77e4712)",
  "byteorder",
  "camino",
  "camino-tempfile",
@@ -3450,7 +3544,7 @@ dependencies = [
  "tokio",
  "toml 0.8.10",
  "uuid 1.7.0",
- "zone",
+ "zone 0.3.0",
 ]
 
 [[package]]
@@ -3907,6 +4001,41 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b024e211b1b371da58cd69e4fb8fa4ed16915edcc0e2e1fb04ac4bad61959f25"
 
+[[package]]
+name = "libfalcon"
+version = "0.1.0"
+source = "git+https://github.com/oxidecomputer/falcon?rev=e69694a1f7cc9fe31fab27f321017280531fb5f7#e69694a1f7cc9fe31fab27f321017280531fb5f7"
+dependencies = [
+ "anstyle",
+ "anyhow",
+ "camino",
+ "clap 4.5.1",
+ "colored",
+ "futures",
+ "libc",
+ "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys?branch=main)",
+ "portpicker",
+ "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af)",
+ "propolis-server-config",
+ "rand 0.8.5",
+ "regex",
+ "reqwest",
+ "ron 0.7.1",
+ "serde",
+ "slog",
+ "slog-async",
+ "slog-envlogger",
+ "slog-term",
+ "smf",
+ "tabwriter",
+ "thiserror",
+ "tokio",
+ "tokio-tungstenite 0.21.0",
+ "toml 0.7.8",
+ "uuid 1.7.0",
+ "zone 0.1.8",
+]
+
 [[package]]
 name = "libloading"
 version = "0.7.4"
@@ -3923,6 +4052,25 @@ version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4"
 
+[[package]]
+name = "libnet"
+version = "0.1.0"
+source = "git+https://github.com/oxidecomputer/netadm-sys?branch=main#d44d9e084f39e844f8083d4d9b39a331061ebbcc"
+dependencies = [
+ "anyhow",
+ "cfg-if",
+ "colored",
+ "dlpi",
+ "libc",
+ "num_enum 0.5.11",
+ "nvpair",
+ "nvpair-sys",
+ "rusty-doors",
+ "socket2 0.4.9",
+ "thiserror",
+ "tracing",
+]
+
 [[package]]
 name = "libnet"
 version = "0.1.0"
@@ -4041,7 +4189,7 @@ dependencies = [
  "const-oid",
  "crc-any",
  "der",
- "env_logger",
+ "env_logger 0.10.0",
  "hex",
  "log",
  "lpc55_areas",
@@ -5168,7 +5316,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "tokio-stream",
- "tokio-tungstenite",
+ "tokio-tungstenite 0.20.1",
  "toml 0.8.10",
  "uuid 1.7.0",
 ]
@@ -5255,7 +5403,7 @@ dependencies = [
  "pq-sys",
  "pretty_assertions",
  "progenitor-client",
- "propolis-client",
+ "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=fdf0585c6a227a7cfbee4a61a36938c3d77e4712)",
  "rand 0.8.5",
  "rcgen",
  "ref-cast",
@@ -5467,7 +5615,7 @@ dependencies = [
  "oximeter-instruments",
  "oximeter-producer",
  "pretty_assertions",
- "propolis-client",
+ "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=fdf0585c6a227a7cfbee4a61a36938c3d77e4712)",
  "propolis-mock-server",
  "rand 0.8.5",
  "rcgen",
@@ -5502,7 +5650,7 @@ dependencies = [
  "usdt 0.5.0",
  "uuid 1.7.0",
  "zeroize",
- "zone",
+ "zone 0.3.0",
 ]
 
 [[package]]
@@ -5516,10 +5664,12 @@ dependencies = [
  "dropshot",
  "expectorate",
  "filetime",
+ "gethostname",
  "headers",
  "hex",
  "http 0.2.12",
  "libc",
+ "libfalcon",
  "nexus-config",
  "omicron-common",
  "omicron-workspace-hack",
@@ -5818,7 +5968,7 @@ version = "0.1.0"
 source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce"
 dependencies = [
  "libc",
- "libnet",
+ "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)",
  "opte",
  "oxide-vpc",
  "postcard",
@@ -6563,6 +6713,15 @@ version = "1.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b"
 
+[[package]]
+name = "portpicker"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be97d76faf1bfab666e1375477b23fde79eccf0276e9b63b92a39d676a889ba9"
+dependencies = [
+ "rand 0.8.5",
+]
+
 [[package]]
 name = "postcard"
 version = "1.0.8"
@@ -6811,6 +6970,57 @@ dependencies = [
  "syn 2.0.52",
 ]
 
+[[package]]
+name = "propolis"
+version = "0.1.0"
+source = "git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af#6dceb9ef69c217cb78a2018bbedafbc19f6ec1af"
+dependencies = [
+ "anyhow",
+ "bhyve_api 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af)",
+ "bitflags 2.4.2",
+ "bitstruct",
+ "byteorder",
+ "dladm",
+ "erased-serde",
+ "futures",
+ "lazy_static",
+ "libc",
+ "pin-project-lite",
+ "propolis_types 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af)",
+ "rfb",
+ "serde",
+ "serde_arrays",
+ "serde_json",
+ "slog",
+ "strum 0.26.1",
+ "thiserror",
+ "tokio",
+ "usdt 0.5.0",
+ "uuid 1.7.0",
+ "viona_api",
+]
+
+[[package]]
+name = "propolis-client"
+version = "0.1.0"
+source = "git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af#6dceb9ef69c217cb78a2018bbedafbc19f6ec1af"
+dependencies = [
+ "async-trait",
+ "base64 0.21.7",
+ "futures",
+ "progenitor",
+ "rand 0.8.5",
+ "reqwest",
+ "schemars",
+ "serde",
+ "serde_json",
+ "slog",
+ "thiserror",
+ "tokio",
+ "tokio-tungstenite 0.20.1",
+ "uuid 1.7.0",
+]
+
 [[package]]
 name = "propolis-client"
 version = "0.1.0"
@@ -6828,7 +7038,7 @@ dependencies = [
  "slog",
  "thiserror",
  "tokio",
- "tokio-tungstenite",
+ "tokio-tungstenite 0.20.1",
  "uuid 1.7.0",
 ]
 
@@ -6845,7 +7055,7 @@ dependencies = [
  "futures",
  "hyper 0.14.28",
  "progenitor",
- "propolis_types",
+ "propolis_types 0.0.0 (git+https://github.com/oxidecomputer/propolis?rev=fdf0585c6a227a7cfbee4a61a36938c3d77e4712)",
  "rand 0.8.5",
  "reqwest",
  "schemars",
@@ -6858,10 +7068,31 @@ dependencies = [
  "slog-term",
  "thiserror",
  "tokio",
- "tokio-tungstenite",
+ "tokio-tungstenite 0.20.1",
  "uuid 1.7.0",
 ]
 
+[[package]]
+name = "propolis-server-config"
+version = "0.0.0"
+source = "git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af#6dceb9ef69c217cb78a2018bbedafbc19f6ec1af"
+dependencies = [
+ "cpuid_profile_config",
+ "serde",
+ "serde_derive",
+ "thiserror",
+ "toml 0.7.8",
+]
+
+[[package]]
+name = "propolis_types"
+version = "0.0.0"
+source = "git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af#6dceb9ef69c217cb78a2018bbedafbc19f6ec1af"
+dependencies = [
+ "schemars",
+ "serde",
+]
+
 [[package]]
 name = "propolis_types"
 version = "0.0.0"
@@ -7368,6 +7599,21 @@ dependencies = [
  "quick-error",
 ]
 
+[[package]]
+name = "rfb"
+version = "0.1.0"
+source = "git+https://github.com/oxidecomputer/rfb?rev=0cac8d9c25eb27acfa35df80f3b9d371de98ab3b#0cac8d9c25eb27acfa35df80f3b9d371de98ab3b"
+dependencies = [
+ "ascii",
+ "async-trait",
+ "bitflags 1.3.2",
+ "env_logger 0.9.3",
+ "futures",
+ "log",
+ "thiserror",
+ "tokio",
+]
+
 [[package]]
 name = "rfc6979"
 version = "0.4.0"
@@ -7408,6 +7654,17 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "ron"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a"
+dependencies = [
+ "base64 0.13.1",
+ "bitflags 1.3.2",
+ "serde",
+]
+
 [[package]]
 name = "ron"
 version = "0.8.1"
@@ -8040,6 +8297,15 @@ dependencies = [
  "smallvec 0.6.14",
 ]
 
+[[package]]
+name = "serde_arrays"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38636132857f68ec3d5f3eb121166d2af33cb55174c4d5ff645db6165cbef0fd"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_derive"
 version = "1.0.197"
@@ -9418,7 +9684,7 @@ name = "tlvc-text"
 version = "0.3.0"
 source = "git+https://github.com/oxidecomputer/tlvc.git#e644a21a7ca973ed31499106ea926bd63ebccc6f"
 dependencies = [
- "ron",
+ "ron 0.8.1",
  "serde",
  "tlvc 0.3.1 (git+https://github.com/oxidecomputer/tlvc.git)",
  "zerocopy 0.6.4",
@@ -9543,7 +9809,19 @@ dependencies = [
  "futures-util",
  "log",
  "tokio",
- "tungstenite",
+ "tungstenite 0.20.1",
+]
+
+[[package]]
+name = "tokio-tungstenite"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38"
+dependencies = [
+ "futures-util",
+ "log",
+ "tokio",
+ "tungstenite 0.21.0",
 ]
 
 [[package]]
@@ -9941,6 +10219,25 @@ dependencies = [
  "utf-8",
 ]
 
+[[package]]
+name = "tungstenite"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1"
+dependencies = [
+ "byteorder",
+ "bytes",
+ "data-encoding",
+ "http 1.0.0",
+ "httparse",
+ "log",
+ "rand 0.8.5",
+ "sha1",
+ "thiserror",
+ "url",
+ "utf-8",
+]
+
 [[package]]
 name = "typed-path"
 version = "0.7.0"
@@ -10342,6 +10639,23 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
+[[package]]
+name = "viona_api"
+version = "0.0.0"
+source = "git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af#6dceb9ef69c217cb78a2018bbedafbc19f6ec1af"
+dependencies = [
+ "libc",
+ "viona_api_sys",
+]
+
+[[package]]
+name = "viona_api_sys"
+version = "0.0.0"
+source = "git+https://github.com/oxidecomputer/propolis?rev=6dceb9ef69c217cb78a2018bbedafbc19f6ec1af#6dceb9ef69c217cb78a2018bbedafbc19f6ec1af"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "vsss-rs"
 version = "3.3.4"
@@ -11093,6 +11407,17 @@ dependencies = [
  "flate2",
 ]
 
+[[package]]
+name = "zone"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3596bbc963cd9dbaa69b02e349af4d061c56c41d211ba64150a2cedb2f722707"
+dependencies = [
+ "itertools 0.10.5",
+ "thiserror",
+ "zone_cfg_derive 0.1.2",
+]
+
 [[package]]
 name = "zone"
 version = "0.3.0"
@@ -11101,7 +11426,7 @@ dependencies = [
  "itertools 0.10.5",
  "thiserror",
  "tokio",
- "zone_cfg_derive",
+ "zone_cfg_derive 0.3.0",
 ]
 
 [[package]]
@@ -11116,7 +11441,20 @@ dependencies = [
  "omicron-workspace-hack",
  "slog",
  "tokio",
- "zone",
+ "zone 0.3.0",
+]
+
+[[package]]
+name = "zone_cfg_derive"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61ac2a898023d86613a7efa7a4195e4b75240009e559acb17ddd7fa876d19527"
+dependencies = [
+ "heck 0.3.3",
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 6e5b23b6ac..98a397978d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -160,7 +160,7 @@ resolver = "2"
 
 [workspace.dependencies]
 anyhow = "1.0"
-anstyle = "1.0"
+anstyle = "1.0.6"
 api_identity = { path = "api_identity" }
 approx = "0.5.1"
 assert_matches = "1.5.0"
@@ -258,6 +258,7 @@ ispf = { git = "https://github.com/oxidecomputer/ispf" }
 key-manager = { path = "key-manager" }
 kstat-rs = "0.2.3"
 libc = "0.2.153"
+libfalcon = { git = "https://github.com/oxidecomputer/falcon", rev = "e69694a1f7cc9fe31fab27f321017280531fb5f7" }
 libnvme = { git = "https://github.com/oxidecomputer/libnvme", rev = "6fffcc81d2c423ed2d2e6c5c2827485554c4ecbe" }
 linear-map = "1.2.0"
 macaddr = { version = "1.0.1", features = ["serde_std"] }
diff --git a/README.adoc b/README.adoc
index 04bc33b2af..0e09fc39df 100644
--- a/README.adoc
+++ b/README.adoc
@@ -60,6 +60,8 @@ $ cargo nextest run
 
 Nextest https://github.com/nextest-rs/nextest/issues/16[does not support doctests]. Run doctests separately with `cargo test --doc`.
 
+Similarly, you can run tests inside a https://github.com/oxidecomputer/falcon[Falcon] based VM. This is described in the `test-utils` https://github.com/oxidecomputer/omicron/tree/main/test-utils[README].
+
 === rustfmt and clippy
 
 You can **format the code** using `cargo fmt`.  Make sure to run this before pushing changes.  The CI checks that the code is correctly formatted.
diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml
index d60f4f55e3..56e3be2efd 100644
--- a/test-utils/Cargo.toml
+++ b/test-utils/Cargo.toml
@@ -37,6 +37,16 @@ uuid.workspace = true
 
 [dev-dependencies]
 expectorate.workspace = true
+gethostname.workspace = true
 
 [features]
 seed-gen = ["dep:filetime"]
+
+[target.'cfg(target_os = "illumos")'.dependencies]
+libfalcon.workspace = true
+
+[[bin]]
+name = "falcon_runner"
+
+[[bin]]
+name = "falcon_runner_cli"
diff --git a/test-utils/README.md b/test-utils/README.md
new file mode 100644
index 0000000000..d6ae4759c6
--- /dev/null
+++ b/test-utils/README.md
@@ -0,0 +1,110 @@
+# Falcon VM based testing
+
+It is now possible to run omicron tests in a VM. Any test can be run in a VM,
+but many of them will not work, because their dependencies are not installed.
+This is possible, and can be done in a number of ways. We just need to think
+about how we want to do it.
+
+Running an individual test in a VM currently works by:
+
+ 1. Building the test binary
+ 2. Copying it into a temporary directory
+ 3. Mounting that temporary directory inside the VM
+ 4. Logging into and running the binary with the given test name over the
+    serial port.
+
+## Running an Omicron test
+
+An individual test can be run in the following manner:
+
+```shell
+cd omicron
+CARGO_TARGET_X86_64_UNKNOWN_ILLUMOS_RUNNER=./test-utils/src/bin/falcon_runner.sh cargo nextest run -p <package> <test name> --nocapture
+```
+
+A concrete example, which can be used for sanity checking that the test actually runs
+in a VM, is provided in the `omicron-test-utils` package, and named `launch`.
+
+```shell
+CARGO_TARGET_X86_64_UNKNOWN_ILLUMOS_RUNNER=./test-utils/src/bin/falcon_runner.sh cargo nextest run -p omicron-test-utils launch --nocapture
+```
+
+## Logging into a test VM
+
+On failure, the test VM remains running for debugging purposes. All VMs
+currently share the name `launchpad_mcduck_test_vm`, although the falcon
+deployment metadata resides in a unique temp directory. At the start of each
+test, and upon failure this directory is printed to the console.
+
+```
+Setting falcon directory to /tmp/.tmpXcLx5g
+...
+Test failed: VM remains running, with falcon dir: /tmp/.tmpXcLx5g
+```
+
+A user can login to that VM via the serial port with the following command:
+
+```shell
+cargo run -p omicron-test-utils --bin falcon_runner_cli serial launchpad_mcduck_test_vm -f /tmp/.tmpXcLx5g
+```
+
+Similarly, a user can execute a command on the VM via:
+
+```shell
+cargo run -p omicron-test-utils --bin falcon_runner_cli exec launchpad_mcduck_test_vm '<CMD>' -f /tmp/.tmpXcLx5g
+```
+
+This can be used to cat log files, etc..
+
+Once a user is done debugging the VM, they should tear it and its associated
+resources down with the following command. Note that here, you don't have to
+name the VM, as the entire deployment is torn down. However, you do need to
+use `pfexec`.
+
+```shell
+pfexec cargo run -p omicron-test-utils --bin falcon_runner_cli destroy -f /tmp/.tmpXcLx5g
+```
+
+## Troubleshooting
+
+If for some reason you fail to tear down your VM and need to do it manually you
+must remove the VM and its backing filesystems. Don't worry about the falcon
+dirs, as they are just small directories in `/tmp`.
+
+VM backing images are created in zfs under `rpool/falcon/topo`. These can be
+destroyed via the following command. Note that this will destroy the backing
+filesystems for **all** of your falcon VMs. If you want to be more precise, you
+should `zfs list` and figure out what you want to kill.
+
+```shell
+pfexec zfs destroy -r rpool/falcon/topo
+```
+
+Similarly, all your bhyve backed VMs live in `/dev/vmm`. All VMs can be torn
+down with the following command, although you should be careful when doing this
+on shared infrastructure.
+
+```shell
+for i in `ls /dev/vmm`; do pfexec bhyvectl --vm=$i --destroy; done
+```
+
+For more troubleshooting info, see the [falcon wiki](https://github.com/oxidecomputer/falcon/wiki/Reference)
+
+## Current Limitations
+
+* Tests must be self-contained and not rely on dependencies. E2E tests will
+not  currently work. We can easily install pre-reqs via our scripts, but this
+will delay each test run. We may instead want to build falcon base images with
+the pre-reqs included. There's a tradeoff here of test run time, vs getting the
+latest bits.
+* The names of the test runner, and VM are currently hardcoded, and start with
+`launchpad_mcduck`. Therefore, tests can only be run serially for now. This is
+an easy limitation to lift, we just need to figure out a naming convention and
+do it.
+* The commands to run tests are quite verbose. We should really have a wrapper
+script.
+
+
+
+
+
diff --git a/test-utils/src/bin/falcon_runner.rs b/test-utils/src/bin/falcon_runner.rs
new file mode 100644
index 0000000000..d7ffd1d823
--- /dev/null
+++ b/test-utils/src/bin/falcon_runner.rs
@@ -0,0 +1,103 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Mechanisms to launch and control propolis VMs via falcon
+
+#[cfg(target_os = "illumos")]
+mod illumos {
+    pub use anyhow::anyhow;
+    pub use camino_tempfile::tempdir;
+    pub use libfalcon::{unit::gb, Runner};
+    pub use omicron_test_utils::dev::test_setup_log;
+    pub use slog::info;
+    pub use std::env;
+}
+
+#[cfg(target_os = "illumos")]
+#[tokio::main]
+async fn main() -> Result<(), anyhow::Error> {
+    use illumos::*;
+    let args: Vec<_> = env::args().collect();
+    let logctx = test_setup_log("falcon_test_runner");
+    let log = logctx.log;
+    info!(log, "args = {:?}", args);
+    let name = "launchpad_mcduck";
+
+    let runner_name = format!("{}_runner", name);
+    let node_name = format!("{}_test_vm", name);
+    let mut d = Runner::new(&runner_name);
+    let vm = d.node(&node_name, "helios-2.0", 2, gb(8));
+
+    let falcon_dir = tempdir()?;
+    info!(log, "Setting falcon directory to {}", falcon_dir.path());
+    d.falcon_dir = falcon_dir.path().into();
+
+    let cargo_bay = tempdir()?;
+    info!(log, "Set cargo-bay to {} on host machine", cargo_bay.path());
+    let source_test_path = &args[1];
+    let test_name = &args[3];
+    let test_file_name = camino::Utf8Path::new(source_test_path)
+        .file_name()
+        .expect("Failed to get test file name");
+    let test_path = cargo_bay.path().to_path_buf().join(test_file_name);
+
+    info!(log, "Copying {source_test_path} to {test_path}");
+    std::fs::copy(camino::Utf8Path::new(source_test_path), &test_path)?;
+
+    d.mount(cargo_bay.path(), "/opt/cargo-bay", vm)?;
+
+    info!(log, "Launching test vm {node_name}");
+    d.launch().await?;
+    info!(log, "Launched test vm {node_name}");
+
+    info!(log, "Running test: {test_file_name}::{test_name}");
+    let run_test = format!(
+        "cd /opt/cargo-bay && chmod +x {test_file_name}; \
+        res=$?; \
+        if [[ $res -eq 0 ]]; then \
+          ./{test_file_name} --color never --exact {test_name} --nocapture; \
+          res=$?; \
+        fi; \
+        echo $res"
+    );
+    let out = d.exec(vm, &run_test).await?;
+
+    // The last line of our output contains the exit code
+    let exit_code_index = out.rfind('\n').expect("No newline found in output");
+
+    // Ensure there is enough data left for an exit code
+    let exit_code: u8 = if exit_code_index + 1 < out.len() {
+        (&out[exit_code_index + 1..]).parse().expect("Invalid exit code")
+    } else {
+        panic!("No exit code available");
+    };
+
+    info!(log, "{}", &out[..=exit_code_index]);
+
+    if exit_code == 0u8 {
+        // We destroy here so that our tempdir doesn't get dropped first and
+        // give us an error.
+        //
+        // `drop` calls `d.destroy()` and also makes sure it doesn't run twice.
+        drop(d);
+        Ok(())
+    } else {
+        // Leave the VM running
+        d.persistent = true;
+
+        // Don't remove the falcon directory
+        info!(
+            log,
+            "Test failed: VM remains running, with falcon dir: {}",
+            falcon_dir.path()
+        );
+        std::mem::forget(falcon_dir);
+
+        Err(anyhow!("Test failed: exit code = {exit_code}"))
+    }
+}
+
+#[cfg(not(target_os = "illumos"))]
+#[tokio::main]
+async fn main() {}
diff --git a/test-utils/src/bin/falcon_runner.sh b/test-utils/src/bin/falcon_runner.sh
new file mode 100755
index 0000000000..32645a6aea
--- /dev/null
+++ b/test-utils/src/bin/falcon_runner.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Override on the command line for your path to Omicron
+OMICRON=${OMICRON:-~/omicron}
+
+if [[ "$*" == *--list* ]]; then
+  exec "$@"
+else
+   cargo build -p omicron-test-utils --bin falcon_runner
+   pfexec $OMICRON/target/debug/falcon_runner $@
+fi
diff --git a/test-utils/src/bin/falcon_runner_cli.rs b/test-utils/src/bin/falcon_runner_cli.rs
new file mode 100644
index 0000000000..9a4f6c1bf6
--- /dev/null
+++ b/test-utils/src/bin/falcon_runner_cli.rs
@@ -0,0 +1,25 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+// Copyright 2022 Oxide Computer Company
+
+#[cfg(target_os = "illumos")]
+mod illumos {
+    pub use libfalcon::{cli::run, error::Error, unit::gb, Runner};
+}
+
+#[cfg(target_os = "illumos")]
+#[tokio::main]
+async fn main() -> Result<(), illumos::Error> {
+    use illumos::*;
+    let mut d = Runner::new("launchpad_mcduck_runner");
+
+    d.node("launchpad_mcduck_test_vm", "helios-2.0", 2, gb(2));
+    run(&mut d).await?;
+    Ok(())
+}
+
+#[cfg(not(target_os = "illumos"))]
+#[tokio::main]
+async fn main() {}
diff --git a/test-utils/src/dev/falcon.rs b/test-utils/src/dev/falcon.rs
new file mode 100644
index 0000000000..0929e74ef5
--- /dev/null
+++ b/test-utils/src/dev/falcon.rs
@@ -0,0 +1,22 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Example test to run in a VM.
+
+#[cfg(all(test, target_os = "illumos"))]
+mod test {
+    use anyhow::anyhow;
+    use gethostname::gethostname;
+
+    #[ignore]
+    #[tokio::test]
+    async fn launch() -> Result<(), anyhow::Error> {
+        if gethostname() == "launchpad_mcduck_test_vm" {
+            eprintln!("hello falcon runner: I am inside the VM");
+            Ok(())
+        } else {
+            Err(anyhow!("Test failed: Not running in a VM"))
+        }
+    }
+}
diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs
index 705d6ff08c..31a448c988 100644
--- a/test-utils/src/dev/mod.rs
+++ b/test-utils/src/dev/mod.rs
@@ -8,6 +8,7 @@
 pub mod clickhouse;
 pub mod db;
 pub mod dendrite;
+pub mod falcon;
 pub mod maghemite;
 pub mod poll;
 #[cfg(feature = "seed-gen")]

From f8c394f08191a95e1d83db33a8bf23a68b2c4a6b Mon Sep 17 00:00:00 2001
From: Ryan Goodfellow <ryan.goodfellow@oxide.computer>
Date: Tue, 26 Mar 2024 08:52:20 -0700
Subject: [PATCH 156/157] net-rpw: ignore v6 loopback addrs not created by
 nexus (#5331)

---
 nexus/src/app/background/sync_switch_configuration.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/nexus/src/app/background/sync_switch_configuration.rs b/nexus/src/app/background/sync_switch_configuration.rs
index 84cc450d83..a53e366daa 100644
--- a/nexus/src/app/background/sync_switch_configuration.rs
+++ b/nexus/src/app/background/sync_switch_configuration.rs
@@ -1143,7 +1143,8 @@ async fn switch_loopback_addresses(
                 .insert((*location, IpAddr::V4(entry.addr)));
         }
 
-        for entry in ipv6_loopbacks.iter() {
+        for entry in ipv6_loopbacks.iter().filter(|x| x.tag == OMICRON_DPD_TAG)
+        {
             current_loopback_addresses
                 .insert((*location, IpAddr::V6(entry.addr)));
         }

From c77a16760f38ff69364720e43bd42e74ce8899b8 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Tue, 26 Mar 2024 14:25:29 -0700
Subject: [PATCH 157/157] feedback

---
 nexus/db-queries/src/db/queries/region_allocation.rs | 11 ++++++-----
 nexus/db-queries/src/db/raw_query_builder.rs         |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/nexus/db-queries/src/db/queries/region_allocation.rs b/nexus/db-queries/src/db/queries/region_allocation.rs
index 776fc6c34b..2e4f4cd776 100644
--- a/nexus/db-queries/src/db/queries/region_allocation.rs
+++ b/nexus/db-queries/src/db/queries/region_allocation.rs
@@ -243,11 +243,12 @@ pub fn allocation_query(
     //
     // Selecting two datasets on the same zpool will not initially be
     // possible, as at the time of writing each zpool only has one dataset.
-    // Additionally, we intend to modify the allocation strategy to select
-    // from 3 distinct sleds, removing the possibility entirely. But, if we
-    // introduce a change that adds another crucible dataset to zpools
-    // before we improve the allocation strategy, this check will make sure
-    // we don't violate drive redundancy, and generate an error instead.
+    // Additionally, provide a configuration option ("distinct_sleds") to modify
+    // the allocation strategy to select from 3 distinct sleds, removing the
+    // possibility entirely. But, if we introduce a change that adds another
+    // crucible dataset to zpools before we improve the allocation strategy,
+    // this check will make sure we don't violate drive redundancy, and generate
+    // an error instead.
     .sql("
   do_insert AS (
     SELECT (((
diff --git a/nexus/db-queries/src/db/raw_query_builder.rs b/nexus/db-queries/src/db/raw_query_builder.rs
index 65ff22f86f..5c803e20ac 100644
--- a/nexus/db-queries/src/db/raw_query_builder.rs
+++ b/nexus/db-queries/src/db/raw_query_builder.rs
@@ -159,7 +159,7 @@ impl QueryBuilder {
 }
 
 /// Diesel's [diesel::query_builder::BoxedSqlQuery] has a few drawbacks that
-/// make wrapper more palatable:
+/// make this wrapper more palatable:
 ///
 /// - It always implements "Query" with SqlType = Untyped, so a caller could try to
 /// execute this query and get back any type.