Allow Nexus to specify Zone filesystem location (#5931)

This is a newer version of
#5050 , focused on a
slightly smaller scope.

This PR adds an optional `filesystem_pool` to Sled Agent's API, and to
Nexus' blueprint for zone construction.
This allows Nexus to have control over which zpool is being used for
zone provisioning, rather than letting the Sled Agent control this
decision as it used to.

By giving more control to Nexus, Nexus can better understand the impact
of faults (e.g., will pulling a physical disk cause a zone to fail?) and
can have more control over zone-reallocation.

This field is optional largely for backwards compatibility, but in the
future, will eventually become mandatory.

Fixes #5048
Part of #5929

clients/nexus-client/src/lib.rs

@@ -43,6 +43,7 @@ progenitor::generate_api!(
         TypedUuidForUpstairsKind = omicron_uuid_kinds::TypedUuid<omicron_uuid_kinds::UpstairsKind>,
         TypedUuidForUpstairsRepairKind = omicron_uuid_kinds::TypedUuid<omicron_uuid_kinds::UpstairsRepairKind>,
         TypedUuidForUpstairsSessionKind = omicron_uuid_kinds::TypedUuid<omicron_uuid_kinds::UpstairsSessionKind>,
+        TypedUuidForZpoolKind = omicron_uuid_kinds::TypedUuid<omicron_uuid_kinds::ZpoolKind>,
     },
     patch = {
         SledAgentInfo = { derives = [PartialEq, Eq] },

dev-tools/reconfigurator-cli/tests/test_basic.rs

@@ -8,6 +8,7 @@ use expectorate::assert_contents;
 use nexus_db_queries::authn;
 use nexus_db_queries::authz;
 use nexus_db_queries::context::OpContext;
+use nexus_test_utils::resource_helpers::DiskTestBuilder;
 use nexus_test_utils::SLED_AGENT_UUID;
 use nexus_test_utils_macros::nexus_test;
 use nexus_types::deployment::Blueprint;
@@ -20,6 +21,7 @@ use omicron_test_utils::dev::test_cmds::path_to_executable;
 use omicron_test_utils::dev::test_cmds::redact_variable;
 use omicron_test_utils::dev::test_cmds::run_command;
 use omicron_test_utils::dev::test_cmds::EXIT_SUCCESS;
+use omicron_uuid_kinds::GenericUuid;
 use omicron_uuid_kinds::SledUuid;
 use slog::debug;
 use std::io::BufReader;
@@ -56,6 +58,24 @@ type ControlPlaneTestContext =
 #[nexus_test]
 async fn test_blueprint_edit(cptestctx: &ControlPlaneTestContext) {
     // Setup
+    //
+    // Add a zpool to both sleds, just to ensure that all new zones can find
+    // a transient filesystem wherever they end up being placed.
+    let _sled_agent_zpools = DiskTestBuilder::new(&cptestctx)
+        .on_sled(SledUuid::from_untyped_uuid(
+            cptestctx.sled_agent.sled_agent.id,
+        ))
+        .with_zpool_count(1)
+        .build()
+        .await;
+    let _sled_agent2_zpools = DiskTestBuilder::new(&cptestctx)
+        .on_sled(SledUuid::from_untyped_uuid(
+            cptestctx.sled_agent2.sled_agent.id,
+        ))
+        .with_zpool_count(1)
+        .build()
+        .await;
+
     let nexus = &cptestctx.server.server_context().nexus;
     let datastore = nexus.datastore();
     let log = &cptestctx.logctx.log;

nexus/db-model/src/deployment.rs

@@ -31,7 +31,7 @@ use omicron_common::disk::DiskIdentity;
 use omicron_uuid_kinds::GenericUuid;
 use omicron_uuid_kinds::SledUuid;
 use omicron_uuid_kinds::ZpoolUuid;
-use omicron_uuid_kinds::{ExternalIpKind, SledKind};
+use omicron_uuid_kinds::{ExternalIpKind, SledKind, ZpoolKind};
 use uuid::Uuid;
 
 /// See [`nexus_types::deployment::Blueprint`].
@@ -248,6 +248,7 @@ pub struct BpOmicronZone {
     disposition: DbBpZoneDisposition,
 
     pub external_ip_id: Option<DbTypedUuid<ExternalIpKind>>,
+    pub filesystem_pool: Option<DbTypedUuid<ZpoolKind>>,
 }
 
 impl BpOmicronZone {
@@ -264,6 +265,7 @@ impl BpOmicronZone {
             sled_id,
             blueprint_zone.id.into_untyped_uuid(),
             blueprint_zone.underlay_address,
+            blueprint_zone.filesystem_pool.as_ref().map(|pool| pool.id()),
             &blueprint_zone.zone_type.clone().into(),
             external_ip_id,
         )?;
@@ -291,6 +293,10 @@ impl BpOmicronZone {
             snat_last_port: zone.snat_last_port,
             disposition: to_db_bp_zone_disposition(blueprint_zone.disposition),
             external_ip_id: zone.external_ip_id.map(From::from),
+            filesystem_pool: blueprint_zone
+                .filesystem_pool
+                .as_ref()
+                .map(|pool| pool.id().into()),
         })
     }
 
@@ -302,6 +308,7 @@ impl BpOmicronZone {
             sled_id: self.sled_id.into(),
             id: self.id,
             underlay_address: self.underlay_address,
+            filesystem_pool: self.filesystem_pool.map(|id| id.into()),
             zone_type: self.zone_type,
             primary_service_ip: self.primary_service_ip,
             primary_service_port: self.primary_service_port,

nexus/db-model/src/inventory.rs

@@ -37,6 +37,7 @@ use omicron_uuid_kinds::CollectionUuid;
 use omicron_uuid_kinds::GenericUuid;
 use omicron_uuid_kinds::SledKind;
 use omicron_uuid_kinds::SledUuid;
+use omicron_uuid_kinds::ZpoolKind;
 use omicron_uuid_kinds::ZpoolUuid;
 use uuid::Uuid;
 
@@ -1025,6 +1026,7 @@ pub struct InvOmicronZone {
     pub snat_ip: Option<IpNetwork>,
     pub snat_first_port: Option<SqlU16>,
     pub snat_last_port: Option<SqlU16>,
+    pub filesystem_pool: Option<DbTypedUuid<ZpoolKind>>,
 }
 
 impl InvOmicronZone {
@@ -1039,6 +1041,7 @@ impl InvOmicronZone {
             sled_id,
             zone.id,
             zone.underlay_address,
+            zone.filesystem_pool.as_ref().map(|pool| pool.id()),
             &zone.zone_type,
             external_ip_id,
         )?;
@@ -1064,6 +1067,7 @@ impl InvOmicronZone {
             snat_ip: zone.snat_ip,
             snat_first_port: zone.snat_first_port,
             snat_last_port: zone.snat_last_port,
+            filesystem_pool: zone.filesystem_pool.map(|id| id.into()),
         })
     }
 
@@ -1075,6 +1079,7 @@ impl InvOmicronZone {
             sled_id: self.sled_id.into(),
             id: self.id,
             underlay_address: self.underlay_address,
+            filesystem_pool: self.filesystem_pool.map(|id| id.into()),
             zone_type: self.zone_type,
             primary_service_ip: self.primary_service_ip,
             primary_service_port: self.primary_service_port,

nexus/db-model/src/omicron_zone_config.rs

@@ -23,8 +23,9 @@ use nexus_types::deployment::{
 };
 use nexus_types::inventory::{NetworkInterface, OmicronZoneType};
 use omicron_common::api::internal::shared::NetworkInterfaceKind;
+use omicron_common::zpool_name::ZpoolName;
 use omicron_uuid_kinds::{
-    ExternalIpUuid, GenericUuid, OmicronZoneUuid, SledUuid,
+    ExternalIpUuid, GenericUuid, OmicronZoneUuid, SledUuid, ZpoolUuid,
 };
 use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6};
 use uuid::Uuid;
@@ -34,6 +35,7 @@ pub(crate) struct OmicronZone {
     pub(crate) sled_id: SledUuid,
     pub(crate) id: Uuid,
     pub(crate) underlay_address: ipv6::Ipv6Addr,
+    pub(crate) filesystem_pool: Option<ZpoolUuid>,
     pub(crate) zone_type: ZoneType,
     pub(crate) primary_service_ip: ipv6::Ipv6Addr,
     pub(crate) primary_service_port: SqlU16,
@@ -60,6 +62,7 @@ impl OmicronZone {
         sled_id: SledUuid,
         zone_id: Uuid,
         zone_underlay_address: Ipv6Addr,
+        filesystem_pool: Option<ZpoolUuid>,
         zone_type: &nexus_types::inventory::OmicronZoneType,
         external_ip_id: Option<ExternalIpUuid>,
     ) -> anyhow::Result<Self> {
@@ -201,6 +204,7 @@ impl OmicronZone {
             sled_id,
             id,
             underlay_address,
+            filesystem_pool,
             zone_type,
             primary_service_ip,
             primary_service_port,
@@ -365,6 +369,9 @@ impl OmicronZone {
             disposition,
             id: OmicronZoneUuid::from_untyped_uuid(common.id),
             underlay_address: std::net::Ipv6Addr::from(common.underlay_address),
+            filesystem_pool: common
+                .filesystem_pool
+                .map(|id| ZpoolName::new_external(id)),
             zone_type,
         })
     }
@@ -468,6 +475,9 @@ impl OmicronZone {
         Ok(nexus_types::inventory::OmicronZoneConfig {
             id: common.id,
             underlay_address: std::net::Ipv6Addr::from(common.underlay_address),
+            filesystem_pool: common
+                .filesystem_pool
+                .map(|id| ZpoolName::new_external(id)),
             zone_type,
         })
     }
@@ -558,6 +568,7 @@ impl OmicronZone {
         Ok(ZoneConfigCommon {
             id: self.id,
             underlay_address: self.underlay_address,
+            filesystem_pool: self.filesystem_pool,
             zone_type: self.zone_type,
             primary_service_address,
             snat_ip: self.snat_ip,
@@ -582,6 +593,7 @@ impl OmicronZone {
 struct ZoneConfigCommon {
     id: Uuid,
     underlay_address: ipv6::Ipv6Addr,
+    filesystem_pool: Option<ZpoolUuid>,
     zone_type: ZoneType,
     primary_service_address: SocketAddrV6,
     snat_ip: Option<IpNetwork>,

nexus/db-model/src/schema.rs

@@ -1476,6 +1476,7 @@ table! {
         snat_ip -> Nullable<Inet>,
         snat_first_port -> Nullable<Int4>,
         snat_last_port -> Nullable<Int4>,
+        filesystem_pool -> Nullable<Uuid>,
     }
 }
 
@@ -1592,6 +1593,7 @@ table! {
         snat_last_port -> Nullable<Int4>,
         disposition -> crate::DbBpZoneDispositionEnum,
         external_ip_id -> Nullable<Uuid>,
+        filesystem_pool -> Nullable<Uuid>,
     }
 }
 

nexus/db-model/src/schema_versions.rs

@@ -17,7 +17,8 @@ use std::collections::BTreeMap;
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(80, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(81, 0, 0);
+
 /// List of all past database schema versions, in *reverse* order
 ///
 /// If you want to change the Omicron database schema, you must update this.
@@ -28,6 +29,7 @@ static KNOWN_VERSIONS: Lazy<Vec<KnownVersion>> = Lazy::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(81, "add-nullable-filesystem-pool"),
         KnownVersion::new(80, "add-instance-id-to-migrations"),
         KnownVersion::new(79, "nic-spoof-allow"),
         KnownVersion::new(78, "vpc-subnet-routing"),

nexus/db-queries/src/db/datastore/rack.rs

@@ -1035,6 +1035,7 @@ mod test {
         IdentityMetadataCreateParams, MacAddr, Vni,
     };
     use omicron_common::api::internal::shared::SourceNatConfig;
+    use omicron_common::zpool_name::ZpoolName;
     use omicron_test_utils::dev;
     use omicron_uuid_kinds::{ExternalIpUuid, OmicronZoneUuid};
     use omicron_uuid_kinds::{GenericUuid, ZpoolUuid};
@@ -1287,6 +1288,10 @@ mod test {
     fn_to_get_all!(ip_pool_range, IpPoolRange);
     fn_to_get_all!(dataset, Dataset);
 
+    fn random_zpool() -> ZpoolName {
+        ZpoolName::new_external(ZpoolUuid::new_v4())
+    }
+
     fn random_dataset() -> OmicronZoneDataset {
         OmicronZoneDataset {
             pool_name: illumos_utils::zpool::ZpoolName::new_external(
@@ -1361,6 +1366,7 @@ mod test {
         let mut macs = MacAddr::iter_system();
 
         let mut blueprint_zones = BTreeMap::new();
+        let dataset = random_dataset();
         blueprint_zones.insert(
             SledUuid::from_untyped_uuid(sled1.id()),
             BlueprintZonesConfig {
@@ -1370,9 +1376,10 @@ mod test {
                         disposition: BlueprintZoneDisposition::InService,
                         id: external_dns_id,
                         underlay_address: Ipv6Addr::LOCALHOST,
+                        filesystem_pool: Some(dataset.pool_name.clone()),
                         zone_type: BlueprintZoneType::ExternalDns(
                             blueprint_zone_type::ExternalDns {
-                                dataset: random_dataset(),
+                                dataset,
                                 http_address: "[::1]:80".parse().unwrap(),
                                 dns_address: OmicronZoneExternalFloatingAddr {
                                     id: ExternalIpUuid::new_v4(),
@@ -1399,6 +1406,7 @@ mod test {
                         disposition: BlueprintZoneDisposition::InService,
                         id: ntp1_id,
                         underlay_address: Ipv6Addr::LOCALHOST,
+                        filesystem_pool: Some(random_zpool()),
                         zone_type: BlueprintZoneType::BoundaryNtp(
                             blueprint_zone_type::BoundaryNtp {
                                 address: "[::1]:80".parse().unwrap(),
@@ -1441,6 +1449,7 @@ mod test {
                         disposition: BlueprintZoneDisposition::InService,
                         id: nexus_id,
                         underlay_address: Ipv6Addr::LOCALHOST,
+                        filesystem_pool: Some(random_zpool()),
                         zone_type: BlueprintZoneType::Nexus(
                             blueprint_zone_type::Nexus {
                                 internal_address: "[::1]:80".parse().unwrap(),
@@ -1473,6 +1482,7 @@ mod test {
                         disposition: BlueprintZoneDisposition::InService,
                         id: ntp2_id,
                         underlay_address: Ipv6Addr::LOCALHOST,
+                        filesystem_pool: Some(random_zpool()),
                         zone_type: BlueprintZoneType::BoundaryNtp(
                             blueprint_zone_type::BoundaryNtp {
                                 address: "[::1]:80".parse().unwrap(),
@@ -1514,6 +1524,7 @@ mod test {
                     disposition: BlueprintZoneDisposition::InService,
                     id: ntp3_id,
                     underlay_address: Ipv6Addr::LOCALHOST,
+                    filesystem_pool: Some(random_zpool()),
                     zone_type: BlueprintZoneType::InternalNtp(
                         blueprint_zone_type::InternalNtp {
                             address: "[::1]:80".parse().unwrap(),
@@ -1696,6 +1707,7 @@ mod test {
                         disposition: BlueprintZoneDisposition::InService,
                         id: nexus_id1,
                         underlay_address: Ipv6Addr::LOCALHOST,
+                        filesystem_pool: Some(random_zpool()),
                         zone_type: BlueprintZoneType::Nexus(
                             blueprint_zone_type::Nexus {
                                 internal_address: "[::1]:80".parse().unwrap(),
@@ -1728,6 +1740,7 @@ mod test {
                         disposition: BlueprintZoneDisposition::InService,
                         id: nexus_id2,
                         underlay_address: Ipv6Addr::LOCALHOST,
+                        filesystem_pool: Some(random_zpool()),
                         zone_type: BlueprintZoneType::Nexus(
                             blueprint_zone_type::Nexus {
                                 internal_address: "[::1]:80".parse().unwrap(),
@@ -1969,6 +1982,7 @@ mod test {
                     disposition: BlueprintZoneDisposition::InService,
                     id: nexus_id,
                     underlay_address: Ipv6Addr::LOCALHOST,
+                    filesystem_pool: Some(random_zpool()),
                     zone_type: BlueprintZoneType::Nexus(
                         blueprint_zone_type::Nexus {
                             internal_address: "[::1]:80".parse().unwrap(),
@@ -2067,6 +2081,7 @@ mod test {
         let mut macs = MacAddr::iter_system();
 
         let mut blueprint_zones = BTreeMap::new();
+        let dataset = random_dataset();
         blueprint_zones.insert(
             SledUuid::from_untyped_uuid(sled.id()),
             BlueprintZonesConfig {
@@ -2076,9 +2091,10 @@ mod test {
                         disposition: BlueprintZoneDisposition::InService,
                         id: external_dns_id,
                         underlay_address: Ipv6Addr::LOCALHOST,
+                        filesystem_pool: Some(dataset.pool_name.clone()),
                         zone_type: BlueprintZoneType::ExternalDns(
                             blueprint_zone_type::ExternalDns {
-                                dataset: random_dataset(),
+                                dataset,
                                 http_address: "[::1]:80".parse().unwrap(),
                                 dns_address: OmicronZoneExternalFloatingAddr {
                                     id: ExternalIpUuid::new_v4(),
@@ -2105,6 +2121,7 @@ mod test {
                         disposition: BlueprintZoneDisposition::InService,
                         id: nexus_id,
                         underlay_address: Ipv6Addr::LOCALHOST,
+                        filesystem_pool: Some(random_zpool()),
                         zone_type: BlueprintZoneType::Nexus(
                             blueprint_zone_type::Nexus {
                                 internal_address: "[::1]:80".parse().unwrap(),

nexus/inventory/src/collector.rs

@@ -379,7 +379,9 @@ mod test {
     use gateway_messages::SpPort;
     use nexus_types::inventory::Collection;
     use omicron_common::api::external::Generation;
+    use omicron_common::zpool_name::ZpoolName;
     use omicron_sled_agent::sim;
+    use omicron_uuid_kinds::ZpoolUuid;
     use std::fmt::Write;
     use std::net::Ipv6Addr;
     use std::net::SocketAddrV6;
@@ -547,6 +549,7 @@ mod test {
         let sled_url = format!("http://{}/", agent.http_server.local_addr());
         let client = sled_agent_client::Client::new(&sled_url, log);
 
+        let filesystem_pool = ZpoolName::new_external(ZpoolUuid::new_v4());
         let zone_address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 123, 0, 0);
         client
             .omicron_zones_put(&sled_agent_client::types::OmicronZonesConfig {
@@ -558,6 +561,7 @@ mod test {
                         sled_agent_client::types::OmicronZoneType::Oximeter {
                             address: zone_address.to_string(),
                         },
+                    filesystem_pool: Some(filesystem_pool),
                 }],
             })
             .await

nexus/reconfigurator/execution/src/datasets.rs

@@ -278,6 +278,7 @@ mod tests {
             disposition: BlueprintZoneDisposition::InService,
             id: OmicronZoneUuid::new_v4(),
             underlay_address: "::1".parse().unwrap(),
+            filesystem_pool: Some(ZpoolName::new_external(new_zpool_id)),
             zone_type: BlueprintZoneType::Crucible(
                 blueprint_zone_type::Crucible {
                     address: "[::1]:0".parse().unwrap(),