diff --git a/Cargo.lock b/Cargo.lock index 58a7909b8f..5d043b902a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5480,6 +5480,7 @@ dependencies = [ name = "nexus-sled-agent-shared" version = "0.1.0" dependencies = [ + "illumos-utils", "omicron-common", "omicron-passwords", "omicron-uuid-kinds", @@ -9847,6 +9848,7 @@ dependencies = [ "omicron-test-utils", "omicron-uuid-kinds", "omicron-workspace-hack", + "once_cell", "rand", "schemars", "serde", diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index be19659c69..5b3202859c 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -70,6 +70,7 @@ progenitor::generate_api!( SledRole = nexus_sled_agent_shared::inventory::SledRole, SourceNatConfig = omicron_common::api::internal::shared::SourceNatConfig, SwitchLocation = omicron_common::api::external::SwitchLocation, + TypedUuidForDatasetKind = omicron_uuid_kinds::DatasetUuid, TypedUuidForInstanceKind = omicron_uuid_kinds::InstanceUuid, TypedUuidForPropolisKind = omicron_uuid_kinds::PropolisUuid, TypedUuidForZpoolKind = omicron_uuid_kinds::ZpoolUuid, diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 58cace3032..fae45faa08 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -593,7 +593,18 @@ impl JsonSchema for RoleName { // // TODO: custom JsonSchema impl to describe i64::MAX limit; this is blocked by // https://github.com/oxidecomputer/typify/issues/589 -#[derive(Copy, Clone, Debug, Serialize, JsonSchema, PartialEq, Eq)] +#[derive( + Copy, + Clone, + Debug, + Serialize, + JsonSchema, + Hash, + PartialEq, + Eq, + PartialOrd, + Ord, +)] pub struct ByteCount(u64); impl<'de> Deserialize<'de> for ByteCount { diff --git a/common/src/disk.rs b/common/src/disk.rs index ed0bf8666e..b3695a3358 100644 --- a/common/src/disk.rs +++ b/common/src/disk.rs @@ -14,7 +14,7 @@ use std::fmt; use uuid::Uuid; use crate::{ - api::external::Generation, + api::external::{ByteCount, Generation}, ledger::Ledgerable, zpool_name::{ZpoolKind, ZpoolName}, }; @@ -244,10 +244,10 @@ pub struct DatasetConfig { pub compression: CompressionAlgorithm, /// The upper bound on the amount of storage used by this dataset - pub quota: Option, + pub quota: Option, /// The lower bound on the amount of storage usable by this dataset - pub reservation: Option, + pub reservation: Option, } #[derive( diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index 48f5137698..642a4fb31b 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -4744,6 +4744,51 @@ fn inv_collection_print_sleds(collection: &Collection) { sled.reservoir_size.to_whole_gibibytes() ); + if !sled.zpools.is_empty() { + println!(" physical disks:"); + } + for disk in &sled.disks { + let nexus_types::inventory::PhysicalDisk { + identity, + variant, + slot, + } = disk; + println!(" {variant:?}: {identity:?} in {slot}"); + } + + if !sled.zpools.is_empty() { + println!(" zpools"); + } + for zpool in &sled.zpools { + let nexus_types::inventory::Zpool { id, total_size, .. } = zpool; + println!(" {id}: total size: {total_size}"); + } + + if !sled.datasets.is_empty() { + println!(" datasets:"); + } + for dataset in &sled.datasets { + let nexus_types::inventory::Dataset { + id, + name, + available, + used, + quota, + reservation, + compression, + } = dataset; + + let id = if let Some(id) = id { + id.to_string() + } else { + String::from("none") + }; + + println!(" {name} - id: {id}, compression: {compression}"); + println!(" available: {available}, used: {used}"); + println!(" reservation: {reservation:?}, quota: {quota:?}"); + } + if let Some(zones) = collection.omicron_zones.get(&sled.sled_id) { println!( " zones collected from {} at {}", diff --git a/dev-tools/omdb/src/bin/omdb/sled_agent.rs b/dev-tools/omdb/src/bin/omdb/sled_agent.rs index 9a9a17eff4..44adbf4763 100644 --- a/dev-tools/omdb/src/bin/omdb/sled_agent.rs +++ b/dev-tools/omdb/src/bin/omdb/sled_agent.rs @@ -38,6 +38,10 @@ enum SledAgentCommands { #[clap(subcommand)] Zpools(ZpoolCommands), + /// print information about datasets + #[clap(subcommand)] + Datasets(DatasetCommands), + /// print information about the local bootstore node #[clap(subcommand)] Bootstore(BootstoreCommands), @@ -55,6 +59,16 @@ enum ZpoolCommands { List, } +#[derive(Debug, Subcommand)] +enum DatasetCommands { + /// Print list of all datasets the sled agent is configured to manage + /// + /// Note that the set of actual datasets on the sled may be distinct, + /// use the `omdb db inventory collections show` command to see the latest + /// set of datasets collected from sleds. + List, +} + #[derive(Debug, Subcommand)] enum BootstoreCommands { /// Show the internal state of the local bootstore node @@ -86,6 +100,9 @@ impl SledAgentArgs { SledAgentCommands::Zpools(ZpoolCommands::List) => { cmd_zpools_list(&client).await } + SledAgentCommands::Datasets(DatasetCommands::List) => { + cmd_datasets_list(&client).await + } SledAgentCommands::Bootstore(BootstoreCommands::Status) => { cmd_bootstore_status(&client).await } @@ -130,6 +147,26 @@ async fn cmd_zpools_list( Ok(()) } +/// Runs `omdb sled-agent datasets list` +async fn cmd_datasets_list( + client: &sled_agent_client::Client, +) -> Result<(), anyhow::Error> { + let response = client.datasets_get().await.context("listing datasets")?; + let response = response.into_inner(); + + println!("dataset configuration @ generation {}:", response.generation); + let datasets = response.datasets; + + if datasets.is_empty() { + println!(" "); + } + for dataset in &datasets { + println!(" {:?}", dataset); + } + + Ok(()) +} + /// Runs `omdb sled-agent bootstore status` async fn cmd_bootstore_status( client: &sled_agent_client::Client, diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out index 55781136b6..380f0ec9e5 100644 --- a/dev-tools/omdb/tests/usage_errors.out +++ b/dev-tools/omdb/tests/usage_errors.out @@ -642,6 +642,7 @@ Usage: omdb sled-agent [OPTIONS] Commands: zones print information about zones zpools print information about zpools + datasets print information about datasets bootstore print information about the local bootstore node help Print this message or the help of the given subcommand(s) diff --git a/illumos-utils/src/zfs.rs b/illumos-utils/src/zfs.rs index 5d512677f8..f92fd5d60f 100644 --- a/illumos-utils/src/zfs.rs +++ b/illumos-utils/src/zfs.rs @@ -5,10 +5,14 @@ //! Utilities for poking at ZFS. use crate::{execute, PFEXEC}; +use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; +use omicron_common::api::external::ByteCount; use omicron_common::disk::CompressionAlgorithm; use omicron_common::disk::DiskIdentity; +use omicron_uuid_kinds::DatasetUuid; use std::fmt; +use std::str::FromStr; // These locations in the ramdisk must only be used by the switch zone. // @@ -203,14 +207,93 @@ pub struct EncryptionDetails { #[derive(Debug, Default)] pub struct SizeDetails { - pub quota: Option, - pub reservation: Option, + pub quota: Option, + pub reservation: Option, pub compression: CompressionAlgorithm, } +#[derive(Debug)] +pub struct DatasetProperties { + /// The Uuid of the dataset + pub id: Option, + /// The full name of the dataset. + pub name: String, + /// Remaining space in the dataset and descendants. + pub avail: ByteCount, + /// Space used by dataset and descendants. + pub used: ByteCount, + /// Maximum space usable by dataset and descendants. + pub quota: Option, + /// Minimum space guaranteed to dataset and descendants. + pub reservation: Option, + /// The compression algorithm used for this dataset. + /// + /// This probably aligns with a value from + /// [omicron_common::disk::CompressionAlgorithm], but is left as an untyped + /// string so that unexpected compression formats don't prevent inventory + /// from being collected. + pub compression: String, +} + +impl DatasetProperties { + // care about. + const ZFS_LIST_STR: &'static str = + "oxide:uuid,name,avail,used,quota,reservation,compression"; +} + +// An inner parsing function, so that the FromStr implementation can always emit +// the string 's' that failed to parse in the error message. +fn dataset_properties_parse( + s: &str, +) -> Result { + let mut iter = s.split_whitespace(); + + let id = match iter.next().context("Missing UUID")? { + "-" => None, + anything_else => Some(anything_else.parse::()?), + }; + + let name = iter.next().context("Missing 'name'")?.to_string(); + let avail = + iter.next().context("Missing 'avail'")?.parse::()?.try_into()?; + let used = + iter.next().context("Missing 'used'")?.parse::()?.try_into()?; + let quota = match iter.next().context("Missing 'quota'")?.parse::()? { + 0 => None, + q => Some(q.try_into()?), + }; + let reservation = + match iter.next().context("Missing 'reservation'")?.parse::()? { + 0 => None, + r => Some(r.try_into()?), + }; + let compression = iter.next().context("Missing 'compression'")?.to_string(); + + Ok(DatasetProperties { + id, + name, + avail, + used, + quota, + reservation, + compression, + }) +} + +impl FromStr for DatasetProperties { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + dataset_properties_parse(s) + .with_context(|| format!("Failed to parse: {s}")) + } +} + #[cfg_attr(any(test, feature = "testing"), mockall::automock, allow(dead_code))] impl Zfs { /// Lists all datasets within a pool or existing dataset. + /// + /// Strips the input `name` from the output dataset names. pub fn list_datasets(name: &str) -> Result, ListDatasetsError> { let mut command = std::process::Command::new(ZFS); let cmd = command.args(&["list", "-d", "1", "-rHpo", "name", name]); @@ -229,6 +312,38 @@ impl Zfs { Ok(filesystems) } + /// Get information about datasets within a list of zpools / datasets. + /// + /// This function is similar to [Zfs::list_datasets], but provides a more + /// substantial results about the datasets found. + /// + /// Sorts results and de-duplicates them by name. + pub fn get_dataset_properties( + datasets: &[String], + ) -> Result, anyhow::Error> { + let mut command = std::process::Command::new(ZFS); + let cmd = command.args(&["list", "-d", "1", "-rHpo"]); + + // Note: this is tightly coupled with the layout of DatasetProperties + cmd.arg(DatasetProperties::ZFS_LIST_STR); + cmd.args(datasets); + + let output = execute(cmd).with_context(|| { + format!("Failed to get dataset properties for {datasets:?}") + })?; + let stdout = String::from_utf8(output.stdout)?; + let mut datasets = stdout + .trim() + .split('\n') + .map(|row| row.parse::()) + .collect::, _>>()?; + + datasets.sort_by(|d1, d2| d1.name.partial_cmp(&d2.name).unwrap()); + datasets.dedup_by(|d1, d2| d1.name.eq(&d2.name)); + + Ok(datasets) + } + /// Return the name of a dataset for a ZFS object. /// /// The object can either be a dataset name, or a path, in which case it @@ -402,15 +517,15 @@ impl Zfs { fn apply_properties( name: &str, mountpoint: &Mountpoint, - quota: Option, - reservation: Option, + quota: Option, + reservation: Option, compression: CompressionAlgorithm, ) -> Result<(), EnsureFilesystemError> { let quota = quota - .map(|q| q.to_string()) + .map(|q| q.to_bytes().to_string()) .unwrap_or_else(|| String::from("none")); let reservation = reservation - .map(|r| r.to_string()) + .map(|r| r.to_bytes().to_string()) .unwrap_or_else(|| String::from("none")); let compression = compression.to_string(); @@ -699,3 +814,121 @@ pub fn get_all_omicron_datasets_for_delete() -> anyhow::Result> { Ok(datasets) } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn parse_dataset_props() { + let input = + "- dataset_name 1234 5678 0 0 off"; + let props = DatasetProperties::from_str(&input) + .expect("Should have parsed data"); + + assert_eq!(props.id, None); + assert_eq!(props.name, "dataset_name"); + assert_eq!(props.avail.to_bytes(), 1234); + assert_eq!(props.used.to_bytes(), 5678); + assert_eq!(props.quota, None); + assert_eq!(props.reservation, None); + assert_eq!(props.compression, "off"); + } + + #[test] + fn parse_dataset_props_with_optionals() { + let input = "d4e1e554-7b98-4413-809e-4a42561c3d0c dataset_name 1234 5678 111 222 off"; + let props = DatasetProperties::from_str(&input) + .expect("Should have parsed data"); + + assert_eq!( + props.id, + Some("d4e1e554-7b98-4413-809e-4a42561c3d0c".parse().unwrap()) + ); + assert_eq!(props.name, "dataset_name"); + assert_eq!(props.avail.to_bytes(), 1234); + assert_eq!(props.used.to_bytes(), 5678); + assert_eq!(props.quota.map(|q| q.to_bytes()), Some(111)); + assert_eq!(props.reservation.map(|r| r.to_bytes()), Some(222)); + assert_eq!(props.compression, "off"); + } + + #[test] + fn parse_dataset_bad_uuid() { + let input = "bad dataset_name 1234 5678 111 222 off"; + let err = DatasetProperties::from_str(&input) + .expect_err("Should have failed to parse"); + assert!( + format!("{err:#}").contains("error parsing UUID (dataset)"), + "{err}" + ); + } + + #[test] + fn parse_dataset_bad_avail() { + let input = "- dataset_name BADAVAIL 5678 111 222 off"; + let err = DatasetProperties::from_str(&input) + .expect_err("Should have failed to parse"); + assert!( + format!("{err:#}").contains("invalid digit found in string"), + "{err}" + ); + } + + #[test] + fn parse_dataset_bad_usage() { + let input = "- dataset_name 1234 BADUSAGE 111 222 off"; + let err = DatasetProperties::from_str(&input) + .expect_err("Should have failed to parse"); + assert!( + format!("{err:#}").contains("invalid digit found in string"), + "{err}" + ); + } + + #[test] + fn parse_dataset_bad_quota() { + let input = "- dataset_name 1234 5678 BADQUOTA 222 off"; + let err = DatasetProperties::from_str(&input) + .expect_err("Should have failed to parse"); + assert!( + format!("{err:#}").contains("invalid digit found in string"), + "{err}" + ); + } + + #[test] + fn parse_dataset_bad_reservation() { + let input = "- dataset_name 1234 5678 111 BADRES off"; + let err = DatasetProperties::from_str(&input) + .expect_err("Should have failed to parse"); + assert!( + format!("{err:#}").contains("invalid digit found in string"), + "{err}" + ); + } + + #[test] + fn parse_dataset_missing_fields() { + let expect_missing = |input: &str, what: &str| { + let err = DatasetProperties::from_str(input) + .expect_err("Should have failed to parse"); + let err = format!("{err:#}"); + assert!(err.contains(&format!("Missing {what}")), "{err}"); + }; + + expect_missing( + "- dataset_name 1234 5678 111 222", + "'compression'", + ); + expect_missing( + "- dataset_name 1234 5678 111", + "'reservation'", + ); + expect_missing("- dataset_name 1234 5678", "'quota'"); + expect_missing("- dataset_name 1234", "'used'"); + expect_missing("- dataset_name", "'avail'"); + expect_missing("-", "'name'"); + expect_missing("", "UUID"); + } +} diff --git a/nexus-sled-agent-shared/Cargo.toml b/nexus-sled-agent-shared/Cargo.toml index 144c755f34..504cd92c37 100644 --- a/nexus-sled-agent-shared/Cargo.toml +++ b/nexus-sled-agent-shared/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" workspace = true [dependencies] +illumos-utils.workspace = true omicron-common.workspace = true omicron-passwords.workspace = true omicron-uuid-kinds.workspace = true diff --git a/nexus-sled-agent-shared/src/inventory.rs b/nexus-sled-agent-shared/src/inventory.rs index 2a94fc50db..faee944105 100644 --- a/nexus-sled-agent-shared/src/inventory.rs +++ b/nexus-sled-agent-shared/src/inventory.rs @@ -14,6 +14,7 @@ use omicron_common::{ disk::DiskVariant, zpool_name::ZpoolName, }; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::ZpoolUuid; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -38,6 +39,49 @@ pub struct InventoryZpool { pub total_size: ByteCount, } +/// Identifies information about datasets within Oxide-managed zpools +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] +pub struct InventoryDataset { + /// Although datasets mandated by the control plane will have UUIDs, + /// datasets can be created (and have been created) without UUIDs. + pub id: Option, + + /// This name is the full path of the dataset. + // This is akin to [sled_storage::dataset::DatasetName::full_name], + // and it's also what you'd see when running "zfs list". + pub name: String, + + /// The amount of remaining space usable by the dataset (and children) + /// assuming there is no other activity within the pool. + pub available: ByteCount, + + /// The amount of space consumed by this dataset and descendents. + pub used: ByteCount, + + /// The maximum amount of space usable by a dataset and all descendents. + pub quota: Option, + + /// The minimum amount of space guaranteed to a dataset and descendents. + pub reservation: Option, + + /// The compression algorithm used for this dataset, if any. + pub compression: String, +} + +impl From for InventoryDataset { + fn from(props: illumos_utils::zfs::DatasetProperties) -> Self { + Self { + id: props.id, + name: props.name, + available: props.avail, + used: props.used, + quota: props.quota, + reservation: props.reservation, + compression: props.compression, + } + } +} + /// Identity and basic status information about this sled agent #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] pub struct Inventory { @@ -50,6 +94,7 @@ pub struct Inventory { pub reservoir_size: ByteCount, pub disks: Vec, pub zpools: Vec, + pub datasets: Vec, } /// Describes the role of the sled within the rack. diff --git a/nexus/db-model/src/inventory.rs b/nexus/db-model/src/inventory.rs index 71e44b4d82..d5754965f2 100644 --- a/nexus/db-model/src/inventory.rs +++ b/nexus/db-model/src/inventory.rs @@ -7,7 +7,7 @@ use crate::omicron_zone_config::{self, OmicronZoneNic}; use crate::schema::{ hw_baseboard_id, inv_caboose, inv_collection, inv_collection_error, - inv_omicron_zone, inv_omicron_zone_nic, inv_physical_disk, + inv_dataset, inv_omicron_zone, inv_omicron_zone_nic, inv_physical_disk, inv_root_of_trust, inv_root_of_trust_page, inv_service_processor, inv_sled_agent, inv_sled_omicron_zones, inv_zpool, sw_caboose, sw_root_of_trust_page, @@ -39,6 +39,7 @@ use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::zpool_name::ZpoolName; use omicron_uuid_kinds::CollectionKind; use omicron_uuid_kinds::CollectionUuid; +use omicron_uuid_kinds::DatasetKind; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::SledKind; use omicron_uuid_kinds::SledUuid; @@ -935,6 +936,56 @@ impl From for nexus_types::inventory::Zpool { } } +/// See [`nexus_types::inventory::Dataset`]. +#[derive(Queryable, Clone, Debug, Selectable, Insertable)] +#[diesel(table_name = inv_dataset)] +pub struct InvDataset { + pub inv_collection_id: DbTypedUuid, + pub sled_id: DbTypedUuid, + pub id: Option>, + pub name: String, + pub available: ByteCount, + pub used: ByteCount, + pub quota: Option, + pub reservation: Option, + pub compression: String, +} + +impl InvDataset { + pub fn new( + inv_collection_id: CollectionUuid, + sled_id: SledUuid, + dataset: &nexus_types::inventory::Dataset, + ) -> Self { + Self { + inv_collection_id: inv_collection_id.into(), + sled_id: sled_id.into(), + + id: dataset.id.map(|id| id.into()), + name: dataset.name.clone(), + available: dataset.available.into(), + used: dataset.used.into(), + quota: dataset.quota.map(|q| q.into()), + reservation: dataset.reservation.map(|r| r.into()), + compression: dataset.compression.clone(), + } + } +} + +impl From for nexus_types::inventory::Dataset { + fn from(dataset: InvDataset) -> Self { + Self { + id: dataset.id.map(|id| id.0), + name: dataset.name, + available: *dataset.available, + used: *dataset.used, + quota: dataset.quota.map(|q| *q), + reservation: dataset.reservation.map(|r| *r), + compression: dataset.compression, + } + } +} + /// See [`nexus_types::inventory::OmicronZonesFound`]. #[derive(Queryable, Clone, Debug, Selectable, Insertable)] #[diesel(table_name = inv_sled_omicron_zones)] diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 733a0657bc..071c7e9229 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -1441,6 +1441,21 @@ table! { } } +table! { + inv_dataset (inv_collection_id, sled_id, name) { + inv_collection_id -> Uuid, + sled_id -> Uuid, + + id -> Nullable, + name -> Text, + available -> Int8, + used -> Int8, + quota -> Nullable, + reservation -> Nullable, + compression -> Text, + } +} + table! { inv_sled_omicron_zones (inv_collection_id, sled_id) { inv_collection_id -> Uuid, diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 81e18386f4..450cda4db4 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(95, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(96, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(96, "inv-dataset"), KnownVersion::new(95, "turn-boot-on-fault-into-auto-restart"), KnownVersion::new(94, "put-back-creating-vmm-state"), KnownVersion::new(93, "dataset-kinds-zone-and-debug"), diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs index 8888f2caaa..95816aae84 100644 --- a/nexus/db-queries/src/db/datastore/inventory.rs +++ b/nexus/db-queries/src/db/datastore/inventory.rs @@ -38,6 +38,7 @@ use nexus_db_model::HwRotSlotEnum; use nexus_db_model::InvCaboose; use nexus_db_model::InvCollection; use nexus_db_model::InvCollectionError; +use nexus_db_model::InvDataset; use nexus_db_model::InvOmicronZone; use nexus_db_model::InvOmicronZoneNic; use nexus_db_model::InvPhysicalDisk; @@ -157,6 +158,17 @@ impl DataStore { }) .collect(); + // Pull datasets out of all sled agents + let datasets: Vec<_> = collection + .sled_agents + .iter() + .flat_map(|(sled_id, sled_agent)| { + sled_agent.datasets.iter().map(|dataset| { + InvDataset::new(collection_id, *sled_id, dataset) + }) + }) + .collect(); + // Partition the sled agents into those with an associated baseboard id // and those without one. We handle these pretty differently. let (sled_agents_baseboards, sled_agents_no_baseboards): ( @@ -745,6 +757,25 @@ impl DataStore { } } + // Insert rows for all the datasets we found. + { + use db::schema::inv_dataset::dsl; + + let batch_size = SQL_BATCH_SIZE.get().try_into().unwrap(); + let mut datasets = datasets.into_iter(); + loop { + let some_datasets = + datasets.by_ref().take(batch_size).collect::>(); + if some_datasets.is_empty() { + break; + } + let _ = diesel::insert_into(dsl::inv_dataset) + .values(some_datasets) + .execute_async(&conn) + .await?; + } + } + // Insert rows for the sled agents that we found. In practice, we'd // expect these to all have baseboards (if using Oxide hardware) or // none have baseboards (if not). @@ -1601,6 +1632,39 @@ impl DataStore { zpools }; + // Mapping of "Sled ID" -> "All datasets reported by that sled" + let datasets: BTreeMap> = { + use db::schema::inv_dataset::dsl; + + let mut datasets = + BTreeMap::>::new(); + let mut paginator = Paginator::new(batch_size); + while let Some(p) = paginator.next() { + let batch = paginated_multicolumn( + dsl::inv_dataset, + (dsl::sled_id, dsl::name), + &p.current_pagparams(), + ) + .filter(dsl::inv_collection_id.eq(db_id)) + .select(InvDataset::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + paginator = p.found_batch(&batch, &|row| { + (row.sled_id, row.name.clone()) + }); + for dataset in batch { + datasets + .entry(dataset.sled_id.into_untyped_uuid()) + .or_default() + .push(dataset.into()); + } + } + datasets + }; + // Collect the unique baseboard ids referenced by SPs, RoTs, and Sled // Agents. let baseboard_id_ids: BTreeSet<_> = sps @@ -1707,6 +1771,10 @@ impl DataStore { .get(sled_id.as_untyped_uuid()) .map(|zpools| zpools.to_vec()) .unwrap_or_default(), + datasets: datasets + .get(sled_id.as_untyped_uuid()) + .map(|datasets| datasets.to_vec()) + .unwrap_or_default(), }; Ok((sled_id, sled_agent)) }) diff --git a/nexus/db-queries/src/db/datastore/physical_disk.rs b/nexus/db-queries/src/db/datastore/physical_disk.rs index dc26e093c0..8162b11f2d 100644 --- a/nexus/db-queries/src/db/datastore/physical_disk.rs +++ b/nexus/db-queries/src/db/datastore/physical_disk.rs @@ -697,6 +697,7 @@ mod test { usable_physical_ram: ByteCount::from(1024 * 1024), disks, zpools: vec![], + datasets: vec![], }, ) .unwrap(); diff --git a/nexus/inventory/src/builder.rs b/nexus/inventory/src/builder.rs index 3b6b4bc007..82be8be091 100644 --- a/nexus/inventory/src/builder.rs +++ b/nexus/inventory/src/builder.rs @@ -516,6 +516,11 @@ impl CollectionBuilder { .into_iter() .map(|z| Zpool::new(time_collected, z)) .collect(), + datasets: inventory + .datasets + .into_iter() + .map(|d| d.into()) + .collect(), }; if let Some(previous) = self.sleds.get(&sled_id) { diff --git a/nexus/inventory/src/examples.rs b/nexus/inventory/src/examples.rs index 2dade3d34f..626f077deb 100644 --- a/nexus/inventory/src/examples.rs +++ b/nexus/inventory/src/examples.rs @@ -13,6 +13,7 @@ use gateway_client::types::SpState; use gateway_client::types::SpType; use nexus_sled_agent_shared::inventory::Baseboard; use nexus_sled_agent_shared::inventory::Inventory; +use nexus_sled_agent_shared::inventory::InventoryDataset; use nexus_sled_agent_shared::inventory::InventoryDisk; use nexus_sled_agent_shared::inventory::InventoryZpool; use nexus_sled_agent_shared::inventory::OmicronZonesConfig; @@ -321,6 +322,7 @@ pub fn representative() -> Representative { }, ]; let zpools = vec![]; + let datasets = vec![]; builder .found_sled_inventory( @@ -335,6 +337,7 @@ pub fn representative() -> Representative { SledRole::Gimlet, disks, zpools, + datasets, ), ) .unwrap(); @@ -361,6 +364,7 @@ pub fn representative() -> Representative { SledRole::Scrimlet, vec![], vec![], + vec![], ), ) .unwrap(); @@ -382,6 +386,7 @@ pub fn representative() -> Representative { SledRole::Gimlet, vec![], vec![], + vec![], ), ) .unwrap(); @@ -401,6 +406,7 @@ pub fn representative() -> Representative { SledRole::Gimlet, vec![], vec![], + vec![], ), ) .unwrap(); @@ -511,6 +517,7 @@ pub fn sled_agent( sled_role: SledRole, disks: Vec, zpools: Vec, + datasets: Vec, ) -> Inventory { Inventory { baseboard, @@ -522,5 +529,6 @@ pub fn sled_agent( usable_physical_ram: ByteCount::from(1024 * 1024), disks, zpools, + datasets, } } diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index 26e4910c5b..538c7bec1b 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -564,9 +564,10 @@ impl Sled { slot: i64::try_from(i).unwrap(), }) .collect(), - // Zpools won't necessarily show up until our first request - // to provision storage, so we omit them. + // Zpools & Datasets won't necessarily show up until our first + // request to provision storage, so we omit them. zpools: vec![], + datasets: vec![], } }; @@ -702,6 +703,7 @@ impl Sled { usable_physical_ram: inv_sled_agent.usable_physical_ram, disks: vec![], zpools: vec![], + datasets: vec![], }; Sled { diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs index 0ec2f6fbdb..9c96c40966 100644 --- a/nexus/types/src/inventory.rs +++ b/nexus/types/src/inventory.rs @@ -17,6 +17,7 @@ pub use gateway_client::types::PowerState; pub use gateway_client::types::RotImageError; pub use gateway_client::types::RotSlot; pub use gateway_client::types::SpType; +use nexus_sled_agent_shared::inventory::InventoryDataset; use nexus_sled_agent_shared::inventory::InventoryDisk; use nexus_sled_agent_shared::inventory::InventoryZpool; use nexus_sled_agent_shared::inventory::OmicronZoneConfig; @@ -28,6 +29,7 @@ pub use omicron_common::api::internal::shared::NetworkInterfaceKind; pub use omicron_common::api::internal::shared::SourceNatConfig; pub use omicron_common::zpool_name::ZpoolName; use omicron_uuid_kinds::CollectionUuid; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use serde::{Deserialize, Serialize}; @@ -396,6 +398,47 @@ impl Zpool { } } +/// A dataset reported by a sled agent. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +pub struct Dataset { + /// Although datasets mandated by the control plane will have UUIDs, + /// datasets can be created (and have been created) without UUIDs. + pub id: Option, + + /// This name is the full path of the dataset. + pub name: String, + + /// The amount of remaining space usable by the dataset (and children) + /// assuming there is no other activity within the pool. + pub available: ByteCount, + + /// The amount of space consumed by this dataset and descendents. + pub used: ByteCount, + + /// The maximum amount of space usable by a dataset and all descendents. + pub quota: Option, + + /// The minimum amount of space guaranteed to a dataset and descendents. + pub reservation: Option, + + /// The compression algorithm used for this dataset, if any. + pub compression: String, +} + +impl From for Dataset { + fn from(disk: InventoryDataset) -> Self { + Self { + id: disk.id, + name: disk.name, + available: disk.available, + used: disk.used, + quota: disk.quota, + reservation: disk.reservation, + compression: disk.compression, + } + } +} + /// Inventory reported by sled agent /// /// This is a software notion of a sled, distinct from an underlying baseboard. @@ -415,6 +458,7 @@ pub struct SledAgent { pub reservoir_size: ByteCount, pub disks: Vec, pub zpools: Vec, + pub datasets: Vec, } #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index bb8e4e0b87..52a4c5ba1f 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -2251,16 +2251,20 @@ "quota": { "nullable": true, "description": "The upper bound on the amount of storage used by this dataset", - "type": "integer", - "format": "uint", - "minimum": 0 + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] }, "reservation": { "nullable": true, "description": "The lower bound on the amount of storage usable by this dataset", - "type": "integer", - "format": "uint", - "minimum": 0 + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] } }, "required": [ @@ -3389,6 +3393,12 @@ "baseboard": { "$ref": "#/components/schemas/Baseboard" }, + "datasets": { + "type": "array", + "items": { + "$ref": "#/components/schemas/InventoryDataset" + } + }, "disks": { "type": "array", "items": { @@ -3425,6 +3435,7 @@ }, "required": [ "baseboard", + "datasets", "disks", "reservoir_size", "sled_agent_address", @@ -3435,6 +3446,69 @@ "zpools" ] }, + "InventoryDataset": { + "description": "Identifies information about datasets within Oxide-managed zpools", + "type": "object", + "properties": { + "available": { + "description": "The amount of remaining space usable by the dataset (and children) assuming there is no other activity within the pool.", + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] + }, + "compression": { + "description": "The compression algorithm used for this dataset, if any.", + "type": "string" + }, + "id": { + "nullable": true, + "description": "Although datasets mandated by the control plane will have UUIDs, datasets can be created (and have been created) without UUIDs.", + "allOf": [ + { + "$ref": "#/components/schemas/TypedUuidForDatasetKind" + } + ] + }, + "name": { + "description": "This name is the full path of the dataset.", + "type": "string" + }, + "quota": { + "nullable": true, + "description": "The maximum amount of space usable by a dataset and all descendents.", + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] + }, + "reservation": { + "nullable": true, + "description": "The minimum amount of space guaranteed to a dataset and descendents.", + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] + }, + "used": { + "description": "The amount of space consumed by this dataset and descendents.", + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] + } + }, + "required": [ + "available", + "compression", + "name", + "used" + ] + }, "InventoryDisk": { "description": "Identifies information about disks which may be attached to Sleds.", "type": "object", diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 388bc81993..49cf268010 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -3230,6 +3230,31 @@ CREATE TABLE IF NOT EXISTS omicron.public.inv_zpool ( -- Allow looking up the most recent Zpool by ID CREATE INDEX IF NOT EXISTS inv_zpool_by_id_and_time ON omicron.public.inv_zpool (id, time_collected DESC); +CREATE TABLE IF NOT EXISTS omicron.public.inv_dataset ( + -- where this observation came from + -- (foreign key into `inv_collection` table) + inv_collection_id UUID NOT NULL, + sled_id UUID NOT NULL, + + -- The control plane ID of the zpool. + -- This is nullable because datasets have been historically + -- self-managed by the Sled Agent, and some don't have explicit UUIDs. + id UUID, + + name TEXT NOT NULL, + available INT8 NOT NULL, + used INT8 NOT NULL, + quota INT8, + reservation INT8, + compression TEXT NOT NULL, + + -- PK consisting of: + -- - Which collection this was + -- - The sled reporting the disk + -- - The name of this dataset + PRIMARY KEY (inv_collection_id, sled_id, name) +); + CREATE TABLE IF NOT EXISTS omicron.public.inv_sled_omicron_zones ( -- where this observation came from -- (foreign key into `inv_collection` table) @@ -4259,7 +4284,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '95.0.0', NULL) + (TRUE, NOW(), NOW(), '96.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/crdb/inv-dataset/up01.sql b/schema/crdb/inv-dataset/up01.sql new file mode 100644 index 0000000000..d3d21d16ae --- /dev/null +++ b/schema/crdb/inv-dataset/up01.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS omicron.public.inv_dataset ( + inv_collection_id UUID NOT NULL, + sled_id UUID NOT NULL, + + id UUID, + + name TEXT NOT NULL, + available INT8 NOT NULL, + used INT8 NOT NULL, + quota INT8, + reservation INT8, + compression TEXT NOT NULL, + + PRIMARY KEY (inv_collection_id, sled_id, name) +); diff --git a/schema/omicron-datasets.json b/schema/omicron-datasets.json index 07fc2cfb13..dba383d9d4 100644 --- a/schema/omicron-datasets.json +++ b/schema/omicron-datasets.json @@ -23,6 +23,12 @@ } }, "definitions": { + "ByteCount": { + "description": "Byte count to express memory or storage capacity.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, "CompressionAlgorithm": { "oneOf": [ { @@ -164,21 +170,25 @@ }, "quota": { "description": "The upper bound on the amount of storage used by this dataset", - "type": [ - "integer", - "null" - ], - "format": "uint", - "minimum": 0.0 + "anyOf": [ + { + "$ref": "#/definitions/ByteCount" + }, + { + "type": "null" + } + ] }, "reservation": { "description": "The lower bound on the amount of storage usable by this dataset", - "type": [ - "integer", - "null" - ], - "format": "uint", - "minimum": 0.0 + "anyOf": [ + { + "$ref": "#/definitions/ByteCount" + }, + { + "type": "null" + } + ] } } }, diff --git a/sled-agent/src/backing_fs.rs b/sled-agent/src/backing_fs.rs index a0f7826db3..1b1e105a28 100644 --- a/sled-agent/src/backing_fs.rs +++ b/sled-agent/src/backing_fs.rs @@ -25,7 +25,9 @@ use camino::Utf8PathBuf; use illumos_utils::zfs::{ EnsureFilesystemError, GetValueError, Mountpoint, SizeDetails, Zfs, }; +use omicron_common::api::external::ByteCount; use omicron_common::disk::CompressionAlgorithm; +use once_cell::sync::Lazy; use std::io; #[derive(Debug, thiserror::Error)] @@ -49,7 +51,7 @@ struct BackingFs<'a> { // Mountpoint mountpoint: &'static str, // Optional quota, in _bytes_ - quota: Option, + quota: Option, // Optional compression mode compression: CompressionAlgorithm, // Linked service @@ -75,7 +77,7 @@ impl<'a> BackingFs<'a> { self } - const fn quota(mut self, quota: usize) -> Self { + const fn quota(mut self, quota: ByteCount) -> Self { self.quota = Some(quota); self } @@ -100,18 +102,19 @@ const BACKING_FMD_DATASET: &'static str = "fmd"; const BACKING_FMD_MOUNTPOINT: &'static str = "/var/fm/fmd"; const BACKING_FMD_SUBDIRS: [&'static str; 3] = ["rsrc", "ckpt", "xprt"]; const BACKING_FMD_SERVICE: &'static str = "svc:/system/fmd:default"; -const BACKING_FMD_QUOTA: usize = 500 * (1 << 20); // 500 MiB +const BACKING_FMD_QUOTA: u64 = 500 * (1 << 20); // 500 MiB const BACKING_COMPRESSION: CompressionAlgorithm = CompressionAlgorithm::On; const BACKINGFS_COUNT: usize = 1; -static BACKINGFS: [BackingFs; BACKINGFS_COUNT] = +static BACKINGFS: Lazy<[BackingFs; BACKINGFS_COUNT]> = Lazy::new(|| { [BackingFs::new(BACKING_FMD_DATASET) .mountpoint(BACKING_FMD_MOUNTPOINT) .subdirs(&BACKING_FMD_SUBDIRS) - .quota(BACKING_FMD_QUOTA) + .quota(ByteCount::try_from(BACKING_FMD_QUOTA).unwrap()) .compression(BACKING_COMPRESSION) - .service(BACKING_FMD_SERVICE)]; + .service(BACKING_FMD_SERVICE)] +}); /// Ensure that the backing filesystems are mounted. /// If the underlying dataset for a backing fs does not exist on the specified diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 17b75c4334..1c1f3cb0b3 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -1629,6 +1629,7 @@ mod test { }) .collect(), zpools: vec![], + datasets: vec![], }, true, ) diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index cf3c022551..eddb425bc2 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -18,7 +18,8 @@ use anyhow::Context; use dropshot::{HttpError, HttpServer}; use futures::lock::Mutex; use nexus_sled_agent_shared::inventory::{ - Inventory, InventoryDisk, InventoryZpool, OmicronZonesConfig, SledRole, + Inventory, InventoryDataset, InventoryDisk, InventoryZpool, + OmicronZonesConfig, SledRole, }; use omicron_common::api::external::{ ByteCount, DiskState, Error, Generation, ResourceType, @@ -855,6 +856,30 @@ impl SledAgent { }) }) .collect::, anyhow::Error>>()?, + // NOTE: We report the "configured" datasets as the "real" datasets + // unconditionally here. No real datasets exist, so we're free + // to lie here, but this information should be taken with a + // particularly careful grain-of-salt -- it's supposed to + // represent the "real" datasets the sled agent can observe. + datasets: storage + .datasets_config_list() + .await + .map(|config| { + config + .datasets + .into_iter() + .map(|(id, config)| InventoryDataset { + id: Some(id), + name: config.name.full_name(), + available: ByteCount::from_kibibytes_u32(0), + used: ByteCount::from_kibibytes_u32(0), + quota: config.quota, + reservation: config.reservation, + compression: config.compression.to_string(), + }) + .collect::>() + }) + .unwrap_or_else(|_| vec![]), }) } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index f13d8caccf..b25bd39bd3 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -32,7 +32,8 @@ use illumos_utils::opte::PortManager; use illumos_utils::zone::PROPOLIS_ZONE_PREFIX; use illumos_utils::zone::ZONE_PREFIX; use nexus_sled_agent_shared::inventory::{ - Inventory, InventoryDisk, InventoryZpool, OmicronZonesConfig, SledRole, + Inventory, InventoryDataset, InventoryDisk, InventoryZpool, + OmicronZonesConfig, SledRole, }; use omicron_common::address::{ get_sled_address, get_switch_zone_address, Ipv6Subnet, SLED_PREFIX, @@ -73,6 +74,7 @@ use sled_agent_types::zone_bundle::{ use sled_hardware::{underlay, HardwareManager}; use sled_hardware_types::underlay::BootstrapInterface; use sled_hardware_types::Baseboard; +use sled_storage::dataset::{CRYPT_DATASET, ZONE_DATASET}; use sled_storage::manager::StorageHandle; use slog::Logger; use std::collections::BTreeMap; @@ -1233,6 +1235,7 @@ impl SledAgent { let mut disks = vec![]; let mut zpools = vec![]; + let mut datasets = vec![]; let all_disks = self.storage().get_latest_disks().await; for (identity, variant, slot, _firmware) in all_disks.iter_all() { disks.push(InventoryDisk { @@ -1261,6 +1264,47 @@ impl SledAgent { id: zpool.id(), total_size: ByteCount::try_from(info.size())?, }); + + // We do care about the total space usage within zpools, but mapping + // the layering back to "datasets we care about" is a little + // awkward. + // + // We could query for all datasets within a pool, but the sled agent + // doesn't really care about the children of datasets that it + // allocates. As an example: Sled Agent might provision a "crucible" + // dataset, but how region allocation occurs within that dataset + // is a detail for Crucible to care about, not the Sled Agent. + // + // To balance this effort, we ask for information about datasets + // that the Sled Agent is directly resopnsible for managing. + let datasets_of_interest = [ + // We care about the zpool itself, and all direct children. + zpool.to_string(), + // Likewise, we care about the encrypted dataset, and all + // direct children. + format!("{zpool}/{CRYPT_DATASET}"), + // The zone dataset gives us additional context on "what zones + // have datasets provisioned". + format!("{zpool}/{ZONE_DATASET}"), + ]; + let inv_props = + match illumos_utils::zfs::Zfs::get_dataset_properties( + datasets_of_interest.as_slice(), + ) { + Ok(props) => props + .into_iter() + .map(|prop| InventoryDataset::from(prop)), + Err(err) => { + warn!( + self.log, + "Failed to access dataset info within zpool"; + "zpool" => %zpool, + "err" => %err + ); + continue; + } + }; + datasets.extend(inv_props); } Ok(Inventory { @@ -1273,6 +1317,7 @@ impl SledAgent { reservoir_size, disks, zpools, + datasets, }) } } diff --git a/sled-agent/src/zone_bundle.rs b/sled-agent/src/zone_bundle.rs index e38d4c963b..88cb853652 100644 --- a/sled-agent/src/zone_bundle.rs +++ b/sled-agent/src/zone_bundle.rs @@ -1733,6 +1733,8 @@ mod illumos_tests { use chrono::TimeZone; use chrono::Timelike; use chrono::Utc; + use omicron_common::api::external::ByteCount; + use once_cell::sync::Lazy; use rand::RngCore; use sled_storage::manager_test_harness::StorageManagerTestHarness; use slog::Drain; @@ -1919,7 +1921,9 @@ mod illumos_tests { // i.e., the "ashift" value. An empty dataset is unlikely to contain more // than one megabyte of overhead, so use that as a conservative test size to // avoid issues. - const TEST_QUOTA: usize = sled_storage::dataset::DEBUG_DATASET_QUOTA; + static TEST_QUOTA: Lazy = Lazy::new(|| { + sled_storage::dataset::DEBUG_DATASET_QUOTA.try_into().unwrap() + }); async fn run_test_with_zfs_dataset(test: T) where @@ -1967,18 +1971,17 @@ mod illumos_tests { // If this needs to change, go modify the "add_vdevs" call in // "setup_storage". assert!( - TEST_QUOTA + *TEST_QUOTA < StorageManagerTestHarness::DEFAULT_VDEV_SIZE .try_into() .unwrap(), - "Quota larger than underlying device (quota: {}, device size: {})", + "Quota larger than underlying device (quota: {:?}, device size: {})", TEST_QUOTA, StorageManagerTestHarness::DEFAULT_VDEV_SIZE, ); anyhow::ensure!( - bundle_utilization.dataset_quota - == u64::try_from(TEST_QUOTA).unwrap(), + bundle_utilization.dataset_quota == TEST_QUOTA.to_bytes(), "computed incorrect dataset quota" ); diff --git a/sled-storage/Cargo.toml b/sled-storage/Cargo.toml index 2439c52aa7..27555ce96d 100644 --- a/sled-storage/Cargo.toml +++ b/sled-storage/Cargo.toml @@ -20,6 +20,7 @@ illumos-utils.workspace = true key-manager.workspace = true omicron-common.workspace = true omicron-uuid-kinds.workspace = true +once_cell.workspace = true rand.workspace = true schemars = { workspace = true, features = [ "chrono", "uuid1" ] } serde.workspace = true diff --git a/sled-storage/src/dataset.rs b/sled-storage/src/dataset.rs index e2b024db11..3be7b0afa4 100644 --- a/sled-storage/src/dataset.rs +++ b/sled-storage/src/dataset.rs @@ -14,10 +14,12 @@ use illumos_utils::zfs::{ }; use illumos_utils::zpool::ZpoolName; use key_manager::StorageKeyRequester; +use omicron_common::api::external::ByteCount; use omicron_common::api::internal::shared::DatasetKind; use omicron_common::disk::{ CompressionAlgorithm, DatasetName, DiskIdentity, DiskVariant, GzipLevel, }; +use once_cell::sync::Lazy; use rand::distributions::{Alphanumeric, DistString}; use slog::{debug, info, Logger}; use std::process::Stdio; @@ -34,16 +36,16 @@ pub const M2_BACKING_DATASET: &'static str = "backing"; cfg_if! { if #[cfg(any(test, feature = "testing"))] { // Tuned for zone_bundle tests - pub const DEBUG_DATASET_QUOTA: usize = 1 << 20; + pub const DEBUG_DATASET_QUOTA: u64 = 1 << 20; } else { // TODO-correctness: This value of 100GiB is a pretty wild guess, and should be // tuned as needed. - pub const DEBUG_DATASET_QUOTA: usize = 100 * (1 << 30); + pub const DEBUG_DATASET_QUOTA: u64 = 100 * (1 << 30); } } // TODO-correctness: This value of 100GiB is a pretty wild guess, and should be // tuned as needed. -pub const DUMP_DATASET_QUOTA: usize = 100 * (1 << 30); +pub const DUMP_DATASET_QUOTA: u64 = 100 * (1 << 30); // passed to zfs create -o compression= pub const DUMP_DATASET_COMPRESSION: CompressionAlgorithm = CompressionAlgorithm::GzipN { level: GzipLevel::new::<9>() }; @@ -57,41 +59,50 @@ pub const U2_DEBUG_DATASET: &'static str = "crypt/debug"; pub const CRYPT_DATASET: &'static str = "crypt"; const U2_EXPECTED_DATASET_COUNT: usize = 2; -static U2_EXPECTED_DATASETS: [ExpectedDataset; U2_EXPECTED_DATASET_COUNT] = [ - // Stores filesystems for zones - ExpectedDataset::new(ZONE_DATASET).wipe(), - // For storing full kernel RAM dumps - ExpectedDataset::new(DUMP_DATASET) - .quota(DUMP_DATASET_QUOTA) - .compression(DUMP_DATASET_COMPRESSION), -]; +static U2_EXPECTED_DATASETS: Lazy< + [ExpectedDataset; U2_EXPECTED_DATASET_COUNT], +> = Lazy::new(|| { + [ + // Stores filesystems for zones + ExpectedDataset::new(ZONE_DATASET).wipe(), + // For storing full kernel RAM dumps + ExpectedDataset::new(DUMP_DATASET) + .quota(ByteCount::try_from(DUMP_DATASET_QUOTA).unwrap()) + .compression(DUMP_DATASET_COMPRESSION), + ] +}); const M2_EXPECTED_DATASET_COUNT: usize = 6; -static M2_EXPECTED_DATASETS: [ExpectedDataset; M2_EXPECTED_DATASET_COUNT] = [ - // Stores software images. - // - // Should be duplicated to both M.2s. - ExpectedDataset::new(INSTALL_DATASET), - // Stores crash dumps. - ExpectedDataset::new(CRASH_DATASET), - // Backing store for OS data that should be persisted across reboots. - // Its children are selectively overlay mounted onto parts of the ramdisk - // root. - ExpectedDataset::new(M2_BACKING_DATASET), - // Stores cluter configuration information. - // - // Should be duplicated to both M.2s. - ExpectedDataset::new(CLUSTER_DATASET), - // Stores configuration data, including: - // - What services should be launched on this sled - // - Information about how to initialize the Sled Agent - // - (For scrimlets) RSS setup information - // - // Should be duplicated to both M.2s. - ExpectedDataset::new(CONFIG_DATASET), - // Store debugging data, such as service bundles. - ExpectedDataset::new(M2_DEBUG_DATASET).quota(DEBUG_DATASET_QUOTA), -]; +static M2_EXPECTED_DATASETS: Lazy< + [ExpectedDataset; M2_EXPECTED_DATASET_COUNT], +> = Lazy::new(|| { + [ + // Stores software images. + // + // Should be duplicated to both M.2s. + ExpectedDataset::new(INSTALL_DATASET), + // Stores crash dumps. + ExpectedDataset::new(CRASH_DATASET), + // Backing store for OS data that should be persisted across reboots. + // Its children are selectively overlay mounted onto parts of the ramdisk + // root. + ExpectedDataset::new(M2_BACKING_DATASET), + // Stores cluter configuration information. + // + // Should be duplicated to both M.2s. + ExpectedDataset::new(CLUSTER_DATASET), + // Stores configuration data, including: + // - What services should be launched on this sled + // - Information about how to initialize the Sled Agent + // - (For scrimlets) RSS setup information + // + // Should be duplicated to both M.2s. + ExpectedDataset::new(CONFIG_DATASET), + // Store debugging data, such as service bundles. + ExpectedDataset::new(M2_DEBUG_DATASET) + .quota(ByteCount::try_from(DEBUG_DATASET_QUOTA).unwrap()), + ] +}); // Helper type for describing expected datasets and their optional quota. #[derive(Clone, Copy, Debug)] @@ -99,7 +110,7 @@ struct ExpectedDataset { // Name for the dataset name: &'static str, // Optional quota, in _bytes_ - quota: Option, + quota: Option, // Identifies if the dataset should be deleted on boot wipe: bool, // Optional compression mode @@ -116,7 +127,7 @@ impl ExpectedDataset { } } - const fn quota(mut self, quota: usize) -> Self { + fn quota(mut self, quota: ByteCount) -> Self { self.quota = Some(quota); self } diff --git a/sled-storage/src/manager.rs b/sled-storage/src/manager.rs index 88e1bbaa34..21d54b4176 100644 --- a/sled-storage/src/manager.rs +++ b/sled-storage/src/manager.rs @@ -1669,7 +1669,8 @@ mod tests { // However, calling it with a different input and the same generation // number should fail. config.generation = current_config_generation; - config.datasets.values_mut().next().unwrap().reservation = Some(1024); + config.datasets.values_mut().next().unwrap().reservation = + Some(1024.into()); let err = harness.handle().datasets_ensure(config.clone()).await.unwrap_err(); assert!(matches!(err, Error::DatasetConfigurationChanged { .. }));