From a83450e7061537d6ac0495eb35f1b2749ce9e2a9 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 26 Aug 2024 16:42:50 -0700 Subject: [PATCH] Add docs, use ByteCount more, better simulated inventory --- Cargo.lock | 1 + common/src/api/external/mod.rs | 13 +++- common/src/disk.rs | 6 +- dev-tools/omdb/src/bin/omdb/sled_agent.rs | 6 +- illumos-utils/src/zfs.rs | 8 +-- sled-agent/src/backing_fs.rs | 15 ++-- sled-agent/src/sim/sled_agent.rs | 31 +++++++- sled-storage/Cargo.toml | 1 + sled-storage/src/dataset.rs | 87 +++++++++++++---------- 9 files changed, 112 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d254480ac4..cd9efeb4cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9656,6 +9656,7 @@ dependencies = [ "omicron-test-utils", "omicron-uuid-kinds", "omicron-workspace-hack", + "once_cell", "rand", "schemars", "serde", diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 07e4fd0b83..fbd5f3450b 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -593,7 +593,18 @@ impl JsonSchema for RoleName { // // TODO: custom JsonSchema impl to describe i64::MAX limit; this is blocked by // https://github.com/oxidecomputer/typify/issues/589 -#[derive(Copy, Clone, Debug, Serialize, JsonSchema, PartialEq, Eq)] +#[derive( + Copy, + Clone, + Debug, + Serialize, + JsonSchema, + Hash, + PartialEq, + Eq, + PartialOrd, + Ord, +)] pub struct ByteCount(u64); impl<'de> Deserialize<'de> for ByteCount { diff --git a/common/src/disk.rs b/common/src/disk.rs index a2016a9442..b60957fc95 100644 --- a/common/src/disk.rs +++ b/common/src/disk.rs @@ -14,7 +14,7 @@ use std::fmt; use uuid::Uuid; use crate::{ - api::external::Generation, + api::external::{ByteCount, Generation}, ledger::Ledgerable, zpool_name::{ZpoolKind, ZpoolName}, }; @@ -159,10 +159,10 @@ pub struct DatasetConfig { pub compression: Option, /// The upper bound on the amount of storage used by this dataset - pub quota: Option, + pub quota: Option, /// The lower bound on the amount of storage usable by this dataset - pub reservation: Option, + pub reservation: Option, } #[derive( diff --git a/dev-tools/omdb/src/bin/omdb/sled_agent.rs b/dev-tools/omdb/src/bin/omdb/sled_agent.rs index b97fb35e8c..44adbf4763 100644 --- a/dev-tools/omdb/src/bin/omdb/sled_agent.rs +++ b/dev-tools/omdb/src/bin/omdb/sled_agent.rs @@ -61,7 +61,11 @@ enum ZpoolCommands { #[derive(Debug, Subcommand)] enum DatasetCommands { - /// Print list of all datasets managed by the sled agent + /// Print list of all datasets the sled agent is configured to manage + /// + /// Note that the set of actual datasets on the sled may be distinct, + /// use the `omdb db inventory collections show` command to see the latest + /// set of datasets collected from sleds. List, } diff --git a/illumos-utils/src/zfs.rs b/illumos-utils/src/zfs.rs index 5968569ebe..18e16f4ec7 100644 --- a/illumos-utils/src/zfs.rs +++ b/illumos-utils/src/zfs.rs @@ -206,8 +206,8 @@ pub struct EncryptionDetails { #[derive(Debug, Default)] pub struct SizeDetails { - pub quota: Option, - pub reservation: Option, + pub quota: Option, + pub reservation: Option, pub compression: Option, } @@ -507,8 +507,8 @@ impl Zfs { fn apply_properties( name: &str, mountpoint: &Mountpoint, - quota: Option, - reservation: Option, + quota: Option, + reservation: Option, compression: Option, ) -> Result<(), EnsureFilesystemError> { let quota = quota diff --git a/sled-agent/src/backing_fs.rs b/sled-agent/src/backing_fs.rs index 48002a8841..a207668a91 100644 --- a/sled-agent/src/backing_fs.rs +++ b/sled-agent/src/backing_fs.rs @@ -25,6 +25,8 @@ use camino::Utf8PathBuf; use illumos_utils::zfs::{ EnsureFilesystemError, GetValueError, Mountpoint, SizeDetails, Zfs, }; +use omicron_common::api::external::ByteCount; +use once_cell::sync::Lazy; use std::io; #[derive(Debug, thiserror::Error)] @@ -48,7 +50,7 @@ struct BackingFs<'a> { // Mountpoint mountpoint: &'static str, // Optional quota, in _bytes_ - quota: Option, + quota: Option, // Optional compression mode compression: Option<&'static str>, // Linked service @@ -74,7 +76,7 @@ impl<'a> BackingFs<'a> { self } - const fn quota(mut self, quota: usize) -> Self { + const fn quota(mut self, quota: ByteCount) -> Self { self.quota = Some(quota); self } @@ -99,18 +101,19 @@ const BACKING_FMD_DATASET: &'static str = "fmd"; const BACKING_FMD_MOUNTPOINT: &'static str = "/var/fm/fmd"; const BACKING_FMD_SUBDIRS: [&'static str; 3] = ["rsrc", "ckpt", "xprt"]; const BACKING_FMD_SERVICE: &'static str = "svc:/system/fmd:default"; -const BACKING_FMD_QUOTA: usize = 500 * (1 << 20); // 500 MiB +const BACKING_FMD_QUOTA: u64 = 500 * (1 << 20); // 500 MiB const BACKING_COMPRESSION: &'static str = "on"; const BACKINGFS_COUNT: usize = 1; -static BACKINGFS: [BackingFs; BACKINGFS_COUNT] = +static BACKINGFS: Lazy<[BackingFs; BACKINGFS_COUNT]> = Lazy::new(|| { [BackingFs::new(BACKING_FMD_DATASET) .mountpoint(BACKING_FMD_MOUNTPOINT) .subdirs(&BACKING_FMD_SUBDIRS) - .quota(BACKING_FMD_QUOTA) + .quota(ByteCount::try_from(BACKING_FMD_QUOTA).unwrap()) .compression(BACKING_COMPRESSION) - .service(BACKING_FMD_SERVICE)]; + .service(BACKING_FMD_SERVICE)] +}); /// Ensure that the backing filesystems are mounted. /// If the underlying dataset for a backing fs does not exist on the specified diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index bc1de2e6b5..786ec51bc1 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -18,7 +18,8 @@ use anyhow::Context; use dropshot::{HttpError, HttpServer}; use futures::lock::Mutex; use nexus_sled_agent_shared::inventory::{ - Inventory, InventoryDisk, InventoryZpool, OmicronZonesConfig, SledRole, + Inventory, InventoryDataset, InventoryDisk, InventoryZpool, + OmicronZonesConfig, SledRole, }; use omicron_common::api::external::{ ByteCount, DiskState, Error, Generation, ResourceType, @@ -902,8 +903,32 @@ impl SledAgent { }) }) .collect::, anyhow::Error>>()?, - // TODO: Make this more real? - datasets: vec![], + // NOTE: We report the "configured" datasets as the "real" datasets + // unconditionally here. No real datasets exist, so we're free + // to lie here, but this information should be taken with a + // particularly careful grain-of-salt -- it's supposed to + // represent the "real" datasets the sled agent can observe. + datasets: storage + .datasets_config_list() + .await + .map(|config| { + config + .datasets + .into_iter() + .map(|(id, config)| InventoryDataset { + id: Some(id), + name: config.name.full_name(), + available: ByteCount::from_kibibytes_u32(0), + used: ByteCount::from_kibibytes_u32(0), + quota: config.quota, + reservation: config.reservation, + compression: config + .compression + .unwrap_or_else(|| String::new()), + }) + .collect::>() + }) + .unwrap_or_else(|_| vec![]), }) } diff --git a/sled-storage/Cargo.toml b/sled-storage/Cargo.toml index 2439c52aa7..27555ce96d 100644 --- a/sled-storage/Cargo.toml +++ b/sled-storage/Cargo.toml @@ -20,6 +20,7 @@ illumos-utils.workspace = true key-manager.workspace = true omicron-common.workspace = true omicron-uuid-kinds.workspace = true +once_cell.workspace = true rand.workspace = true schemars = { workspace = true, features = [ "chrono", "uuid1" ] } serde.workspace = true diff --git a/sled-storage/src/dataset.rs b/sled-storage/src/dataset.rs index b95877418e..f487460a6c 100644 --- a/sled-storage/src/dataset.rs +++ b/sled-storage/src/dataset.rs @@ -14,8 +14,10 @@ use illumos_utils::zfs::{ }; use illumos_utils::zpool::ZpoolName; use key_manager::StorageKeyRequester; +use omicron_common::api::external::ByteCount; use omicron_common::api::internal::shared::DatasetKind; use omicron_common::disk::{DatasetName, DiskIdentity, DiskVariant}; +use once_cell::sync::Lazy; use rand::distributions::{Alphanumeric, DistString}; use slog::{debug, info, Logger}; use std::process::Stdio; @@ -32,16 +34,16 @@ pub const M2_BACKING_DATASET: &'static str = "backing"; cfg_if! { if #[cfg(any(test, feature = "testing"))] { // Tuned for zone_bundle tests - pub const DEBUG_DATASET_QUOTA: usize = 1 << 20; + pub const DEBUG_DATASET_QUOTA: u64 = 1 << 20; } else { // TODO-correctness: This value of 100GiB is a pretty wild guess, and should be // tuned as needed. - pub const DEBUG_DATASET_QUOTA: usize = 100 * (1 << 30); + pub const DEBUG_DATASET_QUOTA: u64 = 100 * (1 << 30); } } // TODO-correctness: This value of 100GiB is a pretty wild guess, and should be // tuned as needed. -pub const DUMP_DATASET_QUOTA: usize = 100 * (1 << 30); +pub const DUMP_DATASET_QUOTA: u64 = 100 * (1 << 30); // passed to zfs create -o compression= pub const DUMP_DATASET_COMPRESSION: &'static str = "gzip-9"; @@ -54,41 +56,50 @@ pub const U2_DEBUG_DATASET: &'static str = "crypt/debug"; pub const CRYPT_DATASET: &'static str = "crypt"; const U2_EXPECTED_DATASET_COUNT: usize = 2; -static U2_EXPECTED_DATASETS: [ExpectedDataset; U2_EXPECTED_DATASET_COUNT] = [ - // Stores filesystems for zones - ExpectedDataset::new(ZONE_DATASET).wipe(), - // For storing full kernel RAM dumps - ExpectedDataset::new(DUMP_DATASET) - .quota(DUMP_DATASET_QUOTA) - .compression(DUMP_DATASET_COMPRESSION), -]; +static U2_EXPECTED_DATASETS: Lazy< + [ExpectedDataset; U2_EXPECTED_DATASET_COUNT], +> = Lazy::new(|| { + [ + // Stores filesystems for zones + ExpectedDataset::new(ZONE_DATASET).wipe(), + // For storing full kernel RAM dumps + ExpectedDataset::new(DUMP_DATASET) + .quota(ByteCount::try_from(DUMP_DATASET_QUOTA).unwrap()) + .compression(DUMP_DATASET_COMPRESSION), + ] +}); const M2_EXPECTED_DATASET_COUNT: usize = 6; -static M2_EXPECTED_DATASETS: [ExpectedDataset; M2_EXPECTED_DATASET_COUNT] = [ - // Stores software images. - // - // Should be duplicated to both M.2s. - ExpectedDataset::new(INSTALL_DATASET), - // Stores crash dumps. - ExpectedDataset::new(CRASH_DATASET), - // Backing store for OS data that should be persisted across reboots. - // Its children are selectively overlay mounted onto parts of the ramdisk - // root. - ExpectedDataset::new(M2_BACKING_DATASET), - // Stores cluter configuration information. - // - // Should be duplicated to both M.2s. - ExpectedDataset::new(CLUSTER_DATASET), - // Stores configuration data, including: - // - What services should be launched on this sled - // - Information about how to initialize the Sled Agent - // - (For scrimlets) RSS setup information - // - // Should be duplicated to both M.2s. - ExpectedDataset::new(CONFIG_DATASET), - // Store debugging data, such as service bundles. - ExpectedDataset::new(M2_DEBUG_DATASET).quota(DEBUG_DATASET_QUOTA), -]; +static M2_EXPECTED_DATASETS: Lazy< + [ExpectedDataset; M2_EXPECTED_DATASET_COUNT], +> = Lazy::new(|| { + [ + // Stores software images. + // + // Should be duplicated to both M.2s. + ExpectedDataset::new(INSTALL_DATASET), + // Stores crash dumps. + ExpectedDataset::new(CRASH_DATASET), + // Backing store for OS data that should be persisted across reboots. + // Its children are selectively overlay mounted onto parts of the ramdisk + // root. + ExpectedDataset::new(M2_BACKING_DATASET), + // Stores cluter configuration information. + // + // Should be duplicated to both M.2s. + ExpectedDataset::new(CLUSTER_DATASET), + // Stores configuration data, including: + // - What services should be launched on this sled + // - Information about how to initialize the Sled Agent + // - (For scrimlets) RSS setup information + // + // Should be duplicated to both M.2s. + ExpectedDataset::new(CONFIG_DATASET), + // Store debugging data, such as service bundles. + ExpectedDataset::new(M2_DEBUG_DATASET) + .quota(ByteCount::try_from(DEBUG_DATASET_QUOTA).unwrap()), + ] +}); // Helper type for describing expected datasets and their optional quota. #[derive(Clone, Copy, Debug)] @@ -96,7 +107,7 @@ struct ExpectedDataset { // Name for the dataset name: &'static str, // Optional quota, in _bytes_ - quota: Option, + quota: Option, // Identifies if the dataset should be deleted on boot wipe: bool, // Optional compression mode @@ -108,7 +119,7 @@ impl ExpectedDataset { ExpectedDataset { name, quota: None, wipe: false, compression: None } } - const fn quota(mut self, quota: usize) -> Self { + fn quota(mut self, quota: ByteCount) -> Self { self.quota = Some(quota); self }