From 67a1043fe5f68e1dfa0ae4eb3df0aae599f8750c Mon Sep 17 00:00:00 2001 From: Artemis Everfree Date: Wed, 18 Oct 2023 21:09:09 +0000 Subject: [PATCH 1/2] idk how genneration of this works --- openapi/sled-agent.json | 66 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 56437ab283..e45edd4b53 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -1150,6 +1150,67 @@ "pool_name" ] }, + "DatasetArcPrimaryCache": { + "description": "What data the dataset should cache in ARC.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "default" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "all" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "metadata" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "none" + ] + } + }, + "required": [ + "type" + ] + }, + ] + }, "DatasetRequest": { "description": "Describes a request to provision a specific dataset", "type": "object", @@ -1163,6 +1224,9 @@ }, "service_address": { "type": "string" + }, + "arc_primary_cache": { + "$ref": "#/components/schemas/DatasetArcPrimaryCache" } }, "required": [ @@ -3281,4 +3345,4 @@ } } } -} \ No newline at end of file +} From d375210eb4f589382e419ed7f567baec7bedc860 Mon Sep 17 00:00:00 2001 From: Artemis Everfree Date: Wed, 18 Oct 2023 23:14:37 +0000 Subject: [PATCH 2/2] untested, but I want to see how CI feels about it --- illumos-utils/src/zfs.rs | 94 +++++++++++++++++++++++ openapi/sled-agent.json | 80 ++++--------------- schema/all-zone-requests.json | 14 ++++ schema/rss-service-plan.json | 14 ++++ sled-agent/src/backing_fs.rs | 1 + sled-agent/src/bootstrap/pre_server.rs | 1 + sled-agent/src/params.rs | 2 + sled-agent/src/rack_setup/plan/service.rs | 8 +- sled-agent/src/sled_agent.rs | 6 +- sled-agent/src/storage/dataset.rs | 25 ++++++ sled-agent/src/storage_manager.rs | 28 ++++++- sled-hardware/src/disk.rs | 2 + 12 files changed, 205 insertions(+), 70 deletions(-) diff --git a/illumos-utils/src/zfs.rs b/illumos-utils/src/zfs.rs index a6af997619..12a38a10b9 100644 --- a/illumos-utils/src/zfs.rs +++ b/illumos-utils/src/zfs.rs @@ -90,6 +90,15 @@ pub struct SetValueError { err: crate::ExecutionError, } +/// Error returned by [`Zfs::inherit_oxide_value`] +#[derive(thiserror::Error, Debug)] +#[error("Failed to inherit value '{name}' on filesystem {filesystem}: {err}")] +pub struct InheritValueError { + filesystem: String, + name: String, + err: crate::ExecutionError, +} + #[derive(thiserror::Error, Debug)] enum GetValueErrorRaw { #[error(transparent)] @@ -183,6 +192,26 @@ pub struct SizeDetails { pub compression: Option<&'static str>, } +#[derive(Debug, Default)] +pub enum ArcPrimaryCacheDetails { + #[default] + Inherit, + All, + Metadata, + NoCache, +} + +impl ArcPrimaryCacheDetails { + fn zfs_value(&self) -> Option<&'static str> { + match self { + ArcPrimaryCacheDetails::Inherit => None, + ArcPrimaryCacheDetails::All => Some("all"), + ArcPrimaryCacheDetails::Metadata => Some("metadata"), + ArcPrimaryCacheDetails::NoCache => Some("none"), + } + } +} + #[cfg_attr(any(test, feature = "testing"), mockall::automock, allow(dead_code))] impl Zfs { /// Lists all datasets within a pool or existing dataset. @@ -247,6 +276,7 @@ impl Zfs { do_format: bool, encryption_details: Option, size_details: Option, + arc_primary_cache_details: Option, additional_options: Option>, ) -> Result<(), EnsureFilesystemError> { let (exists, mounted) = Self::dataset_exists(name, &mountpoint)?; @@ -257,6 +287,16 @@ impl Zfs { Self::apply_properties(name, &mountpoint, quota, compression)?; } + // Apply primary arc cache mode (in case they've changed across + // sled-agent versions since creation) + if let Some(arc_primary_cache_details) = arc_primary_cache_details { + Self::apply_arc_primary_cache( + name, + &mountpoint, + arc_primary_cache_details, + )?; + } + if encryption_details.is_none() { // If the dataset exists, we're done. Unencrypted datasets are // automatically mounted. @@ -319,6 +359,14 @@ impl Zfs { Self::apply_properties(name, &mountpoint, quota, compression)?; } + if let Some(arc_primary_cache_details) = arc_primary_cache_details { + Self::apply_arc_primary_cache( + name, + &mountpoint, + arc_primary_cache_details, + )?; + } + Ok(()) } @@ -354,6 +402,30 @@ impl Zfs { Ok(()) } + fn apply_arc_primary_cache( + name: &str, + mountpoint: &Mountpoint, + arc_primary_cache_details: ArcPrimaryCacheDetails, + ) -> Result<(), EnsureFilesystemError> { + if let Some(arc_val) = arc_primary_cache_details.zfs_value() { + Self::set_value(name, "primarycache", arc_val).map_err(|err| { + EnsureFilesystemError { + name: name.to_string(), + mountpoint: mountpoint.clone(), + err: err.err.into(), + } + }) + } else { + Self::inherit_value(name, "primarycache").map_err(|err| { + EnsureFilesystemError { + name: name.to_string(), + mountpoint: mountpoint.clone(), + err: err.err.into(), + } + }) + } + } + fn mount_encrypted_dataset( name: &str, mountpoint: &Mountpoint, @@ -439,6 +511,28 @@ impl Zfs { Ok(()) } + /// Inherit the value of an Oxide-managed ZFS property from parent dataset + pub fn inherit_oxide_value( + filesystem_name: &str, + name: &str, + ) -> Result<(), InheritValueError> { + Zfs::inherit_value(filesystem_name, &format!("oxide:{}", name)) + } + + fn inherit_value( + filesystem_name: &str, + name: &str, + ) -> Result<(), InheritValueError> { + let mut command = std::process::Command::new(PFEXEC); + let cmd = command.args(&[ZFS, "inherit", &name, filesystem_name]); + execute(cmd).map_err(|err| InheritValueError { + filesystem: filesystem_name.to_string(), + name: name.to_string(), + err, + })?; + Ok(()) + } + /// Get the value of an Oxide-managed ZFS property. pub fn get_oxide_value( filesystem_name: &str, diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index e45edd4b53..59276be5a9 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -1046,6 +1046,16 @@ "target" ] }, + "DatasetArcPrimaryCache": { + "description": "What data should be kept in the primary ARC cache?", + "type": "string", + "enum": [ + "inherit", + "all", + "metadata", + "no_cache" + ] + }, "DatasetKind": { "description": "The type of a dataset, and an auxiliary information necessary to successfully launch a zone managing the associated data.", "oneOf": [ @@ -1150,71 +1160,13 @@ "pool_name" ] }, - "DatasetArcPrimaryCache": { - "description": "What data the dataset should cache in ARC.", - "oneOf": [ - { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "default" - ] - } - }, - "required": [ - "type" - ] - }, - { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "all" - ] - } - }, - "required": [ - "type" - ] - }, - { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "metadata" - ] - } - }, - "required": [ - "type" - ] - }, - { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "none" - ] - } - }, - "required": [ - "type" - ] - }, - ] - }, "DatasetRequest": { "description": "Describes a request to provision a specific dataset", "type": "object", "properties": { + "arc_primary_cache": { + "$ref": "#/components/schemas/DatasetArcPrimaryCache" + }, "id": { "type": "string", "format": "uuid" @@ -1224,12 +1176,10 @@ }, "service_address": { "type": "string" - }, - "arc_primary_cache": { - "$ref": "#/components/schemas/DatasetArcPrimaryCache" } }, "required": [ + "arc_primary_cache", "id", "name", "service_address" @@ -3345,4 +3295,4 @@ } } } -} +} \ No newline at end of file diff --git a/schema/all-zone-requests.json b/schema/all-zone-requests.json index 468f00ee0c..e22f01d693 100644 --- a/schema/all-zone-requests.json +++ b/schema/all-zone-requests.json @@ -18,6 +18,16 @@ } }, "definitions": { + "DatasetArcPrimaryCache": { + "description": "What data should be kept in the primary ARC cache?", + "type": "string", + "enum": [ + "inherit", + "all", + "metadata", + "no_cache" + ] + }, "DatasetKind": { "description": "The type of a dataset, and an auxiliary information necessary to successfully launch a zone managing the associated data.", "oneOf": [ @@ -126,11 +136,15 @@ "description": "Describes a request to provision a specific dataset", "type": "object", "required": [ + "arc_primary_cache", "id", "name", "service_address" ], "properties": { + "arc_primary_cache": { + "$ref": "#/definitions/DatasetArcPrimaryCache" + }, "id": { "type": "string", "format": "uuid" diff --git a/schema/rss-service-plan.json b/schema/rss-service-plan.json index 725caf0900..f2cf4e9b3b 100644 --- a/schema/rss-service-plan.json +++ b/schema/rss-service-plan.json @@ -18,6 +18,16 @@ } }, "definitions": { + "DatasetArcPrimaryCache": { + "description": "What data should be kept in the primary ARC cache?", + "type": "string", + "enum": [ + "inherit", + "all", + "metadata", + "no_cache" + ] + }, "DatasetKind": { "description": "The type of a dataset, and an auxiliary information necessary to successfully launch a zone managing the associated data.", "oneOf": [ @@ -126,11 +136,15 @@ "description": "Describes a request to provision a specific dataset", "type": "object", "required": [ + "arc_primary_cache", "id", "name", "service_address" ], "properties": { + "arc_primary_cache": { + "$ref": "#/definitions/DatasetArcPrimaryCache" + }, "id": { "type": "string", "format": "uuid" diff --git a/sled-agent/src/backing_fs.rs b/sled-agent/src/backing_fs.rs index 5014ac5999..8ea6f842ab 100644 --- a/sled-agent/src/backing_fs.rs +++ b/sled-agent/src/backing_fs.rs @@ -133,6 +133,7 @@ pub(crate) fn ensure_backing_fs( true, // do_format None, // encryption_details, size_details, + None, Some(vec!["canmount=noauto".to_string()]), // options )?; diff --git a/sled-agent/src/bootstrap/pre_server.rs b/sled-agent/src/bootstrap/pre_server.rs index 0c19c30865..463efa31be 100644 --- a/sled-agent/src/bootstrap/pre_server.rs +++ b/sled-agent/src/bootstrap/pre_server.rs @@ -382,6 +382,7 @@ fn ensure_zfs_ramdisk_dataset() -> Result<(), StartError> { encryption_details, quota, None, + None, ) .map_err(StartError::EnsureZfsRamdiskDataset) } diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 84ec1ef0dc..3fe21964c2 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -595,6 +595,7 @@ pub struct DatasetRequest { pub id: Uuid, pub name: crate::storage::dataset::DatasetName, pub service_address: SocketAddrV6, + pub arc_primary_cache: crate::storage::dataset::DatasetArcPrimaryCache, } impl From for sled_agent_client::types::DatasetRequest { @@ -603,6 +604,7 @@ impl From for sled_agent_client::types::DatasetRequest { id: d.id, name: d.name.into(), service_address: d.service_address.to_string(), + arc_primary_cache: d.arc_primary_cache.into(), } } } diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index 2183aa7b63..f233b3734a 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -10,7 +10,7 @@ use crate::params::{ ServiceZoneService, ZoneType, }; use crate::rack_setup::config::SetupServiceConfig as Config; -use crate::storage::dataset::DatasetName; +use crate::storage::dataset::{DatasetArcPrimaryCache, DatasetName}; use crate::storage_manager::StorageResources; use camino::Utf8PathBuf; use dns_service_client::types::DnsConfigParams; @@ -342,6 +342,7 @@ impl Plan { id, name: dataset_name, service_address: http_address, + arc_primary_cache: DatasetArcPrimaryCache::Inherit, }), services: vec![ServiceZoneService { id, @@ -380,6 +381,7 @@ impl Plan { id, name: dataset_name, service_address: address, + arc_primary_cache: DatasetArcPrimaryCache::Inherit, }), services: vec![ServiceZoneService { id, @@ -427,6 +429,7 @@ impl Plan { id, name: dataset_name, service_address: http_address, + arc_primary_cache: DatasetArcPrimaryCache::Inherit, }), services: vec![ServiceZoneService { id, @@ -549,6 +552,7 @@ impl Plan { id, name: dataset_name, service_address: address, + arc_primary_cache: DatasetArcPrimaryCache::Inherit, }), services: vec![ServiceZoneService { id, @@ -589,6 +593,7 @@ impl Plan { id, name: dataset_name, service_address: address, + arc_primary_cache: DatasetArcPrimaryCache::Inherit, }), services: vec![ServiceZoneService { id, @@ -654,6 +659,7 @@ impl Plan { DatasetKind::Crucible, ), service_address: address, + arc_primary_cache: DatasetArcPrimaryCache::Metadata, }), services: vec![ServiceZoneService { id, diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index b6f910220e..dae95cc40e 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -736,7 +736,11 @@ impl SledAgent { // First, ensure the dataset exists self.inner .storage - .upsert_filesystem(dataset.id, dataset.name.clone()) + .upsert_filesystem( + dataset.id, + dataset.name.clone(), + dataset.arc_primary_cache.clone(), + ) .await?; } diff --git a/sled-agent/src/storage/dataset.rs b/sled-agent/src/storage/dataset.rs index 4efc0f320a..440dcb0557 100644 --- a/sled-agent/src/storage/dataset.rs +++ b/sled-agent/src/storage/dataset.rs @@ -48,6 +48,31 @@ impl From for sled_agent_client::types::DatasetName { } } +/// What data should be kept in the primary ARC cache? +#[derive( + Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +#[serde(rename_all = "snake_case")] +pub enum DatasetArcPrimaryCache { + Inherit, + All, + Metadata, + NoCache, +} + +impl From + for sled_agent_client::types::DatasetArcPrimaryCache +{ + fn from(d: DatasetArcPrimaryCache) -> Self { + match d { + DatasetArcPrimaryCache::Inherit => Self::Inherit, + DatasetArcPrimaryCache::All => Self::All, + DatasetArcPrimaryCache::Metadata => Self::Metadata, + DatasetArcPrimaryCache::NoCache => Self::NoCache, + } + } +} + #[cfg(test)] mod test { use super::*; diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs index c31a4dc0bc..d8b0c362c6 100644 --- a/sled-agent/src/storage_manager.rs +++ b/sled-agent/src/storage_manager.rs @@ -5,6 +5,7 @@ //! Management of sled-local storage. use crate::nexus::NexusClientWithResolver; +use crate::storage::dataset::DatasetArcPrimaryCache; use crate::storage::dataset::DatasetName; use crate::storage::dump_setup::DumpSetup; use crate::zone_bundle::ZoneBundler; @@ -13,6 +14,7 @@ use derive_more::From; use futures::stream::FuturesOrdered; use futures::FutureExt; use futures::StreamExt; +use illumos_utils::zfs::ArcPrimaryCacheDetails; use illumos_utils::zpool::{ZpoolKind, ZpoolName}; use illumos_utils::{zfs::Mountpoint, zpool::ZpoolInfo}; use key_manager::StorageKeyRequester; @@ -171,6 +173,7 @@ type NotifyFut = struct NewFilesystemRequest { dataset_id: Uuid, dataset_name: DatasetName, + dataset_arc_primary_cache: DatasetArcPrimaryCache, responder: oneshot::Sender>, } @@ -404,12 +407,21 @@ impl StorageWorker { &mut self, dataset_id: Uuid, dataset_name: &DatasetName, + dataset_arc_primary_cache: &DatasetArcPrimaryCache, ) -> Result<(), Error> { let zoned = true; let fs_name = &dataset_name.full(); let do_format = true; let encryption_details = None; let size_details = None; + let arc_primary_cache_details = match dataset_arc_primary_cache { + DatasetArcPrimaryCache::Inherit => ArcPrimaryCacheDetails::Inherit, + DatasetArcPrimaryCache::All => ArcPrimaryCacheDetails::All, + DatasetArcPrimaryCache::Metadata => { + ArcPrimaryCacheDetails::Metadata + } + DatasetArcPrimaryCache::NoCache => ArcPrimaryCacheDetails::NoCache, + }; Zfs::ensure_filesystem( &dataset_name.full(), Mountpoint::Path(Utf8PathBuf::from("/data")), @@ -417,6 +429,7 @@ impl StorageWorker { do_format, encryption_details, size_details, + Some(arc_primary_cache_details), None, )?; // Ensure the dataset has a usable UUID. @@ -1016,7 +1029,11 @@ impl StorageWorker { pool.name.clone(), request.dataset_name.dataset().clone(), ); - self.ensure_dataset(request.dataset_id, &dataset_name)?; + self.ensure_dataset( + request.dataset_id, + &dataset_name, + &request.dataset_arc_primary_cache, + )?; Ok(dataset_name) } @@ -1384,10 +1401,15 @@ impl StorageManager { &self, dataset_id: Uuid, dataset_name: DatasetName, + dataset_arc_primary_cache: DatasetArcPrimaryCache, ) -> Result { let (tx, rx) = oneshot::channel(); - let request = - NewFilesystemRequest { dataset_id, dataset_name, responder: tx }; + let request = NewFilesystemRequest { + dataset_id, + dataset_name, + dataset_arc_primary_cache, + responder: tx, + }; self.inner .tx diff --git a/sled-hardware/src/disk.rs b/sled-hardware/src/disk.rs index e3078cbeea..897f04d67e 100644 --- a/sled-hardware/src/disk.rs +++ b/sled-hardware/src/disk.rs @@ -530,6 +530,7 @@ impl Disk { Some(encryption_details), None, None, + None, ); keyfile.zero_and_unlink().await.map_err(|error| { @@ -595,6 +596,7 @@ impl Disk { encryption_details, size_details, None, + None, )?; if dataset.wipe {