From c5b5d4a44644e530e9c9312e28f0629f6a6df38f Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 25 Sep 2024 12:41:47 -0700 Subject: [PATCH 01/37] APIs --- Cargo.lock | 1 + sled-agent/api/Cargo.toml | 1 + sled-agent/api/src/lib.rs | 68 +++++++++++++++++++++++++- sled-agent/src/http_entrypoints.rs | 28 +++++++++++ sled-agent/src/sim/http_entrypoints.rs | 28 +++++++++++ uuid-kinds/src/lib.rs | 1 + 6 files changed, 126 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index ceba971d3c..b6a0515f2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10250,6 +10250,7 @@ name = "sled-agent-api" version = "0.1.0" dependencies = [ "camino", + "chrono", "dropshot 0.10.2-dev", "nexus-sled-agent-shared", "omicron-common", diff --git a/sled-agent/api/Cargo.toml b/sled-agent/api/Cargo.toml index 046f17574b..17a1294f1f 100644 --- a/sled-agent/api/Cargo.toml +++ b/sled-agent/api/Cargo.toml @@ -9,6 +9,7 @@ workspace = true [dependencies] camino.workspace = true +chrono.workspace = true dropshot.workspace = true nexus-sled-agent-shared.workspace = true omicron-common.workspace = true diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index d9e49a5c56..c53f3ff8f7 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -26,7 +26,7 @@ use omicron_common::{ DisksManagementResult, OmicronPhysicalDisksConfig, }, }; -use omicron_uuid_kinds::{PropolisUuid, ZpoolUuid}; +use omicron_uuid_kinds::{DatasetUuid, PropolisUuid, SupportBundleUuid, ZpoolUuid}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sled_agent_types::{ @@ -154,6 +154,46 @@ pub trait SledAgentApi { rqctx: RequestContext, ) -> Result>, HttpError>; + /// List all support bundles stored on this sled + #[endpoint { + method = GET, + path = "/support-bundles" + }] + async fn support_bundle_list( + rqctx: RequestContext, + ) -> Result>, HttpError>; + + /// Create a service bundle within a particular dataset + #[endpoint { + method = POST, + path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}" + }] + async fn support_bundle_create( + rqctx: RequestContext, + path_params: Path, + body: StreamingBody, + ) -> Result, HttpError>; + + /// Fetch a service bundle from a particular dataset + #[endpoint { + method = GET, + path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}" + }] + async fn support_bundle_get( + rqctx: RequestContext, + path_params: Path, + ) -> Result>, HttpError>; + + /// Delete a service bundle from a particular dataset + #[endpoint { + method = DELETE, + path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}" + }] + async fn support_bundle_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + #[endpoint { method = GET, path = "/omicron-zones", @@ -539,6 +579,32 @@ pub struct VmmPathParam { pub propolis_id: PropolisUuid, } +/// Path parameters for Support Bundle requests (sled agent API) +#[derive(Deserialize, JsonSchema)] +pub struct SupportBundlePathParam { + /// The zpool on which this support bundle was provisioned + pub zpool_id: ZpoolUuid, + + /// The dataset on which this support bundle was provisioned + pub dataset_id: DatasetUuid, + + /// The ID of the support bundle itself + pub support_bundle_id: SupportBundleUuid, +} + +/// Metadata about a support bundle +#[derive(Deserialize, Serialize, JsonSchema)] +pub struct SupportBundleMetadata { + pub support_bundle_id: SupportBundleUuid, + + /// The zpool on which this support bundle was provisioned + pub zpool_id: ZpoolUuid, + /// The dataset on which this support bundle was provisioned + pub dataset_id: DatasetUuid, + + pub time_created: chrono::DateTime, +} + /// Path parameters for Disk requests (sled agent API) #[derive(Deserialize, JsonSchema)] pub struct DiskPathParam { diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 2886c1380d..8ac9350cb9 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -220,6 +220,34 @@ impl SledAgentApi for SledAgentImpl { .map_err(HttpError::from) } + async fn support_bundle_list( + _rqctx: RequestContext, + ) -> Result>, HttpError> { + todo!(); + } + + async fn support_bundle_create( + _rqctx: RequestContext, + _path_params: Path, + _body: StreamingBody, + ) -> Result, HttpError> { + todo!(); + } + + async fn support_bundle_get( + _rqctx: RequestContext, + _path_params: Path, + ) -> Result>, HttpError> { + todo!(); + } + + async fn support_bundle_delete( + _rqctx: RequestContext, + _path_params: Path, + ) -> Result, HttpError> { + todo!(); + } + async fn datasets_put( rqctx: RequestContext, body: TypedBody, diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 53d209725d..d15d6021ab 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -432,6 +432,34 @@ impl SledAgentApi for SledAgentSimImpl { method_unimplemented() } + async fn support_bundle_list( + _rqctx: RequestContext, + ) -> Result>, HttpError> { + method_unimplemented() + } + + async fn support_bundle_create( + _rqctx: RequestContext, + _path_params: Path, + _body: StreamingBody, + ) -> Result, HttpError> { + method_unimplemented() + } + + async fn support_bundle_get( + _rqctx: RequestContext, + _path_params: Path, + ) -> Result>, HttpError> { + method_unimplemented() + } + + async fn support_bundle_delete( + _rqctx: RequestContext, + _path_params: Path, + ) -> Result, HttpError> { + method_unimplemented() + } + async fn zones_list( _rqctx: RequestContext, ) -> Result>, HttpError> { diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs index ba586c03a5..851cea325c 100644 --- a/uuid-kinds/src/lib.rs +++ b/uuid-kinds/src/lib.rs @@ -64,6 +64,7 @@ impl_typed_uuid_kind! { RackReset => "rack_reset", Region => "region", Sled => "sled", + SupportBundle => "support_bundle", TufRepo => "tuf_repo", Upstairs => "upstairs", UpstairsRepair => "upstairs_repair", From 1b7a9afeeabb0aa809757201cf2b6f168f561e92 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 2 Oct 2024 16:56:39 -0700 Subject: [PATCH 02/37] Add API for creating nested datasets within nexus-managed datasets --- common/src/disk.rs | 146 +++++++++++++++- sled-agent/api/src/lib.rs | 4 +- sled-agent/src/http_entrypoints.rs | 5 +- sled-agent/src/sim/http_entrypoints.rs | 3 +- sled-agent/src/sim/sled_agent.rs | 6 +- sled-storage/src/dataset.rs | 3 + sled-storage/src/error.rs | 5 +- sled-storage/src/manager.rs | 227 ++++++++++++++++++++----- 8 files changed, 344 insertions(+), 55 deletions(-) diff --git a/common/src/disk.rs b/common/src/disk.rs index ac9232e257..1884a96615 100644 --- a/common/src/disk.rs +++ b/common/src/disk.rs @@ -12,6 +12,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fmt; +use std::str::FromStr; use uuid::Uuid; use crate::{ @@ -186,6 +187,18 @@ impl GzipLevel { } } +impl FromStr for GzipLevel { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let level = s.parse::()?; + if level < GZIP_LEVEL_MIN || level > GZIP_LEVEL_MAX { + bail!("Invalid gzip compression level: {level}"); + } + Ok(Self(level)) + } +} + #[derive( Copy, Clone, @@ -242,7 +255,30 @@ impl fmt::Display for CompressionAlgorithm { } } -/// Configuration information necessary to request a single dataset +impl FromStr for CompressionAlgorithm { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + use CompressionAlgorithm::*; + let c = match s { + "on" => On, + "" | "off" => Off, + "gzip" => Gzip, + "lz4" => Lz4, + "lzjb" => Lzjb, + "zle" => Zle, + _ => { + let Some(suffix) = s.strip_prefix("gzip-") else { + bail!("Unknown compression algorithm {s}"); + }; + GzipN { level: suffix.parse()? } + } + }; + Ok(c) + } +} + +/// Shared configuration information to request a dataset. #[derive( Clone, Debug, @@ -255,13 +291,7 @@ impl fmt::Display for CompressionAlgorithm { PartialOrd, Ord, )] -pub struct DatasetConfig { - /// The UUID of the dataset being requested - pub id: DatasetUuid, - - /// The dataset's name - pub name: DatasetName, - +pub struct SharedDatasetConfig { /// The compression mode to be used by the dataset pub compression: CompressionAlgorithm, @@ -272,6 +302,106 @@ pub struct DatasetConfig { pub reservation: Option, } +#[derive( + Clone, + Debug, + Deserialize, + Serialize, + JsonSchema, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, +)] +pub struct NestedDatasetLocation { + /// A path, within the dataset root, which is being requested. + pub path: String, + + /// The UUID within which the dataset being requested + pub id: DatasetUuid, + + /// The root in which this dataset is being requested + pub root: DatasetName, +} + +impl NestedDatasetLocation { + pub fn mountpoint(&self, root: &Utf8Path) -> Utf8PathBuf { + let mut path = Utf8Path::new(&self.path); + + // This path must be nested, so we need it to be relative to + // "self.root". However, joining paths in Rust is quirky, + // as it chooses to replace the path entirely if the argument + // to `.join(...)` is absolute. + // + // Here, we "fix" the path to make non-absolute before joining + // the paths. + while path.is_absolute() { + path = path + .strip_prefix("/") + .expect("Path is absolute, but we cannot strip '/' character"); + } + + self.root.mountpoint(root).join(path) + } + + pub fn full_name(&self) -> String { + format!("{}/{}", self.root.full_name(), self.path) + } +} + +// TODO: Does this need to be here? Feels a little like an internal detail... +/// Configuration information necessary to request a single nested dataset. +/// +/// These datasets must be placed within one of the top-level datasets +/// managed directly by Nexus. +#[derive( + Clone, + Debug, + Deserialize, + Serialize, + JsonSchema, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, +)] +pub struct NestedDatasetConfig { + /// Location of this nested dataset + pub name: NestedDatasetLocation, + + /// Configuration of this dataset + #[serde(flatten)] + pub inner: SharedDatasetConfig, +} + +/// Configuration information necessary to request a single dataset. +/// +/// These datasets are tracked directly by Nexus. +#[derive( + Clone, + Debug, + Deserialize, + Serialize, + JsonSchema, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, +)] +pub struct DatasetConfig { + /// The UUID of the dataset being requested + pub id: DatasetUuid, + + /// The dataset's name + pub name: DatasetName, + + #[serde(flatten)] + pub inner: SharedDatasetConfig, +} + #[derive( Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, )] diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index c53f3ff8f7..aea3dfea29 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -26,7 +26,9 @@ use omicron_common::{ DisksManagementResult, OmicronPhysicalDisksConfig, }, }; -use omicron_uuid_kinds::{DatasetUuid, PropolisUuid, SupportBundleUuid, ZpoolUuid}; +use omicron_uuid_kinds::{ + DatasetUuid, PropolisUuid, SupportBundleUuid, ZpoolUuid, +}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sled_agent_types::{ diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 14a5e69559..b7719ad1f3 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -234,13 +234,16 @@ impl SledAgentApi for SledAgentImpl { _path_params: Path, _body: StreamingBody, ) -> Result, HttpError> { + // TODO: I think I need an API to request creation/listing from + // the storage manager within an existing dataset? todo!(); } async fn support_bundle_get( _rqctx: RequestContext, _path_params: Path, - ) -> Result>, HttpError> { + ) -> Result>, HttpError> + { todo!(); } diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index d15d6021ab..f0fff33aa6 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -449,7 +449,8 @@ impl SledAgentApi for SledAgentSimImpl { async fn support_bundle_get( _rqctx: RequestContext, _path_params: Path, - ) -> Result>, HttpError> { + ) -> Result>, HttpError> + { method_unimplemented() } diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 0652c021cb..d500a3b0e1 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -880,9 +880,9 @@ impl SledAgent { name: config.name.full_name(), available: ByteCount::from_kibibytes_u32(0), used: ByteCount::from_kibibytes_u32(0), - quota: config.quota, - reservation: config.reservation, - compression: config.compression.to_string(), + quota: config.inner.quota, + reservation: config.inner.reservation, + compression: config.inner.compression.to_string(), }) .collect::>() }) diff --git a/sled-storage/src/dataset.rs b/sled-storage/src/dataset.rs index a715a33a69..a57e50908a 100644 --- a/sled-storage/src/dataset.rs +++ b/sled-storage/src/dataset.rs @@ -153,6 +153,9 @@ pub enum DatasetError { }, #[error("Failed to make datasets encrypted")] EncryptionMigration(#[from] DatasetEncryptionMigrationError), + + #[error(transparent)] + Other(#[from] anyhow::Error), } /// Ensure that the zpool contains all the datasets we would like it to diff --git a/sled-storage/src/error.rs b/sled-storage/src/error.rs index 988f7f363a..9c78157a14 100644 --- a/sled-storage/src/error.rs +++ b/sled-storage/src/error.rs @@ -9,7 +9,6 @@ use crate::disk::DiskError; use camino::Utf8PathBuf; use omicron_common::api::external::ByteCountRangeError; use omicron_common::api::external::Generation; -use omicron_common::disk::DatasetName; use uuid::Uuid; #[derive(thiserror::Error, Debug)] @@ -58,8 +57,8 @@ pub enum Error { err: uuid::Error, }, - #[error("Dataset {name:?} exists with a different uuid (has {old}, requested {new})")] - UuidMismatch { name: Box, old: Uuid, new: Uuid }, + #[error("Dataset {name} exists with a different uuid (has {old}, requested {new})")] + UuidMismatch { name: String, old: Uuid, new: Uuid }, #[error("Error parsing pool {name}'s size: {err}")] BadPoolSize { diff --git a/sled-storage/src/manager.rs b/sled-storage/src/manager.rs index f5deeb814b..58d60a2e9d 100644 --- a/sled-storage/src/manager.rs +++ b/sled-storage/src/manager.rs @@ -20,7 +20,8 @@ use key_manager::StorageKeyRequester; use omicron_common::disk::{ DatasetConfig, DatasetManagementStatus, DatasetName, DatasetsConfig, DatasetsManagementResult, DiskIdentity, DiskVariant, DisksManagementResult, - OmicronPhysicalDisksConfig, + NestedDatasetConfig, NestedDatasetLocation, OmicronPhysicalDisksConfig, + SharedDatasetConfig, }; use omicron_common::ledger::Ledger; use omicron_uuid_kinds::DatasetUuid; @@ -130,6 +131,17 @@ pub(crate) enum StorageRequest { tx: DebugIgnore>>, }, + NestedDatasetEnsure { + config: NestedDatasetConfig, + tx: DebugIgnore>>, + }, + NestedDatasetList { + name: NestedDatasetLocation, + tx: DebugIgnore< + oneshot::Sender, Error>>, + >, + }, + // Requests to explicitly manage or stop managing a set of devices OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig, @@ -154,6 +166,13 @@ pub(crate) enum StorageRequest { GetLatestResources(DebugIgnore>), } +#[derive(Debug)] +struct DatasetCreationDetails { + zoned: bool, + mountpoint: Mountpoint, + full_name: String, +} + /// A mechanism for interacting with the [`StorageManager`] #[derive(Clone)] pub struct StorageHandle { @@ -281,6 +300,32 @@ impl StorageHandle { rx.await.unwrap() } + pub async fn nested_dataset_ensure( + &self, + config: NestedDatasetConfig, + ) -> Result<(), Error> { + let (tx, rx) = oneshot::channel(); + self.tx + .send(StorageRequest::NestedDatasetEnsure { config, tx: tx.into() }) + .await + .unwrap(); + + rx.await.unwrap() + } + + pub async fn nested_dataset_list( + &self, + name: NestedDatasetLocation, + ) -> Result, Error> { + let (tx, rx) = oneshot::channel(); + self.tx + .send(StorageRequest::NestedDatasetList { name, tx: tx.into() }) + .await + .unwrap(); + + rx.await.unwrap() + } + pub async fn omicron_physical_disks_ensure( &self, config: OmicronPhysicalDisksConfig, @@ -479,6 +524,12 @@ impl StorageManager { StorageRequest::DatasetsList { tx } => { let _ = tx.0.send(self.datasets_config_list().await); } + StorageRequest::NestedDatasetEnsure { config, tx } => { + let _ = tx.0.send(self.nested_dataset_ensure(config).await); + } + StorageRequest::NestedDatasetList { name, tx } => { + let _ = tx.0.send(self.nested_dataset_list(name).await); + } StorageRequest::OmicronPhysicalDisksEnsure { config, tx } => { let _ = tx.0.send(self.omicron_physical_disks_ensure(config).await); @@ -787,7 +838,23 @@ impl StorageManager { err: None, }; - if let Err(err) = self.ensure_dataset(config).await { + let mountpoint_path = + config.name.mountpoint(ZPOOL_MOUNTPOINT_ROOT.into()); + let details = DatasetCreationDetails { + zoned: config.name.dataset().zoned(), + mountpoint: Mountpoint::Path(mountpoint_path), + full_name: config.name.full_name(), + }; + + if let Err(err) = self + .ensure_dataset_with_id( + config.name.pool(), + config.id, + &config.inner, + &details, + ) + .await + { warn!(log, "Failed to ensure dataset"; "dataset" => ?status.dataset_name, "err" => ?err); status.err = Some(err.to_string()); }; @@ -815,6 +882,74 @@ impl StorageManager { } } + // Ensures that a dataset exists, nested somewhere arbitrary within + // a Nexus-controlled dataset. + async fn nested_dataset_ensure( + &mut self, + config: NestedDatasetConfig, + ) -> Result<(), Error> { + let log = self.log.new(o!("request" => "nested_dataset_ensure")); + info!(log, "Ensuring nested dataset"); + + let mountpoint_path = + config.name.mountpoint(ZPOOL_MOUNTPOINT_ROOT.into()); + + let details = DatasetCreationDetails { + zoned: false, + mountpoint: Mountpoint::Path(mountpoint_path), + full_name: config.name.full_name(), + }; + + self.ensure_dataset(config.name.root.pool(), &config.inner, &details) + .await?; + + Ok(()) + } + + // Lists the properties of 'name' and all children within + async fn nested_dataset_list( + &mut self, + name: NestedDatasetLocation, + ) -> Result, Error> { + let log = self.log.new(o!("request" => "nested_dataset_list")); + info!(log, "Listing nested datasets"); + + let full_name = name.full_name(); + let properties = + illumos_utils::zfs::Zfs::get_dataset_properties(&[full_name]) + .map_err(|e| { + warn!( + log, + "Failed to access nested dataset"; + "name" => ?name + ); + crate::dataset::DatasetError::Other(e) + })?; + + let root_path = name.root.full_name(); + Ok(properties + .into_iter() + .filter_map(|prop| { + Some(NestedDatasetConfig { + // The output of our "zfs list" command could be nested away + // from the root - so we actually copy our input to our + // output here, and update the path relative to the input + // root. + name: NestedDatasetLocation { + path: prop.name.strip_prefix(&root_path)?.to_string(), + id: name.id, + root: name.root.clone(), + }, + inner: SharedDatasetConfig { + compression: prop.compression.parse().ok()?, + quota: prop.quota, + reservation: prop.reservation, + }, + }) + }) + .collect()) + } + // Makes an U.2 disk managed by the control plane within [`StorageResources`]. async fn omicron_physical_disks_ensure( &mut self, @@ -986,12 +1121,49 @@ impl StorageManager { } } - // Ensures a dataset exists within a zpool, according to `config`. + // Invokes [Self::ensure_dataset] and also ensures the dataset has an + // expected UUID as a ZFS property. + async fn ensure_dataset_with_id( + &mut self, + zpool: &ZpoolName, + id: DatasetUuid, + config: &SharedDatasetConfig, + details: &DatasetCreationDetails, + ) -> Result<(), Error> { + self.ensure_dataset(zpool, config, details).await?; + + // Ensure the dataset has a usable UUID. + if let Ok(id_str) = Zfs::get_oxide_value(&details.full_name, "uuid") { + if let Ok(found_id) = id_str.parse::() { + if found_id != id { + return Err(Error::UuidMismatch { + name: details.full_name.clone(), + old: found_id.into_untyped_uuid(), + new: id.into_untyped_uuid(), + }); + } + return Ok(()); + } + } + Zfs::set_oxide_value(&details.full_name, "uuid", &id.to_string())?; + Ok(()) + } + + // Ensures a dataset exists within a zpool. + // + // Confirms that the zpool exists and is managed by this sled. async fn ensure_dataset( &mut self, - config: &DatasetConfig, + zpool: &ZpoolName, + config: &SharedDatasetConfig, + details: &DatasetCreationDetails, ) -> Result<(), Error> { - info!(self.log, "ensure_dataset"; "config" => ?config); + info!( + self.log, + "ensure_dataset"; + "config" => ?config, + "details" => ?details, + ); // We can only place datasets within managed disks. // If a disk is attached to this sled, but not a part of the Control @@ -1000,22 +1172,13 @@ impl StorageManager { .resources .disks() .iter_managed() - .any(|(_, disk)| disk.zpool_name() == config.name.pool()) + .any(|(_, disk)| disk.zpool_name() == zpool) { - return Err(Error::ZpoolNotFound(format!( - "{}", - config.name.pool(), - ))); + return Err(Error::ZpoolNotFound(format!("{}", zpool,))); } - let zoned = config.name.dataset().zoned(); - let mountpoint_path = - config.name.mountpoint(ZPOOL_MOUNTPOINT_ROOT.into()); - let mountpoint = Mountpoint::Path(mountpoint_path); - - let fs_name = &config.name.full_name(); + let DatasetCreationDetails { zoned, mountpoint, full_name } = details; let do_format = true; - // The "crypt" dataset needs these details, but should already exist // by the time we're creating datasets inside. let encryption_details = None; @@ -1025,28 +1188,14 @@ impl StorageManager { compression: config.compression, }); Zfs::ensure_filesystem( - fs_name, - mountpoint, - zoned, + &full_name, + mountpoint.clone(), + *zoned, do_format, encryption_details, size_details, None, )?; - // Ensure the dataset has a usable UUID. - if let Ok(id_str) = Zfs::get_oxide_value(&fs_name, "uuid") { - if let Ok(id) = id_str.parse::() { - if id != config.id { - return Err(Error::UuidMismatch { - name: Box::new(config.name.clone()), - old: id.into_untyped_uuid(), - new: config.id.into_untyped_uuid(), - }); - } - return Ok(()); - } - } - Zfs::set_oxide_value(&fs_name, "uuid", &config.id.to_string())?; Ok(()) } @@ -1088,7 +1237,7 @@ impl StorageManager { if let Ok(id) = id_str.parse::() { if id != request.dataset_id { return Err(Error::UuidMismatch { - name: Box::new(request.dataset_name.clone()), + name: request.dataset_name.full_name(), old: id, new: request.dataset_id, }); @@ -1628,9 +1777,11 @@ mod tests { DatasetConfig { id, name, - compression: CompressionAlgorithm::Off, - quota: None, - reservation: None, + inner: SharedDatasetConfig { + compression: CompressionAlgorithm::Off, + quota: None, + reservation: None, + }, }, )]); // "Generation = 1" is reserved as "no requests seen yet", so we jump From 9565d7b197eac7eab3b0874a18fdadf0e14d9e14 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 3 Oct 2024 14:38:45 -0700 Subject: [PATCH 03/37] PUT and GET ALL --- Cargo.lock | 1 + sled-agent/Cargo.toml | 1 + sled-agent/api/src/lib.rs | 30 +++- sled-agent/src/http_entrypoints.rs | 38 ++++- sled-agent/src/sim/http_entrypoints.rs | 2 + sled-agent/src/sled_agent.rs | 225 ++++++++++++++++++++++++- 6 files changed, 277 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40a7516cc5..767afafb6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6904,6 +6904,7 @@ dependencies = [ "serde", "serde_human_bytes", "serde_json", + "sha2", "sha3", "sled-agent-api", "sled-agent-client", diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 3208f1c031..416db6f5c4 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -69,6 +69,7 @@ semver.workspace = true serde.workspace = true serde_human_bytes.workspace = true serde_json = { workspace = true, features = ["raw_value"] } +sha2.workspace = true sha3.workspace = true sled-agent-api.workspace = true sled-agent-client.workspace = true diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index aea3dfea29..930d663c75 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -25,6 +25,7 @@ use omicron_common::{ DatasetsConfig, DatasetsManagementResult, DiskVariant, DisksManagementResult, OmicronPhysicalDisksConfig, }, + update::ArtifactHash, }; use omicron_uuid_kinds::{ DatasetUuid, PropolisUuid, SupportBundleUuid, ZpoolUuid, @@ -156,13 +157,14 @@ pub trait SledAgentApi { rqctx: RequestContext, ) -> Result>, HttpError>; - /// List all support bundles stored on this sled + /// List all support bundles within a particular dataset #[endpoint { method = GET, - path = "/support-bundles" + path = "/support-bundles/{zpool_id}/{dataset_id}" }] async fn support_bundle_list( rqctx: RequestContext, + path_params: Path, ) -> Result>, HttpError>; /// Create a service bundle within a particular dataset @@ -173,6 +175,7 @@ pub trait SledAgentApi { async fn support_bundle_create( rqctx: RequestContext, path_params: Path, + query_params: Query, body: StreamingBody, ) -> Result, HttpError>; @@ -581,6 +584,16 @@ pub struct VmmPathParam { pub propolis_id: PropolisUuid, } +/// Path parameters for Support Bundle requests (sled agent API) +#[derive(Deserialize, JsonSchema)] +pub struct SupportBundleListPathParam { + /// The zpool on which this support bundle was provisioned + pub zpool_id: ZpoolUuid, + + /// The dataset on which this support bundle was provisioned + pub dataset_id: DatasetUuid, +} + /// Path parameters for Support Bundle requests (sled agent API) #[derive(Deserialize, JsonSchema)] pub struct SupportBundlePathParam { @@ -594,17 +607,16 @@ pub struct SupportBundlePathParam { pub support_bundle_id: SupportBundleUuid, } +/// Metadata about a support bundle +#[derive(Deserialize, Serialize, JsonSchema)] +pub struct SupportBundleQueryParams { + pub hash: ArtifactHash, +} + /// Metadata about a support bundle #[derive(Deserialize, Serialize, JsonSchema)] pub struct SupportBundleMetadata { pub support_bundle_id: SupportBundleUuid, - - /// The zpool on which this support bundle was provisioned - pub zpool_id: ZpoolUuid, - /// The dataset on which this support bundle was provisioned - pub dataset_id: DatasetUuid, - - pub time_created: chrono::DateTime, } /// Path parameters for Disk requests (sled agent API) diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index b7719ad1f3..4c512adb39 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -224,19 +224,41 @@ impl SledAgentApi for SledAgentImpl { } async fn support_bundle_list( - _rqctx: RequestContext, + rqctx: RequestContext, + path_params: Path, ) -> Result>, HttpError> { - todo!(); + let sa = rqctx.context(); + + let SupportBundleListPathParam { zpool_id, dataset_id } = + path_params.into_inner(); + + let bundles = sa.support_bundle_list(zpool_id, dataset_id).await?; + + Ok(HttpResponseOk(bundles)) } async fn support_bundle_create( - _rqctx: RequestContext, - _path_params: Path, - _body: StreamingBody, + rqctx: RequestContext, + path_params: Path, + query_params: Query, + body: StreamingBody, ) -> Result, HttpError> { - // TODO: I think I need an API to request creation/listing from - // the storage manager within an existing dataset? - todo!(); + let sa = rqctx.context(); + + let SupportBundlePathParam { zpool_id, dataset_id, support_bundle_id } = + path_params.into_inner(); + let SupportBundleQueryParams { hash } = query_params.into_inner(); + + sa.support_bundle_create( + zpool_id, + dataset_id, + support_bundle_id, + hash, + body, + ) + .await?; + + Ok(HttpResponseCreated(())) } async fn support_bundle_get( diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index f0fff33aa6..fb7328fe57 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -434,6 +434,7 @@ impl SledAgentApi for SledAgentSimImpl { async fn support_bundle_list( _rqctx: RequestContext, + _path_params: Path, ) -> Result>, HttpError> { method_unimplemented() } @@ -441,6 +442,7 @@ impl SledAgentApi for SledAgentSimImpl { async fn support_bundle_create( _rqctx: RequestContext, _path_params: Path, + _query_params: Query, _body: StreamingBody, ) -> Result, HttpError> { method_unimplemented() diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 78a61c894e..770ec51d54 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -23,9 +23,11 @@ use crate::vmm_reservoir::{ReservoirMode, VmmReservoirManager}; use crate::zone_bundle; use crate::zone_bundle::BundleError; use bootstore::schemes::v0 as bootstore; +use camino::Utf8Path; use camino::Utf8PathBuf; use derive_more::From; use dropshot::HttpError; +use dropshot::StreamingBody; use futures::stream::FuturesUnordered; use futures::StreamExt; use illumos_utils::opte::PortManager; @@ -52,11 +54,19 @@ use omicron_common::backoff::{ retry_notify, retry_policy_internal_service_aggressive, BackoffError, }; use omicron_common::disk::{ - DatasetsConfig, DatasetsManagementResult, DisksManagementResult, - OmicronPhysicalDisksConfig, + CompressionAlgorithm, DatasetKind, DatasetName, DatasetsConfig, + DatasetsManagementResult, DisksManagementResult, NestedDatasetConfig, + NestedDatasetLocation, OmicronPhysicalDisksConfig, SharedDatasetConfig, }; +use omicron_common::update::ArtifactHash; +use omicron_common::zpool_name::ZpoolName; use omicron_ddm_admin_client::Client as DdmAdminClient; -use omicron_uuid_kinds::{GenericUuid, PropolisUuid, SledUuid}; +use omicron_uuid_kinds::{ + DatasetUuid, GenericUuid, PropolisUuid, SledUuid, SupportBundleUuid, + ZpoolUuid, +}; +use sha2::{Digest, Sha256}; +use sled_agent_api::SupportBundleMetadata; use sled_agent_api::Zpool; use sled_agent_types::disk::DiskStateRequested; use sled_agent_types::early_networking::EarlyNetworkConfig; @@ -78,8 +88,11 @@ use sled_storage::manager::StorageHandle; use slog::Logger; use sprockets_tls::keys::SprocketsConfig; use std::collections::BTreeMap; +use std::io::Write; use std::net::{Ipv6Addr, SocketAddrV6}; use std::sync::Arc; +use tokio::io::AsyncReadExt; +use tokio::io::AsyncWriteExt; use uuid::Uuid; use illumos_utils::running_zone::ZoneBuilderFactory; @@ -159,6 +172,9 @@ pub enum Error { #[error("Failed to deserialize early network config: {0}")] EarlyNetworkDeserialize(serde_json::Error), + #[error("Support bundle error: {0}")] + SupportBundle(String), + #[error("Zone bundle error: {0}")] ZoneBundle(#[from] BundleError), @@ -842,6 +858,209 @@ impl SledAgent { Ok(datasets_result) } + pub async fn support_bundle_list( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + ) -> Result, Error> { + let root = DatasetName::new( + ZpoolName::new_external(zpool_id), + DatasetKind::Debug, + ); + let root_bundle_name = format!("{}/", root.full_name()); + + let dataset_location = omicron_common::disk::NestedDatasetLocation { + path: String::from(""), + id: dataset_id, + root, + }; + let datasets = + self.storage().nested_dataset_list(dataset_location).await?; + + // TODO: We SHOULD check the mountpaths, to see that the datasets were + // actually created successfully? + // + // Could list them either way, maybe returning a state? + + let bundles = datasets + .into_iter() + .map(|dataset| { + Ok::(SupportBundleMetadata { + support_bundle_id: dataset + .name + .path + .strip_prefix(&root_bundle_name) + .ok_or_else(|| { + Error::SupportBundle(format!( + "Unexpected dataset name: {}", + dataset.name.path + )) + })? + .parse::() + .map_err(|err| { + Error::SupportBundle(format!( + "Cannot parse uuid from dataset: {err}" + )) + })?, + }) + }) + .collect::, Error>>()?; + + Ok(bundles) + } + + pub async fn support_bundle_create( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + expected_hash: ArtifactHash, + body: StreamingBody, + ) -> Result<(), Error> { + let log = self.log.new(o!( + "operation" => "support_bundle_create", + "zpool_id" => zpool_id.to_string(), + "dataset_id" => dataset_id.to_string(), + "bundle_id" => support_bundle_id.to_string(), + )); + + let dataset = NestedDatasetLocation { + path: support_bundle_id.to_string(), + id: dataset_id, + root: DatasetName::new( + ZpoolName::new_external(zpool_id), + DatasetKind::Debug, + ), + }; + // The mounted root of the support bundle dataset + let support_bundle_dir = dataset + .mountpoint(illumos_utils::zpool::ZPOOL_MOUNTPOINT_ROOT.into()); + let support_bundle_path = support_bundle_dir.join("bundle"); + let support_bundle_path_tmp = support_bundle_dir.join("bundle.tmp"); + + // Ensure that the dataset exists. + info!(log, "Ensuring dataset exists for bundle"); + self.storage() + .nested_dataset_ensure(NestedDatasetConfig { + name: dataset, + inner: SharedDatasetConfig { + compression: CompressionAlgorithm::On, + quota: None, + reservation: None, + }, + }) + .await?; + + // Exit early if the support bundle already exists + if tokio::fs::try_exists(&support_bundle_path).await.map_err(|e| { + Error::SupportBundle(format!("Cannot check if bundle exists: {e}")) + })? { + if !Self::sha2_checksum_matches( + &support_bundle_path, + &expected_hash, + ) + .await? + { + warn!(log, "Support bundle exists, but the hash doesn't match"); + return Err(Error::SupportBundle("Hash mismatch".to_string())); + } + + info!(log, "Support bundle already exists"); + return Ok(()); + } + + // Stream the file into the dataset (as a temporary name) + info!(log, "Streaming bundle to storage"); + let tmp_file = tokio::fs::File::create(&support_bundle_path_tmp) + .await + .map_err(|err| { + Error::SupportBundle(format!("Cannot create file: {err}")) + })?; + if let Err(err) = Self::write_and_finalize_bundle( + tmp_file, + &support_bundle_path_tmp, + &support_bundle_path, + expected_hash, + body, + ) + .await + { + warn!(log, "Failed to write bundle to storage"; "error" => ?err); + if let Err(unlink_err) = + tokio::fs::remove_file(support_bundle_path_tmp).await + { + warn!(log, "Failed to unlink bundle after previous error"; "error" => ?unlink_err); + } + return Err(err); + } + + info!(log, "Bundle written successfully"); + Ok(()) + } + + /// Returns the hex, lowercase sha2 checksum of a file at `path`. + async fn sha2_checksum_matches( + path: &Utf8Path, + expected: &ArtifactHash, + ) -> Result { + let mut buf = vec![0u8; 65536]; + let mut file = tokio::fs::File::open(path).await.map_err(|e| { + Error::SupportBundle(format!("Failed to open {path}: {e}")) + })?; + let mut ctx = sha2::Sha256::new(); + loop { + let n = file.read(&mut buf).await.map_err(|e| { + Error::SupportBundle(format!("Failed to read from {path}: {e}")) + })?; + if n == 0 { + break; + } + ctx.write_all(&buf[0..n]).map_err(|e| { + Error::SupportBundle(format!("Failed to hash {path}: {e}")) + })?; + } + + let digest = ctx.finalize(); + return Ok(digest.as_slice() == expected.as_ref()); + } + + // A helper function which streams the contents of a bundle to a file. + // + // If at any point this function fails, the temporary file still exists, + // and should be removed. + async fn write_and_finalize_bundle( + mut tmp_file: tokio::fs::File, + from: &Utf8Path, + to: &Utf8Path, + expected_hash: ArtifactHash, + body: StreamingBody, + ) -> Result<(), Error> { + let stream = body.into_stream(); + futures::pin_mut!(stream); + + // Write the body to the file + let mut hasher = Sha256::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| { + Error::SupportBundle(format!("Failed to stream bundle: {e}")) + })?; + hasher.update(&chunk); + tmp_file.write_all(&chunk).await.map_err(|e| { + Error::SupportBundle(format!("Failed to write bundle: {e}")) + })?; + } + let digest = hasher.finalize(); + if digest.as_slice() != expected_hash.as_ref() { + return Err(Error::SupportBundle(format!("Hash mismatch"))); + } + + // Rename the file to indicate it's ready + tokio::fs::rename(from, to).await.map_err(|e| { + Error::SupportBundle(format!("Cannot finalize bundle: {e}")) + })?; + Ok(()) + } + /// Requests the set of physical disks currently managed by the Sled Agent. /// /// This should be contrasted by the set of disks in the inventory, which From 11069599f0ae9fd7c11e83b8e47ae2f2ae0b497c Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 4 Oct 2024 10:52:24 -0700 Subject: [PATCH 04/37] GET, LIST --- common/src/disk.rs | 6 +- sled-agent/api/src/lib.rs | 9 +- sled-agent/src/http_entrypoints.rs | 40 +++++++-- sled-agent/src/sim/http_entrypoints.rs | 2 +- sled-agent/src/sled_agent.rs | 116 ++++++++++++++++++------- sled-storage/src/error.rs | 3 + sled-storage/src/manager.rs | 42 ++++++++- 7 files changed, 176 insertions(+), 42 deletions(-) diff --git a/common/src/disk.rs b/common/src/disk.rs index 1884a96615..7ca1420681 100644 --- a/common/src/disk.rs +++ b/common/src/disk.rs @@ -346,7 +346,11 @@ impl NestedDatasetLocation { } pub fn full_name(&self) -> String { - format!("{}/{}", self.root.full_name(), self.path) + if self.path.is_empty() { + format!("{}", self.root.full_name()) + } else { + format!("{}/{}", self.root.full_name(), self.path) + } } } diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index 930d663c75..375081258b 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -197,7 +197,7 @@ pub trait SledAgentApi { async fn support_bundle_delete( rqctx: RequestContext, path_params: Path, - ) -> Result, HttpError>; + ) -> Result; #[endpoint { method = GET, @@ -613,10 +613,17 @@ pub struct SupportBundleQueryParams { pub hash: ArtifactHash, } +#[derive(Deserialize, Serialize, JsonSchema, PartialEq)] +pub enum SupportBundleState { + Complete, + Incomplete, +} + /// Metadata about a support bundle #[derive(Deserialize, Serialize, JsonSchema)] pub struct SupportBundleMetadata { pub support_bundle_id: SupportBundleUuid, + pub state: SupportBundleState, } /// Path parameters for Disk requests (sled agent API) diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 4c512adb39..3182ad4914 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -262,18 +262,44 @@ impl SledAgentApi for SledAgentImpl { } async fn support_bundle_get( - _rqctx: RequestContext, - _path_params: Path, + rqctx: RequestContext, + path_params: Path, ) -> Result>, HttpError> { - todo!(); + let sa = rqctx.context(); + let SupportBundlePathParam { zpool_id, dataset_id, support_bundle_id } = + path_params.into_inner(); + + let file = sa + .support_bundle_get(zpool_id, dataset_id, support_bundle_id) + .await?; + + let file_access = hyper_staticfile::vfs::TokioFileAccess::new(file); + let file_stream = + hyper_staticfile::util::FileBytesStream::new(file_access); + let body = Body::wrap(hyper_staticfile::Body::Full(file_stream)); + let body = FreeformBody(body); + let mut response = + HttpResponseHeaders::new_unnamed(HttpResponseOk(body)); + response.headers_mut().append( + http::header::CONTENT_TYPE, + "application/gzip".try_into().unwrap(), + ); + Ok(response) } async fn support_bundle_delete( - _rqctx: RequestContext, - _path_params: Path, - ) -> Result, HttpError> { - todo!(); + rqctx: RequestContext, + path_params: Path, + ) -> Result { + let sa = rqctx.context(); + + let SupportBundlePathParam { zpool_id, dataset_id, support_bundle_id } = + path_params.into_inner(); + + sa.support_bundle_delete(zpool_id, dataset_id, support_bundle_id) + .await?; + Ok(HttpResponseDeleted()) } async fn datasets_put( diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index fb7328fe57..b4aaeece11 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -459,7 +459,7 @@ impl SledAgentApi for SledAgentSimImpl { async fn support_bundle_delete( _rqctx: RequestContext, _path_params: Path, - ) -> Result, HttpError> { + ) -> Result { method_unimplemented() } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 770ec51d54..7e6a63111a 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -67,6 +67,7 @@ use omicron_uuid_kinds::{ }; use sha2::{Digest, Sha256}; use sled_agent_api::SupportBundleMetadata; +use sled_agent_api::SupportBundleState; use sled_agent_api::Zpool; use sled_agent_types::disk::DiskStateRequested; use sled_agent_types::early_networking::EarlyNetworkConfig; @@ -867,8 +868,6 @@ impl SledAgent { ZpoolName::new_external(zpool_id), DatasetKind::Debug, ); - let root_bundle_name = format!("{}/", root.full_name()); - let dataset_location = omicron_common::disk::NestedDatasetLocation { path: String::from(""), id: dataset_id, @@ -877,34 +876,38 @@ impl SledAgent { let datasets = self.storage().nested_dataset_list(dataset_location).await?; - // TODO: We SHOULD check the mountpaths, to see that the datasets were - // actually created successfully? - // - // Could list them either way, maybe returning a state? + let mut bundles = Vec::with_capacity(datasets.len()); + for dataset in datasets { + // We should be able to parse each dataset name as a support bundle UUID + let Ok(support_bundle_id) = + dataset.name.path.parse::() + else { + warn!(self.log, "Dataset path not a UUID"; "path" => dataset.name.path); + continue; + }; - let bundles = datasets - .into_iter() - .map(|dataset| { - Ok::(SupportBundleMetadata { - support_bundle_id: dataset - .name - .path - .strip_prefix(&root_bundle_name) - .ok_or_else(|| { - Error::SupportBundle(format!( - "Unexpected dataset name: {}", - dataset.name.path - )) - })? - .parse::() - .map_err(|err| { - Error::SupportBundle(format!( - "Cannot parse uuid from dataset: {err}" - )) - })?, - }) - }) - .collect::, Error>>()?; + // The dataset for a support bundle exists. + let support_bundle_path = dataset + .name + .mountpoint(illumos_utils::zpool::ZPOOL_MOUNTPOINT_ROOT.into()) + .join("bundle"); + + // Identify whether or not the final "bundle" file exists. + // + // This is a signal that the support bundle has been fully written. + let state = if tokio::fs::try_exists( + &support_bundle_path + ).await.map_err(|e| { + Error::SupportBundle(format!("Cannot check filesystem for {support_bundle_path}: {e}")) + })? { + SupportBundleState::Complete + } else { + SupportBundleState::Incomplete + }; + + let bundle = SupportBundleMetadata { support_bundle_id, state }; + bundles.push(bundle); + } Ok(bundles) } @@ -969,7 +972,8 @@ impl SledAgent { return Ok(()); } - // Stream the file into the dataset (as a temporary name) + // Stream the file into the dataset, first as a temporary file, + // and then renaming to the final location. info!(log, "Streaming bundle to storage"); let tmp_file = tokio::fs::File::create(&support_bundle_path_tmp) .await @@ -998,6 +1002,58 @@ impl SledAgent { Ok(()) } + pub async fn support_bundle_delete( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + ) -> Result<(), Error> { + let log = self.log.new(o!( + "operation" => "support_bundle_delete", + "zpool_id" => zpool_id.to_string(), + "dataset_id" => dataset_id.to_string(), + "bundle_id" => support_bundle_id.to_string(), + )); + info!(log, "Destroying support bundle"); + self.storage() + .nested_dataset_destroy(NestedDatasetLocation { + path: support_bundle_id.to_string(), + id: dataset_id, + root: DatasetName::new( + ZpoolName::new_external(zpool_id), + DatasetKind::Debug, + ), + }) + .await?; + + Ok(()) + } + + pub async fn support_bundle_get( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + ) -> Result { + let dataset = NestedDatasetLocation { + path: support_bundle_id.to_string(), + id: dataset_id, + root: DatasetName::new( + ZpoolName::new_external(zpool_id), + DatasetKind::Debug, + ), + }; + // The mounted root of the support bundle dataset + let support_bundle_dir = dataset + .mountpoint(illumos_utils::zpool::ZPOOL_MOUNTPOINT_ROOT.into()); + let path = support_bundle_dir.join("bundle"); + + let f = tokio::fs::File::open(&path).await.map_err(|e| { + Error::SupportBundle(format!("Failed to open {path}: {e}")) + })?; + Ok(f) + } + /// Returns the hex, lowercase sha2 checksum of a file at `path`. async fn sha2_checksum_matches( path: &Utf8Path, diff --git a/sled-storage/src/error.rs b/sled-storage/src/error.rs index 9c78157a14..351dd7f353 100644 --- a/sled-storage/src/error.rs +++ b/sled-storage/src/error.rs @@ -100,6 +100,9 @@ pub enum Error { #[error("Zpool Not Found: {0}")] ZpoolNotFound(String), + + #[error(transparent)] + Other(#[from] anyhow::Error), } impl From for omicron_common::api::external::Error { diff --git a/sled-storage/src/manager.rs b/sled-storage/src/manager.rs index 58d60a2e9d..75a168ed92 100644 --- a/sled-storage/src/manager.rs +++ b/sled-storage/src/manager.rs @@ -4,13 +4,12 @@ //! The storage manager task -use std::collections::HashSet; - use crate::config::MountConfig; use crate::dataset::CONFIG_DATASET; use crate::disk::RawDisk; use crate::error::Error; use crate::resources::{AllDisks, StorageResources}; +use anyhow::anyhow; use camino::Utf8PathBuf; use debug_ignore::DebugIgnore; use futures::future::FutureExt; @@ -27,6 +26,7 @@ use omicron_common::ledger::Ledger; use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::GenericUuid; use slog::{error, info, o, warn, Logger}; +use std::collections::HashSet; use std::future::Future; use tokio::sync::{mpsc, oneshot, watch}; use tokio::time::{interval, Duration, MissedTickBehavior}; @@ -135,6 +135,10 @@ pub(crate) enum StorageRequest { config: NestedDatasetConfig, tx: DebugIgnore>>, }, + NestedDatasetDestroy { + name: NestedDatasetLocation, + tx: DebugIgnore>>, + }, NestedDatasetList { name: NestedDatasetLocation, tx: DebugIgnore< @@ -313,6 +317,19 @@ impl StorageHandle { rx.await.unwrap() } + pub async fn nested_dataset_destroy( + &self, + name: NestedDatasetLocation, + ) -> Result<(), Error> { + let (tx, rx) = oneshot::channel(); + self.tx + .send(StorageRequest::NestedDatasetDestroy { name, tx: tx.into() }) + .await + .unwrap(); + + rx.await.unwrap() + } + pub async fn nested_dataset_list( &self, name: NestedDatasetLocation, @@ -527,6 +544,9 @@ impl StorageManager { StorageRequest::NestedDatasetEnsure { config, tx } => { let _ = tx.0.send(self.nested_dataset_ensure(config).await); } + StorageRequest::NestedDatasetDestroy { name, tx } => { + let _ = tx.0.send(self.nested_dataset_destroy(name).await); + } StorageRequest::NestedDatasetList { name, tx } => { let _ = tx.0.send(self.nested_dataset_list(name).await); } @@ -906,6 +926,24 @@ impl StorageManager { Ok(()) } + async fn nested_dataset_destroy( + &mut self, + name: NestedDatasetLocation, + ) -> Result<(), Error> { + let log = self.log.new(o!("request" => "nested_dataset_destroy")); + let full_name = name.full_name(); + info!(log, "Destroying nested dataset"; "name" => full_name.clone()); + + if name.path.is_empty() { + let msg = "Cannot destroy nested dataset with empty name"; + warn!(log, "{msg}"); + return Err(anyhow!(msg).into()); + } + + Zfs::destroy_dataset(&full_name).map_err(|e| anyhow!(e))?; + Ok(()) + } + // Lists the properties of 'name' and all children within async fn nested_dataset_list( &mut self, From c23eaa6198934fd312169eeb500f64d56d7a3d6a Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 4 Oct 2024 18:17:01 -0700 Subject: [PATCH 05/37] Add tests, more explicit child-listing options for datasets --- schema/omicron-datasets.json | 2 +- sled-agent/src/sled_agent.rs | 10 +- sled-storage/src/manager.rs | 219 ++++++++++++++++++++++++++++++++++- 3 files changed, 223 insertions(+), 8 deletions(-) diff --git a/schema/omicron-datasets.json b/schema/omicron-datasets.json index dba383d9d4..452b99ac7d 100644 --- a/schema/omicron-datasets.json +++ b/schema/omicron-datasets.json @@ -136,7 +136,7 @@ ] }, "DatasetConfig": { - "description": "Configuration information necessary to request a single dataset", + "description": "Configuration information necessary to request a single dataset.\n\nThese datasets are tracked directly by Nexus.", "type": "object", "required": [ "compression", diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 7e6a63111a..f635df2627 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -85,6 +85,7 @@ use sled_hardware::{underlay, HardwareManager}; use sled_hardware_types::underlay::BootstrapInterface; use sled_hardware_types::Baseboard; use sled_storage::dataset::{CRYPT_DATASET, ZONE_DATASET}; +use sled_storage::manager::NestedDatasetListOptions; use sled_storage::manager::StorageHandle; use slog::Logger; use sprockets_tls::keys::SprocketsConfig; @@ -873,8 +874,13 @@ impl SledAgent { id: dataset_id, root, }; - let datasets = - self.storage().nested_dataset_list(dataset_location).await?; + let datasets = self + .storage() + .nested_dataset_list( + dataset_location, + NestedDatasetListOptions::ChildrenOnly, + ) + .await?; let mut bundles = Vec::with_capacity(datasets.len()); for dataset in datasets { diff --git a/sled-storage/src/manager.rs b/sled-storage/src/manager.rs index 75a168ed92..0fcceec360 100644 --- a/sled-storage/src/manager.rs +++ b/sled-storage/src/manager.rs @@ -100,6 +100,14 @@ pub(crate) struct NewFilesystemRequest { responder: DebugIgnore>>, } +#[derive(Debug)] +pub enum NestedDatasetListOptions { + /// Returns children of the requested dataset, but not the dataset itself. + ChildrenOnly, + /// Returns both the requested dataset as well as all children. + SelfAndChildren, +} + #[derive(Debug)] pub(crate) enum StorageRequest { // Requests to manage which devices the sled considers active. @@ -141,6 +149,7 @@ pub(crate) enum StorageRequest { }, NestedDatasetList { name: NestedDatasetLocation, + options: NestedDatasetListOptions, tx: DebugIgnore< oneshot::Sender, Error>>, >, @@ -333,10 +342,15 @@ impl StorageHandle { pub async fn nested_dataset_list( &self, name: NestedDatasetLocation, + options: NestedDatasetListOptions, ) -> Result, Error> { let (tx, rx) = oneshot::channel(); self.tx - .send(StorageRequest::NestedDatasetList { name, tx: tx.into() }) + .send(StorageRequest::NestedDatasetList { + name, + options, + tx: tx.into(), + }) .await .unwrap(); @@ -547,8 +561,9 @@ impl StorageManager { StorageRequest::NestedDatasetDestroy { name, tx } => { let _ = tx.0.send(self.nested_dataset_destroy(name).await); } - StorageRequest::NestedDatasetList { name, tx } => { - let _ = tx.0.send(self.nested_dataset_list(name).await); + StorageRequest::NestedDatasetList { name, options, tx } => { + let _ = + tx.0.send(self.nested_dataset_list(name, options).await); } StorageRequest::OmicronPhysicalDisksEnsure { config, tx } => { let _ = @@ -948,6 +963,7 @@ impl StorageManager { async fn nested_dataset_list( &mut self, name: NestedDatasetLocation, + options: NestedDatasetListOptions, ) -> Result, Error> { let log = self.log.new(o!("request" => "nested_dataset_list")); info!(log, "Listing nested datasets"); @@ -968,13 +984,27 @@ impl StorageManager { Ok(properties .into_iter() .filter_map(|prop| { + let path = if prop.name == root_path { + match options { + NestedDatasetListOptions::ChildrenOnly => return None, + NestedDatasetListOptions::SelfAndChildren => { + String::new() + } + } + } else { + prop.name + .strip_prefix(&root_path)? + .strip_prefix("/")? + .to_string() + }; + Some(NestedDatasetConfig { // The output of our "zfs list" command could be nested away // from the root - so we actually copy our input to our // output here, and update the path relative to the input // root. name: NestedDatasetLocation { - path: prop.name.strip_prefix(&root_path)?.to_string(), + path, id: name.id, root: name.root.clone(), }, @@ -1853,7 +1883,7 @@ mod tests { // However, calling it with a different input and the same generation // number should fail. config.generation = current_config_generation; - config.datasets.values_mut().next().unwrap().reservation = + config.datasets.values_mut().next().unwrap().inner.reservation = Some(1024.into()); let err = harness.handle().datasets_ensure(config.clone()).await.unwrap_err(); @@ -1869,6 +1899,185 @@ mod tests { harness.cleanup().await; logctx.cleanup_successful(); } + + #[tokio::test] + async fn nested_dataset() { + illumos_utils::USE_MOCKS.store(false, Ordering::SeqCst); + let logctx = test_setup_log("nested_dataset"); + let mut harness = StorageManagerTestHarness::new(&logctx.log).await; + + // Test setup: Add a U.2 and M.2, adopt them into the "control plane" + // for usage. + harness.handle().key_manager_ready().await; + let raw_disks = + harness.add_vdevs(&["u2_under_test.vdev", "m2_helping.vdev"]).await; + let config = harness.make_config(1, &raw_disks); + let result = harness + .handle() + .omicron_physical_disks_ensure(config.clone()) + .await + .expect("Ensuring disks should work after key manager is ready"); + assert!(!result.has_error(), "{:?}", result); + + // Create a dataset on the newly formatted U.2 + // + // NOTE: The choice of "Update" dataset here is kinda arbitrary, + // with a couple caveats: + // + // - This dataset must not be "zoned", as if it is, the nested datasets + // (which are opinionated about being mountable) do not work. + // - Calling "omicron_physical_disks_ensure" automatically creates + // some datasets (see: U2_EXPECTED_DATASETS). We want to avoid + // colliding with those, though in practice it would be fine to re-use + // them. + let id = DatasetUuid::new_v4(); + let zpool_name = ZpoolName::new_external(config.disks[0].pool_id); + let name = DatasetName::new(zpool_name.clone(), DatasetKind::Update); + let root_config = SharedDatasetConfig { + compression: CompressionAlgorithm::Off, + quota: None, + reservation: None, + }; + + let datasets = BTreeMap::from([( + id, + DatasetConfig { + id, + name: name.clone(), + inner: root_config.clone(), + }, + )]); + let generation = Generation::new().next(); + let config = DatasetsConfig { generation, datasets }; + let status = + harness.handle().datasets_ensure(config.clone()).await.unwrap(); + assert!(!status.has_error(), "{:?}", status); + + // Start querying the state of nested datasets. + // + // When we ask about the root of a dataset, we only get information + // about the dataset we're asking for. + let root_location = NestedDatasetLocation { + path: String::new(), + id, + root: name.clone(), + }; + let nested_datasets = harness + .handle() + .nested_dataset_list( + root_location.clone(), + NestedDatasetListOptions::SelfAndChildren, + ) + .await + .unwrap(); + assert_eq!(nested_datasets.len(), 1); + assert_eq!(nested_datasets[0].name, root_location); + + // If we ask about children of this dataset, we see nothing. + let nested_datasets = harness + .handle() + .nested_dataset_list( + root_location.clone(), + NestedDatasetListOptions::ChildrenOnly, + ) + .await + .unwrap(); + assert_eq!(nested_datasets.len(), 0); + + // We can't destroy non-nested datasets through this API + let err = harness + .handle() + .nested_dataset_destroy(root_location.clone()) + .await + .expect_err("Should not be able to delete dataset root"); + assert!( + err.to_string() + .contains("Cannot destroy nested dataset with empty name"), + "{err:?}" + ); + + // Create a nested dataset within the root one + let nested_location = NestedDatasetLocation { + path: "nested".to_string(), + ..root_location.clone() + }; + let nested_config = SharedDatasetConfig { + compression: CompressionAlgorithm::On, + quota: None, + reservation: None, + }; + harness + .handle() + .nested_dataset_ensure(NestedDatasetConfig { + name: nested_location.clone(), + inner: nested_config.clone(), + }) + .await + .unwrap(); + + // We can re-send the ensure request + harness + .handle() + .nested_dataset_ensure(NestedDatasetConfig { + name: nested_location.clone(), + inner: nested_config.clone(), + }) + .await + .expect("Ensuring nested datasets should be idempotent"); + + // We can observe the nested dataset + let nested_datasets = harness + .handle() + .nested_dataset_list( + root_location.clone(), + NestedDatasetListOptions::SelfAndChildren, + ) + .await + .unwrap(); + assert_eq!(nested_datasets.len(), 2); + assert_eq!(nested_datasets[0].name, root_location); + assert_eq!(nested_datasets[1].name, nested_location); + let nested_datasets = harness + .handle() + .nested_dataset_list( + root_location.clone(), + NestedDatasetListOptions::ChildrenOnly, + ) + .await + .unwrap(); + assert_eq!(nested_datasets.len(), 1); + assert_eq!(nested_datasets[0].name, nested_location); + + // We can also destroy the nested dataset + harness + .handle() + .nested_dataset_destroy(nested_location.clone()) + .await + .expect("Should have been able to destroy nested dataset"); + + let err = harness + .handle() + .nested_dataset_destroy(nested_location.clone()) + .await + .expect_err( + "Should not be able to destroy nested dataset a second time", + ); + assert!(err.to_string().contains("Dataset not found"), "{err:?}"); + + // The nested dataset should now be gone + let nested_datasets = harness + .handle() + .nested_dataset_list( + root_location.clone(), + NestedDatasetListOptions::ChildrenOnly, + ) + .await + .unwrap(); + assert_eq!(nested_datasets.len(), 0); + + harness.cleanup().await; + logctx.cleanup_successful(); + } } #[cfg(test)] From 2b78066d2afba78cddf9e69dd494b9cf550028b8 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 4 Oct 2024 18:49:33 -0700 Subject: [PATCH 06/37] clippy --- common/src/disk.rs | 2 +- sled-agent/src/sled_agent.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/disk.rs b/common/src/disk.rs index 7ca1420681..18e311b447 100644 --- a/common/src/disk.rs +++ b/common/src/disk.rs @@ -347,7 +347,7 @@ impl NestedDatasetLocation { pub fn full_name(&self) -> String { if self.path.is_empty() { - format!("{}", self.root.full_name()) + self.root.full_name().to_string() } else { format!("{}/{}", self.root.full_name(), self.path) } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index f635df2627..8cd6cc718c 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -1113,7 +1113,7 @@ impl SledAgent { } let digest = hasher.finalize(); if digest.as_slice() != expected_hash.as_ref() { - return Err(Error::SupportBundle(format!("Hash mismatch"))); + return Err(Error::SupportBundle("Hash mismatch".to_string())); } // Rename the file to indicate it's ready From 76ba0c6fd815c2d56bb9f61d67b34e130ca3f7a2 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 7 Oct 2024 11:34:07 -0700 Subject: [PATCH 07/37] Patch APIs, re-generate openapi specs --- openapi/sled-agent.json | 243 ++++++++++++++++++++++++- sled-agent/api/src/lib.rs | 3 +- sled-agent/src/http_entrypoints.rs | 6 +- sled-agent/src/sim/http_entrypoints.rs | 2 +- sled-agent/src/sled_agent.rs | 14 +- 5 files changed, 259 insertions(+), 9 deletions(-) diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index c6f029ea9a..5530eb65d4 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -524,6 +524,220 @@ } } }, + "/support-bundles/{zpool_id}/{dataset_id}": { + "get": { + "summary": "List all support bundles within a particular dataset", + "operationId": "support_bundle_list", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForDatasetKind" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForZpoolKind" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_SupportBundleMetadata", + "type": "array", + "items": { + "$ref": "#/components/schemas/SupportBundleMetadata" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}": { + "get": { + "summary": "Fetch a service bundle from a particular dataset", + "operationId": "support_bundle_get", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForDatasetKind" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForSupportBundleKind" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForZpoolKind" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "summary": "Create a service bundle within a particular dataset", + "operationId": "support_bundle_create", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForDatasetKind" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForSupportBundleKind" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForZpoolKind" + } + }, + { + "in": "query", + "name": "hash", + "required": true, + "schema": { + "type": "string", + "format": "hex string (32 bytes)" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SupportBundleMetadata" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Delete a service bundle from a particular dataset", + "operationId": "support_bundle_delete", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForDatasetKind" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForSupportBundleKind" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForZpoolKind" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/switch-ports": { "post": { "operationId": "uplink_ensure", @@ -2249,7 +2463,7 @@ ] }, "DatasetConfig": { - "description": "Configuration information necessary to request a single dataset", + "description": "Configuration information necessary to request a single dataset.\n\nThese datasets are tracked directly by Nexus.", "type": "object", "properties": { "compression": { @@ -5045,6 +5259,29 @@ "format": "uint8", "minimum": 0 }, + "SupportBundleMetadata": { + "description": "Metadata about a support bundle", + "type": "object", + "properties": { + "state": { + "$ref": "#/components/schemas/SupportBundleState" + }, + "support_bundle_id": { + "$ref": "#/components/schemas/TypedUuidForSupportBundleKind" + } + }, + "required": [ + "state", + "support_bundle_id" + ] + }, + "SupportBundleState": { + "type": "string", + "enum": [ + "complete", + "incomplete" + ] + }, "SwitchLocation": { "description": "Identifies switch physical location", "oneOf": [ @@ -5139,6 +5376,10 @@ "type": "string", "format": "uuid" }, + "TypedUuidForSupportBundleKind": { + "type": "string", + "format": "uuid" + }, "TypedUuidForZpoolKind": { "type": "string", "format": "uuid" diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index 375081258b..cf821f4d97 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -177,7 +177,7 @@ pub trait SledAgentApi { path_params: Path, query_params: Query, body: StreamingBody, - ) -> Result, HttpError>; + ) -> Result, HttpError>; /// Fetch a service bundle from a particular dataset #[endpoint { @@ -614,6 +614,7 @@ pub struct SupportBundleQueryParams { } #[derive(Deserialize, Serialize, JsonSchema, PartialEq)] +#[serde(rename_all = "snake_case")] pub enum SupportBundleState { Complete, Incomplete, diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 3182ad4914..c4f0de18d6 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -242,14 +242,14 @@ impl SledAgentApi for SledAgentImpl { path_params: Path, query_params: Query, body: StreamingBody, - ) -> Result, HttpError> { + ) -> Result, HttpError> { let sa = rqctx.context(); let SupportBundlePathParam { zpool_id, dataset_id, support_bundle_id } = path_params.into_inner(); let SupportBundleQueryParams { hash } = query_params.into_inner(); - sa.support_bundle_create( + let metadata = sa.support_bundle_create( zpool_id, dataset_id, support_bundle_id, @@ -258,7 +258,7 @@ impl SledAgentApi for SledAgentImpl { ) .await?; - Ok(HttpResponseCreated(())) + Ok(HttpResponseCreated(metadata)) } async fn support_bundle_get( diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index b4aaeece11..aa655dc9db 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -444,7 +444,7 @@ impl SledAgentApi for SledAgentSimImpl { _path_params: Path, _query_params: Query, _body: StreamingBody, - ) -> Result, HttpError> { + ) -> Result, HttpError> { method_unimplemented() } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 8cd6cc718c..1a38d8a901 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -925,7 +925,7 @@ impl SledAgent { support_bundle_id: SupportBundleUuid, expected_hash: ArtifactHash, body: StreamingBody, - ) -> Result<(), Error> { + ) -> Result { let log = self.log.new(o!( "operation" => "support_bundle_create", "zpool_id" => zpool_id.to_string(), @@ -975,7 +975,11 @@ impl SledAgent { } info!(log, "Support bundle already exists"); - return Ok(()); + let metadata = SupportBundleMetadata { + support_bundle_id, + state: SupportBundleState::Complete, + }; + return Ok(metadata); } // Stream the file into the dataset, first as a temporary file, @@ -1005,7 +1009,11 @@ impl SledAgent { } info!(log, "Bundle written successfully"); - Ok(()) + let metadata = SupportBundleMetadata { + support_bundle_id, + state: SupportBundleState::Complete, + }; + Ok(metadata) } pub async fn support_bundle_delete( From 26d364021218cdb7ba8df1e5ca655d8479a52cbc Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 7 Oct 2024 12:02:10 -0700 Subject: [PATCH 08/37] fmt --- sled-agent/src/http_entrypoints.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index c4f0de18d6..b99e10745d 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -249,14 +249,15 @@ impl SledAgentApi for SledAgentImpl { path_params.into_inner(); let SupportBundleQueryParams { hash } = query_params.into_inner(); - let metadata = sa.support_bundle_create( - zpool_id, - dataset_id, - support_bundle_id, - hash, - body, - ) - .await?; + let metadata = sa + .support_bundle_create( + zpool_id, + dataset_id, + support_bundle_id, + hash, + body, + ) + .await?; Ok(HttpResponseCreated(metadata)) } From 77ceebed221238d548715f28d3072d3ef1bdfd4a Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 7 Oct 2024 13:25:47 -0700 Subject: [PATCH 09/37] More specific name for QueryParams --- sled-agent/api/src/lib.rs | 4 ++-- sled-agent/src/http_entrypoints.rs | 4 ++-- sled-agent/src/sim/http_entrypoints.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index cf821f4d97..7ad8fd7499 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -175,7 +175,7 @@ pub trait SledAgentApi { async fn support_bundle_create( rqctx: RequestContext, path_params: Path, - query_params: Query, + query_params: Query, body: StreamingBody, ) -> Result, HttpError>; @@ -609,7 +609,7 @@ pub struct SupportBundlePathParam { /// Metadata about a support bundle #[derive(Deserialize, Serialize, JsonSchema)] -pub struct SupportBundleQueryParams { +pub struct SupportBundleCreateQueryParams { pub hash: ArtifactHash, } diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index b99e10745d..126c101e64 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -240,14 +240,14 @@ impl SledAgentApi for SledAgentImpl { async fn support_bundle_create( rqctx: RequestContext, path_params: Path, - query_params: Query, + query_params: Query, body: StreamingBody, ) -> Result, HttpError> { let sa = rqctx.context(); let SupportBundlePathParam { zpool_id, dataset_id, support_bundle_id } = path_params.into_inner(); - let SupportBundleQueryParams { hash } = query_params.into_inner(); + let SupportBundleCreateQueryParams { hash } = query_params.into_inner(); let metadata = sa .support_bundle_create( diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index aa655dc9db..97f1d979b2 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -442,7 +442,7 @@ impl SledAgentApi for SledAgentSimImpl { async fn support_bundle_create( _rqctx: RequestContext, _path_params: Path, - _query_params: Query, + _query_params: Query, _body: StreamingBody, ) -> Result, HttpError> { method_unimplemented() From 6b8099a19e47a924434040592285e80ace220a4a Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 7 Oct 2024 17:28:20 -0700 Subject: [PATCH 10/37] Implemented simulated sled agent side of support bundle (IDs only) --- nexus/test-utils/src/resource_helpers.rs | 18 +- sled-agent/src/sim/http_entrypoints.rs | 98 ++++++--- sled-agent/src/sim/server.rs | 12 +- sled-agent/src/sim/sled_agent.rs | 115 +++++++++- sled-agent/src/sim/storage.rs | 264 +++++++++++++++++++---- 5 files changed, 416 insertions(+), 91 deletions(-) diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index 8b1d0e5f6b..8f92a1fa63 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -45,7 +45,9 @@ use omicron_common::disk::DiskIdentity; use omicron_sled_agent::sim::SledAgent; use omicron_test_utils::dev::poll::wait_for_condition; use omicron_test_utils::dev::poll::CondCheckError; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use oxnet::Ipv4Net; @@ -852,7 +854,7 @@ pub async fn projects_list( } pub struct TestDataset { - pub id: Uuid, + pub id: DatasetUuid, } pub struct TestZpool { @@ -1012,9 +1014,9 @@ impl<'a, N: NexusServer> DiskTest<'a, N> { pub async fn add_zpool_with_dataset(&mut self, sled_id: SledUuid) { self.add_zpool_with_dataset_ext( sled_id, - Uuid::new_v4(), + PhysicalDiskUuid::new_v4(), ZpoolUuid::new_v4(), - Uuid::new_v4(), + DatasetUuid::new_v4(), Self::DEFAULT_ZPOOL_SIZE_GIB, ) .await @@ -1045,9 +1047,9 @@ impl<'a, N: NexusServer> DiskTest<'a, N> { pub async fn add_zpool_with_dataset_ext( &mut self, sled_id: SledUuid, - physical_disk_id: Uuid, + physical_disk_id: PhysicalDiskUuid, zpool_id: ZpoolUuid, - dataset_id: Uuid, + dataset_id: DatasetUuid, gibibytes: u32, ) { let cptestctx = self.cptestctx; @@ -1068,7 +1070,7 @@ impl<'a, N: NexusServer> DiskTest<'a, N> { let physical_disk_request = nexus_types::internal_api::params::PhysicalDiskPutRequest { - id: physical_disk_id, + id: *physical_disk_id.as_untyped_uuid(), vendor: disk_identity.vendor.clone(), serial: disk_identity.serial.clone(), model: disk_identity.model.clone(), @@ -1080,7 +1082,7 @@ impl<'a, N: NexusServer> DiskTest<'a, N> { let zpool_request = nexus_types::internal_api::params::ZpoolPutRequest { id: zpool.id.into_untyped_uuid(), - physical_disk_id, + physical_disk_id: *physical_disk_id.as_untyped_uuid(), sled_id: sled_id.into_untyped_uuid(), }; @@ -1140,7 +1142,7 @@ impl<'a, N: NexusServer> DiskTest<'a, N> { .upsert_crucible_dataset( physical_disk_request.clone(), zpool_request.clone(), - dataset.id, + *dataset.id.as_untyped_uuid(), address, ) .await; diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 97f1d979b2..c356c91d44 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -365,6 +365,73 @@ impl SledAgentApi for SledAgentSimImpl { Ok(HttpResponseUpdatedNoContent()) } + async fn support_bundle_list( + rqctx: RequestContext, + path_params: Path, + ) -> Result>, HttpError> { + let sa = rqctx.context(); + + let SupportBundleListPathParam { zpool_id, dataset_id } = + path_params.into_inner(); + + let bundles = sa.support_bundle_list(zpool_id, dataset_id).await?; + Ok(HttpResponseOk(bundles)) + } + + async fn support_bundle_create( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + _body: StreamingBody, + ) -> Result, HttpError> { + let sa = rqctx.context(); + + let SupportBundlePathParam { zpool_id, dataset_id, support_bundle_id } = + path_params.into_inner(); + let SupportBundleCreateQueryParams { hash } = query_params.into_inner(); + + Ok(HttpResponseCreated( + sa.support_bundle_create( + zpool_id, + dataset_id, + support_bundle_id, + hash, + ) + .await?, + )) + } + + async fn support_bundle_get( + rqctx: RequestContext, + path_params: Path, + ) -> Result>, HttpError> + { + let sa = rqctx.context(); + let SupportBundlePathParam { zpool_id, dataset_id, support_bundle_id } = + path_params.into_inner(); + + sa.support_bundle_get(zpool_id, dataset_id, support_bundle_id).await?; + + let body = FreeformBody("simulated support bundle; do not eat".into()); + let response = HttpResponseHeaders::new_unnamed(HttpResponseOk(body)); + Ok(response) + } + + async fn support_bundle_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result { + let sa = rqctx.context(); + + let SupportBundlePathParam { zpool_id, dataset_id, support_bundle_id } = + path_params.into_inner(); + + sa.support_bundle_delete(zpool_id, dataset_id, support_bundle_id) + .await?; + + Ok(HttpResponseDeleted()) + } + // --- Unimplemented endpoints --- async fn zone_bundle_list_all( @@ -432,37 +499,6 @@ impl SledAgentApi for SledAgentSimImpl { method_unimplemented() } - async fn support_bundle_list( - _rqctx: RequestContext, - _path_params: Path, - ) -> Result>, HttpError> { - method_unimplemented() - } - - async fn support_bundle_create( - _rqctx: RequestContext, - _path_params: Path, - _query_params: Query, - _body: StreamingBody, - ) -> Result, HttpError> { - method_unimplemented() - } - - async fn support_bundle_get( - _rqctx: RequestContext, - _path_params: Path, - ) -> Result>, HttpError> - { - method_unimplemented() - } - - async fn support_bundle_delete( - _rqctx: RequestContext, - _path_params: Path, - ) -> Result { - method_unimplemented() - } - async fn zones_list( _rqctx: RequestContext, ) -> Result>, HttpError> { diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index 03efa56369..9784675517 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -41,8 +41,10 @@ use omicron_common::backoff::{ }; use omicron_common::disk::DiskIdentity; use omicron_common::FileKv; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; +use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::ZpoolUuid; use oxnet::Ipv6Net; use sled_agent_types::rack_init::RecoverySiloConfig; @@ -177,7 +179,7 @@ impl Server { // Crucible dataset for each. This emulates the setup we expect to have // on the physical rack. for zpool in &config.storage.zpools { - let physical_disk_id = Uuid::new_v4(); + let physical_disk_id = PhysicalDiskUuid::new_v4(); let zpool_id = ZpoolUuid::new_v4(); let vendor = "synthetic-vendor".to_string(); let serial = format!("synthetic-serial-{zpool_id}"); @@ -196,13 +198,13 @@ impl Server { sled_agent .create_zpool(zpool_id, physical_disk_id, zpool.size) .await; - let dataset_id = Uuid::new_v4(); + let dataset_id = DatasetUuid::new_v4(); let address = sled_agent.create_crucible_dataset(zpool_id, dataset_id).await; datasets.push(NexusTypes::DatasetCreateRequest { zpool_id: zpool_id.into_untyped_uuid(), - dataset_id, + dataset_id: dataset_id.into_untyped_uuid(), request: NexusTypes::DatasetPutRequest { address: address.to_string(), kind: DatasetKind::Crucible, @@ -522,11 +524,11 @@ pub async fn run_standalone_server( for zpool in &zpools { let zpool_id = ZpoolUuid::from_untyped_uuid(zpool.id); for (dataset_id, address) in - server.sled_agent.get_datasets(zpool_id).await + server.sled_agent.get_crucible_datasets(zpool_id).await { datasets.push(NexusTypes::DatasetCreateRequest { zpool_id: zpool.id, - dataset_id, + dataset_id: *dataset_id.as_untyped_uuid(), request: NexusTypes::DatasetPutRequest { address: address.to_string(), kind: DatasetKind::Crucible, diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index d500a3b0e1..5ddafc05e0 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -36,12 +36,18 @@ use omicron_common::disk::{ DatasetsConfig, DatasetsManagementResult, DiskIdentity, DiskVariant, DisksManagementResult, OmicronPhysicalDisksConfig, }; -use omicron_uuid_kinds::{GenericUuid, PropolisUuid, SledUuid, ZpoolUuid}; +use omicron_common::update::ArtifactHash; +use omicron_uuid_kinds::{ + DatasetUuid, GenericUuid, PhysicalDiskUuid, PropolisUuid, SledUuid, + SupportBundleUuid, ZpoolUuid, +}; use oxnet::Ipv6Net; use propolis_client::{ types::VolumeConstructionRequest, Client as PropolisClient, }; use propolis_mock_server::Context as PropolisContext; +use sled_agent_api::SupportBundleMetadata; +use sled_agent_api::SupportBundleState; use sled_agent_types::disk::DiskStateRequested; use sled_agent_types::early_networking::{ EarlyNetworkConfig, EarlyNetworkConfigBody, @@ -579,7 +585,7 @@ impl SledAgent { /// Adds a Physical Disk to the simulated sled agent. pub async fn create_external_physical_disk( &self, - id: Uuid, + id: PhysicalDiskUuid, identity: DiskIdentity, ) { let variant = DiskVariant::U2; @@ -602,18 +608,18 @@ impl SledAgent { self.storage.lock().await.get_all_zpools() } - pub async fn get_datasets( + pub async fn get_crucible_datasets( &self, zpool_id: ZpoolUuid, - ) -> Vec<(Uuid, SocketAddr)> { - self.storage.lock().await.get_all_datasets(zpool_id) + ) -> Vec<(DatasetUuid, SocketAddr)> { + self.storage.lock().await.get_all_crucible_datasets(zpool_id) } /// Adds a Zpool to the simulated sled agent. pub async fn create_zpool( &self, id: ZpoolUuid, - physical_disk_id: Uuid, + physical_disk_id: PhysicalDiskUuid, size: u64, ) { self.storage @@ -623,22 +629,43 @@ impl SledAgent { .await; } + /// Adds a debug dataset within a zpool + pub async fn create_debug_dataset( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + ) { + self.storage + .lock() + .await + .insert_debug_dataset(zpool_id, dataset_id) + .await + } + /// Adds a Crucible Dataset within a zpool. pub async fn create_crucible_dataset( &self, zpool_id: ZpoolUuid, - dataset_id: Uuid, + dataset_id: DatasetUuid, ) -> SocketAddr { - self.storage.lock().await.insert_dataset(zpool_id, dataset_id).await + self.storage + .lock() + .await + .insert_crucible_dataset(zpool_id, dataset_id) + .await } /// Returns a crucible dataset within a particular zpool. pub async fn get_crucible_dataset( &self, zpool_id: ZpoolUuid, - dataset_id: Uuid, + dataset_id: DatasetUuid, ) -> Arc { - self.storage.lock().await.get_dataset(zpool_id, dataset_id).await + self.storage + .lock() + .await + .get_crucible_dataset(zpool_id, dataset_id) + .await } /// Issue a snapshot request for a Crucible disk attached to an instance. @@ -890,6 +917,68 @@ impl SledAgent { }) } + pub async fn support_bundle_list( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + ) -> Result, HttpError> { + self.storage + .lock() + .await + .support_bundle_list(zpool_id, dataset_id) + .await + } + + pub async fn support_bundle_create( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + expected_hash: ArtifactHash, + ) -> Result { + self.storage + .lock() + .await + .support_bundle_create( + zpool_id, + dataset_id, + support_bundle_id, + expected_hash, + ) + .await?; + + Ok(SupportBundleMetadata { + support_bundle_id, + state: SupportBundleState::Complete, + }) + } + + pub async fn support_bundle_get( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + ) -> Result<(), HttpError> { + self.storage + .lock() + .await + .support_bundle_exists(zpool_id, dataset_id, support_bundle_id) + .await + } + + pub async fn support_bundle_delete( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + ) -> Result<(), HttpError> { + self.storage + .lock() + .await + .support_bundle_delete(zpool_id, dataset_id, support_bundle_id) + .await + } + pub async fn datasets_ensure( &self, config: DatasetsConfig, @@ -927,7 +1016,11 @@ impl SledAgent { *self.fake_zones.lock().await = requested_zones; } - pub async fn drop_dataset(&self, zpool_id: ZpoolUuid, dataset_id: Uuid) { + pub async fn drop_dataset( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + ) { self.storage.lock().await.drop_dataset(zpool_id, dataset_id) } diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 589ba87700..0c5f8aedea 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -28,12 +28,18 @@ use omicron_common::disk::DiskManagementStatus; use omicron_common::disk::DiskVariant; use omicron_common::disk::DisksManagementResult; use omicron_common::disk::OmicronPhysicalDisksConfig; +use omicron_common::update::ArtifactHash; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; +use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::PropolisUuid; +use omicron_uuid_kinds::SupportBundleUuid; use omicron_uuid_kinds::ZpoolUuid; use propolis_client::types::VolumeConstructionRequest; use serde::Serialize; +use sled_agent_api::SupportBundleMetadata; +use sled_agent_api::SupportBundleState; use slog::Logger; use std::collections::HashMap; use std::collections::HashSet; @@ -808,39 +814,68 @@ impl CrucibleServer { } } +#[derive(Default)] +pub(crate) struct DebugData { + bundles: HashMap, +} + pub(crate) struct PhysicalDisk { pub(crate) identity: DiskIdentity, pub(crate) variant: DiskVariant, pub(crate) slot: i64, } +/// Describes data being simulated within a dataset. +pub(crate) enum DatasetContents { + Crucible(CrucibleServer), + Debug(DebugData), +} + pub(crate) struct Zpool { id: ZpoolUuid, - physical_disk_id: Uuid, + physical_disk_id: PhysicalDiskUuid, total_size: u64, - datasets: HashMap, + datasets: HashMap, } impl Zpool { - fn new(id: ZpoolUuid, physical_disk_id: Uuid, total_size: u64) -> Self { + fn new( + id: ZpoolUuid, + physical_disk_id: PhysicalDiskUuid, + total_size: u64, + ) -> Self { Zpool { id, physical_disk_id, total_size, datasets: HashMap::new() } } - fn insert_dataset( + fn insert_debug_dataset(&mut self, id: DatasetUuid) { + self.datasets.insert(id, DatasetContents::Debug(DebugData::default())); + } + + fn insert_crucible_dataset( &mut self, log: &Logger, - id: Uuid, + id: DatasetUuid, crucible_ip: IpAddr, start_port: u16, end_port: u16, ) -> &CrucibleServer { self.datasets.insert( id, - CrucibleServer::new(log, crucible_ip, start_port, end_port), + DatasetContents::Crucible(CrucibleServer::new( + log, + crucible_ip, + start_port, + end_port, + )), ); - self.datasets + let DatasetContents::Crucible(crucible) = self + .datasets .get(&id) .expect("Failed to get the dataset we just inserted") + else { + panic!("Should have just inserted Crucible dataset"); + }; + crucible } pub fn total_size(&self) -> u64 { @@ -852,10 +887,12 @@ impl Zpool { region_id: Uuid, ) -> Option> { for dataset in self.datasets.values() { - for region in &dataset.data().list().await { - let id = Uuid::from_str(®ion.id.0).unwrap(); - if id == region_id { - return Some(dataset.data()); + if let DatasetContents::Crucible(dataset) = dataset { + for region in &dataset.data().list().await { + let id = Uuid::from_str(®ion.id.0).unwrap(); + if id == region_id { + return Some(dataset.data()); + } } } } @@ -867,13 +904,15 @@ impl Zpool { let mut regions = vec![]; for dataset in self.datasets.values() { - for region in &dataset.data().list().await { - if region.state == State::Destroyed { - continue; - } + if let DatasetContents::Crucible(dataset) = dataset { + for region in &dataset.data().list().await { + if region.state == State::Destroyed { + continue; + } - if port == region.port_number { - regions.push(region.clone()); + if port == region.port_number { + regions.push(region.clone()); + } } } } @@ -884,7 +923,7 @@ impl Zpool { regions.pop() } - pub fn drop_dataset(&mut self, id: Uuid) { + pub fn drop_dataset(&mut self, id: DatasetUuid) { let _ = self.datasets.remove(&id).expect("Failed to get the dataset"); } } @@ -895,7 +934,7 @@ pub struct Storage { log: Logger, config: Option, dataset_config: Option, - physical_disks: HashMap, + physical_disks: HashMap, next_disk_slot: i64, zpools: HashMap, crucible_ip: IpAddr, @@ -918,7 +957,7 @@ impl Storage { } /// Returns an immutable reference to all (currently known) physical disks - pub fn physical_disks(&self) -> &HashMap { + pub fn physical_disks(&self) -> &HashMap { &self.physical_disks } @@ -1002,7 +1041,7 @@ impl Storage { pub async fn insert_physical_disk( &mut self, - id: Uuid, + id: PhysicalDiskUuid, identity: DiskIdentity, variant: DiskVariant, ) { @@ -1016,7 +1055,7 @@ impl Storage { pub async fn insert_zpool( &mut self, zpool_id: ZpoolUuid, - disk_id: Uuid, + disk_id: PhysicalDiskUuid, size: u64, ) { // Update our local data @@ -1028,18 +1067,153 @@ impl Storage { &self.zpools } - /// Adds a Dataset to the sled's simulated storage. - pub async fn insert_dataset( + fn get_debug_dataset( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + ) -> Result<&DebugData, HttpError> { + let Some(zpool) = self.zpools.get(&zpool_id) else { + return Err(HttpError::for_not_found( + None, + format!("zpool does not exist {zpool_id}"), + )); + }; + let Some(dataset) = zpool.datasets.get(&dataset_id) else { + return Err(HttpError::for_not_found( + None, + format!("dataset does not exist {dataset_id}"), + )); + }; + + let DatasetContents::Debug(debug) = dataset else { + return Err(HttpError::for_bad_request( + None, + format!("Not a debug dataset"), + )); + }; + + Ok(debug) + } + + fn get_debug_dataset_mut( + &mut self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + ) -> Result<&mut DebugData, HttpError> { + let Some(zpool) = self.zpools.get_mut(&zpool_id) else { + return Err(HttpError::for_not_found( + None, + format!("zpool does not exist {zpool_id}"), + )); + }; + let Some(dataset) = zpool.datasets.get_mut(&dataset_id) else { + return Err(HttpError::for_not_found( + None, + format!("dataset does not exist {dataset_id}"), + )); + }; + + let DatasetContents::Debug(debug) = dataset else { + return Err(HttpError::for_bad_request( + None, + format!("Not a debug dataset"), + )); + }; + + Ok(debug) + } + + pub async fn support_bundle_list( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + ) -> Result, HttpError> { + let debug = self.get_debug_dataset(zpool_id, dataset_id)?; + + Ok(debug + .bundles + .keys() + .map(|id| SupportBundleMetadata { + support_bundle_id: *id, + state: SupportBundleState::Complete, + }) + .collect()) + } + + pub async fn support_bundle_create( + &mut self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + hash: ArtifactHash, + ) -> Result<(), HttpError> { + let debug = self.get_debug_dataset_mut(zpool_id, dataset_id)?; + + // This is for the simulated server, so we totally ignore the "contents" + // of the bundle and just accept that it should exist. + debug.bundles.insert(support_bundle_id, hash); + + Ok(()) + } + + pub async fn support_bundle_exists( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + ) -> Result<(), HttpError> { + let debug = self.get_debug_dataset(zpool_id, dataset_id)?; + + if !debug.bundles.contains_key(&support_bundle_id) { + return Err(HttpError::for_not_found( + None, + format!("Support bundle not found {support_bundle_id}"), + )); + } + Ok(()) + } + + pub async fn support_bundle_delete( + &mut self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + ) -> Result<(), HttpError> { + let debug = self.get_debug_dataset_mut(zpool_id, dataset_id)?; + + if debug.bundles.remove(&support_bundle_id).is_none() { + return Err(HttpError::for_not_found( + None, + format!("Support bundle not found {support_bundle_id}"), + )); + } + Ok(()) + } + + /// Adds a debug dataset to the sled's simulated storage + pub async fn insert_debug_dataset( &mut self, zpool_id: ZpoolUuid, - dataset_id: Uuid, + dataset_id: DatasetUuid, + ) { + self.zpools + .get_mut(&zpool_id) + .expect("Zpool does not exist") + .insert_debug_dataset(dataset_id); + } + + /// Adds a Crucible dataset to the sled's simulated storage. + pub async fn insert_crucible_dataset( + &mut self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, ) -> SocketAddr { // Update our local data let dataset = self .zpools .get_mut(&zpool_id) .expect("Zpool does not exist") - .insert_dataset( + .insert_crucible_dataset( &self.log, dataset_id, self.crucible_ip, @@ -1068,7 +1242,7 @@ impl Storage { }; nexus_client::types::PhysicalDiskPutRequest { - id: *id, + id: *id.as_untyped_uuid(), vendor: disk.identity.vendor.clone(), serial: disk.identity.serial.clone(), model: disk.identity.model.clone(), @@ -1085,37 +1259,51 @@ impl Storage { .map(|pool| nexus_client::types::ZpoolPutRequest { id: pool.id.into_untyped_uuid(), sled_id: self.sled_id, - physical_disk_id: pool.physical_disk_id, + physical_disk_id: *pool.physical_disk_id.as_untyped_uuid(), }) .collect() } - pub fn get_all_datasets( + pub fn get_all_crucible_datasets( &self, zpool_id: ZpoolUuid, - ) -> Vec<(Uuid, SocketAddr)> { + ) -> Vec<(DatasetUuid, SocketAddr)> { let zpool = self.zpools.get(&zpool_id).expect("Zpool does not exist"); zpool .datasets .iter() - .map(|(id, server)| (*id, server.address())) + .filter_map(|(id, dataset)| match dataset { + DatasetContents::Crucible(server) => { + Some((*id, server.address())) + } + _ => None, + }) .collect() } pub async fn get_dataset( &self, zpool_id: ZpoolUuid, - dataset_id: Uuid, - ) -> Arc { + dataset_id: DatasetUuid, + ) -> &DatasetContents { self.zpools .get(&zpool_id) .expect("Zpool does not exist") .datasets .get(&dataset_id) .expect("Dataset does not exist") - .data - .clone() + } + + pub async fn get_crucible_dataset( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + ) -> Arc { + match self.get_dataset(zpool_id, dataset_id).await { + DatasetContents::Crucible(crucible) => crucible.data.clone(), + _ => panic!("{zpool_id} / {dataset_id} is not a crucible dataset"), + } } pub async fn get_dataset_for_region( @@ -1146,7 +1334,11 @@ impl Storage { regions.pop() } - pub fn drop_dataset(&mut self, zpool_id: ZpoolUuid, dataset_id: Uuid) { + pub fn drop_dataset( + &mut self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + ) { self.zpools .get_mut(&zpool_id) .expect("Zpool does not exist") From 6b622562e6ec86291901cdf700d8ea77a0ab72ba Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 8 Oct 2024 09:47:53 -0700 Subject: [PATCH 11/37] Patch test APIs start using typed UUIDs, a liiiiittle bit --- .../region_snapshot_replacement_start.rs | 2 +- nexus/src/app/sagas/disk_create.rs | 3 +- .../src/app/sagas/region_replacement_start.rs | 3 +- nexus/tests/integration_tests/disks.rs | 12 +++---- nexus/tests/integration_tests/unauthorized.rs | 3 +- .../integration_tests/volume_management.rs | 35 +++++++++++++++---- 6 files changed, 41 insertions(+), 17 deletions(-) diff --git a/nexus/src/app/background/tasks/region_snapshot_replacement_start.rs b/nexus/src/app/background/tasks/region_snapshot_replacement_start.rs index 8fd1e55975..444cb15495 100644 --- a/nexus/src/app/background/tasks/region_snapshot_replacement_start.rs +++ b/nexus/src/app/background/tasks/region_snapshot_replacement_start.rs @@ -401,7 +401,7 @@ mod test { datastore .region_snapshot_create(RegionSnapshot::new( - dataset.id, + *dataset.id.as_untyped_uuid(), region_id, snapshot_id, String::from("[fd00:1122:3344::101]:12345"), diff --git a/nexus/src/app/sagas/disk_create.rs b/nexus/src/app/sagas/disk_create.rs index d2e3053668..424b62cb0e 100644 --- a/nexus/src/app/sagas/disk_create.rs +++ b/nexus/src/app/sagas/disk_create.rs @@ -839,6 +839,7 @@ pub(crate) mod test { use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::Name; use omicron_sled_agent::sim::SledAgent; + use omicron_uuid_kinds::GenericUuid; use uuid::Uuid; type ControlPlaneTestContext = @@ -988,7 +989,7 @@ pub(crate) mod test { for zpool in test.zpools() { for dataset in &zpool.datasets { if datastore - .regions_total_occupied_size(dataset.id) + .regions_total_occupied_size(*dataset.id.as_untyped_uuid()) .await .unwrap() != 0 diff --git a/nexus/src/app/sagas/region_replacement_start.rs b/nexus/src/app/sagas/region_replacement_start.rs index a71a7498ac..9c32f6fb01 100644 --- a/nexus/src/app/sagas/region_replacement_start.rs +++ b/nexus/src/app/sagas/region_replacement_start.rs @@ -793,6 +793,7 @@ pub(crate) mod test { use nexus_test_utils_macros::nexus_test; use nexus_types::identity::Asset; use omicron_common::api::internal::shared::DatasetKind; + use omicron_uuid_kinds::GenericUuid; use sled_agent_client::types::VolumeConstructionRequest; use uuid::Uuid; @@ -1040,7 +1041,7 @@ pub(crate) mod test { for zpool in test.zpools() { for dataset in &zpool.datasets { if datastore - .regions_total_occupied_size(dataset.id) + .regions_total_occupied_size(*dataset.id.as_untyped_uuid()) .await .unwrap() != 0 diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index c9659d6bb8..f695f7c0f6 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -1562,7 +1562,7 @@ async fn test_disk_size_accounting(cptestctx: &ControlPlaneTestContext) { for dataset in &zpool.datasets { assert_eq!( datastore - .regions_total_occupied_size(dataset.id) + .regions_total_occupied_size(*dataset.id.as_untyped_uuid()) .await .unwrap(), 0 @@ -1601,7 +1601,7 @@ async fn test_disk_size_accounting(cptestctx: &ControlPlaneTestContext) { for dataset in &zpool.datasets { assert_eq!( datastore - .regions_total_occupied_size(dataset.id) + .regions_total_occupied_size(*dataset.id.as_untyped_uuid()) .await .unwrap(), ByteCount::from_gibibytes_u32(7).to_bytes(), @@ -1638,7 +1638,7 @@ async fn test_disk_size_accounting(cptestctx: &ControlPlaneTestContext) { for dataset in &zpool.datasets { assert_eq!( datastore - .regions_total_occupied_size(dataset.id) + .regions_total_occupied_size(*dataset.id.as_untyped_uuid()) .await .unwrap(), ByteCount::from_gibibytes_u32(7).to_bytes(), @@ -1662,7 +1662,7 @@ async fn test_disk_size_accounting(cptestctx: &ControlPlaneTestContext) { for dataset in &zpool.datasets { assert_eq!( datastore - .regions_total_occupied_size(dataset.id) + .regions_total_occupied_size(*dataset.id.as_untyped_uuid()) .await .unwrap(), 0, @@ -1698,7 +1698,7 @@ async fn test_disk_size_accounting(cptestctx: &ControlPlaneTestContext) { for dataset in &zpool.datasets { assert_eq!( datastore - .regions_total_occupied_size(dataset.id) + .regions_total_occupied_size(*dataset.id.as_untyped_uuid()) .await .unwrap(), ByteCount::from_gibibytes_u32(10).to_bytes(), @@ -2100,7 +2100,7 @@ async fn test_single_region_allocate(cptestctx: &ControlPlaneTestContext) { for zpool in disk_test.zpools() { for dataset in &zpool.datasets { let total_size = datastore - .regions_total_occupied_size(dataset.id) + .regions_total_occupied_size(*dataset.id.as_untyped_uuid()) .await .unwrap(); diff --git a/nexus/tests/integration_tests/unauthorized.rs b/nexus/tests/integration_tests/unauthorized.rs index 93e40dbc2e..6581dcb9e9 100644 --- a/nexus/tests/integration_tests/unauthorized.rs +++ b/nexus/tests/integration_tests/unauthorized.rs @@ -19,6 +19,7 @@ use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; use nexus_test_utils::http_testing::TestResponse; use nexus_test_utils_macros::nexus_test; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::ZpoolUuid; use once_cell::sync::Lazy; @@ -63,7 +64,7 @@ async fn test_unauthorized(cptestctx: &ControlPlaneTestContext) { sled_id, nexus_test_utils::PHYSICAL_DISK_UUID.parse().unwrap(), ZpoolUuid::new_v4(), - uuid::Uuid::new_v4(), + DatasetUuid::new_v4(), DiskTest::DEFAULT_ZPOOL_SIZE_GIB, ) .await; diff --git a/nexus/tests/integration_tests/volume_management.rs b/nexus/tests/integration_tests/volume_management.rs index 8765218d33..5038bf185d 100644 --- a/nexus/tests/integration_tests/volume_management.rs +++ b/nexus/tests/integration_tests/volume_management.rs @@ -30,6 +30,7 @@ use omicron_common::api::external::Name; use omicron_common::api::internal; use omicron_uuid_kinds::DownstairsKind; use omicron_uuid_kinds::DownstairsRegionKind; +use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::TypedUuid; use omicron_uuid_kinds::UpstairsKind; use omicron_uuid_kinds::UpstairsRepairKind; @@ -2225,7 +2226,7 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { ®ion_snapshots[i]; datastore .region_snapshot_create(nexus_db_model::RegionSnapshot { - dataset_id: *dataset_id, + dataset_id: *dataset_id.as_untyped_uuid(), region_id: *region_id, snapshot_id: *snapshot_id, snapshot_addr: snapshot_addr.clone(), @@ -2292,7 +2293,11 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { for i in 0..3 { let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; let region_snapshot = datastore - .region_snapshot_get(dataset_id, region_id, snapshot_id) + .region_snapshot_get( + *dataset_id.as_untyped_uuid(), + region_id, + snapshot_id, + ) .await .unwrap() .unwrap(); @@ -2309,7 +2314,11 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { for i in 0..3 { let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; let region_snapshot = datastore - .region_snapshot_get(dataset_id, region_id, snapshot_id) + .region_snapshot_get( + *dataset_id.as_untyped_uuid(), + region_id, + snapshot_id, + ) .await .unwrap() .unwrap(); @@ -2335,7 +2344,7 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { ®ion_snapshots[i]; datastore .region_snapshot_create(nexus_db_model::RegionSnapshot { - dataset_id: *dataset_id, + dataset_id: *dataset_id.as_untyped_uuid(), region_id: *region_id, snapshot_id: *snapshot_id, snapshot_addr: snapshot_addr.clone(), @@ -2402,7 +2411,11 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { for i in 0..3 { let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; let region_snapshot = datastore - .region_snapshot_get(dataset_id, region_id, snapshot_id) + .region_snapshot_get( + *dataset_id.as_untyped_uuid(), + region_id, + snapshot_id, + ) .await .unwrap() .unwrap(); @@ -2413,7 +2426,11 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { for i in 3..6 { let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; let region_snapshot = datastore - .region_snapshot_get(dataset_id, region_id, snapshot_id) + .region_snapshot_get( + *dataset_id.as_untyped_uuid(), + region_id, + snapshot_id, + ) .await .unwrap() .unwrap(); @@ -2432,7 +2449,11 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { for i in 0..6 { let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; let region_snapshot = datastore - .region_snapshot_get(dataset_id, region_id, snapshot_id) + .region_snapshot_get( + *dataset_id.as_untyped_uuid(), + region_id, + snapshot_id, + ) .await .unwrap() .unwrap(); From 6330be4d5f32edece81d37d31fb4ce610705edbe Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 8 Oct 2024 11:02:23 -0700 Subject: [PATCH 12/37] error messages for clippy --- sled-agent/src/sim/storage.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 0c5f8aedea..22de5f07cd 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -1088,7 +1088,7 @@ impl Storage { let DatasetContents::Debug(debug) = dataset else { return Err(HttpError::for_bad_request( None, - format!("Not a debug dataset"), + format!("Not a debug dataset: {zpool_id} / {dataset_id}"), )); }; @@ -1116,7 +1116,7 @@ impl Storage { let DatasetContents::Debug(debug) = dataset else { return Err(HttpError::for_bad_request( None, - format!("Not a debug dataset"), + format!("Not a debug dataset: {zpool_id} / {dataset_id}"), )); }; From c0bcc1ce4420f67b09135c454e3a1093ebc341ac Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 11 Oct 2024 08:45:40 -0700 Subject: [PATCH 13/37] implement single-file GET API --- Cargo.lock | 4 +- openapi/sled-agent.json | 78 ++++++++++++++++++++- sled-agent/Cargo.toml | 5 ++ sled-agent/api/src/lib.rs | 26 +++++++ sled-agent/src/http_entrypoints.rs | 93 ++++++++++++++++++++++++-- sled-agent/src/sim/http_entrypoints.rs | 1 + 6 files changed, 200 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29827a6083..5ac9d8b9ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4757,7 +4757,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -6828,6 +6828,7 @@ dependencies = [ "guppy", "hex", "http 1.1.0", + "http-body-util", "hyper 1.4.1", "hyper-staticfile", "illumos-utils", @@ -6896,6 +6897,7 @@ dependencies = [ "uuid", "walkdir", "zeroize", + "zip", "zone 0.3.0", ] diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 5530eb65d4..31d14f0f1e 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -574,7 +574,7 @@ }, "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}": { "get": { - "summary": "Fetch a service bundle from a particular dataset", + "summary": "Fetch a whole service bundle from a particular dataset", "operationId": "support_bundle_get", "parameters": [ { @@ -605,6 +605,16 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SupportBundleGetQueryParams" + } + } + }, + "required": true + }, "responses": { "200": { "description": "", @@ -5259,6 +5269,18 @@ "format": "uint8", "minimum": 0 }, + "SupportBundleGetQueryParams": { + "description": "Query parameters for reading the support bundle", + "type": "object", + "properties": { + "query_type": { + "$ref": "#/components/schemas/SupportBundleQueryType" + } + }, + "required": [ + "query_type" + ] + }, "SupportBundleMetadata": { "description": "Metadata about a support bundle", "type": "object", @@ -5275,6 +5297,60 @@ "support_bundle_id" ] }, + "SupportBundleQueryType": { + "description": "Describes the type of access to the support bundle", + "oneOf": [ + { + "description": "Access the whole support bundle", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "whole" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Access the names of all files within the support bundle", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "index" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Access a specific file within the support bundle", + "type": "object", + "properties": { + "file_path": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "path" + ] + } + }, + "required": [ + "file_path", + "type" + ] + } + ] + }, "SupportBundleState": { "type": "string", "enum": [ diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 416db6f5c4..222fccecf0 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -36,6 +36,8 @@ futures.workspace = true glob.workspace = true hex.workspace = true http.workspace = true +http-body-util.workspace = true +hyper.workspace = true hyper-staticfile.workspace = true gateway-client.workspace = true illumos-utils.workspace = true @@ -88,6 +90,8 @@ tar.workspace = true thiserror.workspace = true tofino.workspace = true tokio = { workspace = true, features = ["full"] } +tokio-stream.workspace = true +tokio-util.workspace = true toml.workspace = true usdt.workspace = true uuid.workspace = true @@ -97,6 +101,7 @@ static_assertions.workspace = true omicron-workspace-hack.workspace = true slog-error-chain.workspace = true walkdir.workspace = true +zip.workspace = true [target.'cfg(target_os = "illumos")'.dependencies] opte-ioctl.workspace = true diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index 7ad8fd7499..10f6e2e980 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -187,6 +187,7 @@ pub trait SledAgentApi { async fn support_bundle_get( rqctx: RequestContext, path_params: Path, + body: TypedBody, ) -> Result>, HttpError>; /// Delete a service bundle from a particular dataset @@ -607,12 +608,37 @@ pub struct SupportBundlePathParam { pub support_bundle_id: SupportBundleUuid, } +/// Path parameters for Support Bundle requests (sled agent API) +#[derive(Deserialize, JsonSchema)] +pub struct SupportBundleFilePathParam { + #[serde(flatten)] + pub parent: SupportBundlePathParam, +} + /// Metadata about a support bundle #[derive(Deserialize, Serialize, JsonSchema)] pub struct SupportBundleCreateQueryParams { pub hash: ArtifactHash, } +/// Query parameters for reading the support bundle +#[derive(Deserialize, Serialize, JsonSchema)] +pub struct SupportBundleGetQueryParams { + pub query_type: SupportBundleQueryType, +} + +/// Describes the type of access to the support bundle +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum SupportBundleQueryType { + /// Access the whole support bundle + Whole, + /// Access the names of all files within the support bundle + Index, + /// Access a specific file within the support bundle + Path { file_path: String }, +} + #[derive(Deserialize, Serialize, JsonSchema, PartialEq)] #[serde(rename_all = "snake_case")] pub enum SupportBundleState { diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 126c101e64..60580ac30e 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -16,6 +16,7 @@ use dropshot::{ HttpResponseUpdatedNoContent, Path, Query, RequestContext, StreamingBody, TypedBody, }; +use futures::TryStreamExt; use nexus_sled_agent_shared::inventory::{ Inventory, OmicronZonesConfig, SledRole, }; @@ -51,6 +52,8 @@ use sled_agent_types::zone_bundle::{ StorageLimit, ZoneBundleId, ZoneBundleMetadata, }; use std::collections::BTreeMap; +use std::io::Read; +use tokio_util::io::ReaderStream; type SledApiDescription = ApiDescription; @@ -62,6 +65,60 @@ pub fn api() -> SledApiDescription { enum SledAgentImpl {} +// TODO: Patch the error kinds? +// TODO: Maybe move this impl out of this file? + +fn stream_zip_entry_helper( + tx: &tokio::sync::mpsc::Sender, HttpError>>, + file: std::fs::File, + entry_path: String, +) -> Result<(), HttpError> { + let mut archive = zip::ZipArchive::new(file) + .map_err(|err| HttpError::for_unavail(None, err.to_string()))?; + let mut reader = archive + .by_name(&entry_path) + .map_err(|err| HttpError::for_unavail(None, err.to_string()))?; + if !reader.is_file() { + return Err(HttpError::for_unavail(None, "Not a file".to_string())); + } + + loop { + let mut buf = vec![0; 4096]; + let n = reader + .read(&mut buf) + .map_err(|err| HttpError::for_unavail(None, err.to_string()))?; + if n == 0 { + return Ok::<(), HttpError>(()); + } + buf.truncate(n); + tx.blocking_send(Ok(buf)) + .map_err(|err| HttpError::for_unavail(None, err.to_string()))?; + } +} + +// Returns a stream of bytes representing an entry within a zipfile. +// +// Q: Why does this spawn a task? +// A: Two reasons - first, the "zip" crate is synchronous, and secondly, +// it has strong opinions about the "archive" living as long as the "entry +// reader". Without a task, streaming an entry from the archive would require +// a self-referential struct, as described in: +// https://morestina.net/blog/1868/self-referential-types-for-fun-and-profit +fn stream_zip_entry( + file: std::fs::File, + entry_path: String, +) -> tokio_stream::wrappers::ReceiverStream, HttpError>> { + let (tx, rx) = tokio::sync::mpsc::channel(16); + + tokio::task::spawn_blocking(move || { + if let Err(err) = stream_zip_entry_helper(&tx, file, entry_path) { + let _ = tx.blocking_send(Err(err)); + } + }); + + tokio_stream::wrappers::ReceiverStream::new(rx) +} + impl SledAgentApi for SledAgentImpl { type Context = SledAgent; @@ -265,6 +322,7 @@ impl SledAgentApi for SledAgentImpl { async fn support_bundle_get( rqctx: RequestContext, path_params: Path, + body: TypedBody, ) -> Result>, HttpError> { let sa = rqctx.context(); @@ -275,16 +333,41 @@ impl SledAgentApi for SledAgentImpl { .support_bundle_get(zpool_id, dataset_id, support_bundle_id) .await?; - let file_access = hyper_staticfile::vfs::TokioFileAccess::new(file); - let file_stream = - hyper_staticfile::util::FileBytesStream::new(file_access); - let body = Body::wrap(hyper_staticfile::Body::Full(file_stream)); + let query = body.into_inner().query_type; + let (body, content_type) = match query { + SupportBundleQueryType::Whole => { + let data_stream = ReaderStream::new(file); + let body = http_body_util::StreamBody::new( + data_stream.map_ok(|b| hyper::body::Frame::data(b)), + ); + (Body::wrap(body), "application/zip") + } + SupportBundleQueryType::Index => { + let file_std = file.into_std().await; + let archive = + zip::ZipArchive::new(file_std).map_err(|err| { + HttpError::for_unavail(None, err.to_string()) + })?; + let names: Vec<&str> = archive.file_names().collect(); + let all_names = names.join("\n"); + (Body::wrap(all_names), "text/plain") + } + SupportBundleQueryType::Path { file_path } => { + let file_std = file.into_std().await; + let streamer = http_body_util::StreamBody::new( + stream_zip_entry(file_std, file_path) + .map_ok(|b| hyper::body::Frame::data(b.into())), + ); + (Body::wrap(streamer), "text/plain") + } + }; + let body = FreeformBody(body); let mut response = HttpResponseHeaders::new_unnamed(HttpResponseOk(body)); response.headers_mut().append( http::header::CONTENT_TYPE, - "application/gzip".try_into().unwrap(), + content_type.try_into().unwrap(), ); Ok(response) } diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index c356c91d44..1da17cb827 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -404,6 +404,7 @@ impl SledAgentApi for SledAgentSimImpl { async fn support_bundle_get( rqctx: RequestContext, path_params: Path, + _body: TypedBody, ) -> Result>, HttpError> { let sa = rqctx.context(); From 071ecbab90ea3d495580afa1e546e45cd1bda57f Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 14 Oct 2024 17:13:55 -0700 Subject: [PATCH 14/37] Upgrade zip, add range request support to bundles --- Cargo.lock | 97 +++- Cargo.toml | 3 +- openapi/sled-agent.json | 10 +- sled-agent/Cargo.toml | 1 + sled-agent/api/Cargo.toml | 1 + sled-agent/api/src/lib.rs | 9 +- sled-agent/src/http_entrypoints.rs | 111 +--- sled-agent/src/lib.rs | 1 + sled-agent/src/range.rs | 175 +++++++ sled-agent/src/sim/http_entrypoints.rs | 13 +- sled-agent/src/sled_agent.rs | 295 +---------- sled-agent/src/sled_agent/support_bundle.rs | 550 ++++++++++++++++++++ tufaceous-lib/src/archive.rs | 17 +- 13 files changed, 863 insertions(+), 420 deletions(-) create mode 100644 sled-agent/src/range.rs create mode 100644 sled-agent/src/sled_agent/support_bundle.rs diff --git a/Cargo.lock b/Cargo.lock index 5ac9d8b9ca..919e2caf49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -184,6 +184,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.7.1" @@ -1580,9 +1589,9 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -1673,9 +1682,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crossterm" @@ -2099,6 +2108,17 @@ dependencies = [ "syn 2.0.79", ] +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "derive_builder" version = "0.20.0" @@ -2294,6 +2314,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d305e5a3904ee14166439a70feef04853c1234226dbb27ede127b88dc5a4a9d" +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "dlpi" version = "0.2.0" @@ -3229,7 +3260,7 @@ dependencies = [ "usdt", "uuid", "version_check", - "zip", + "zip 0.6.6", ] [[package]] @@ -3927,7 +3958,7 @@ dependencies = [ "toml 0.7.8", "x509-cert", "zerocopy 0.6.6", - "zip", + "zip 0.6.6", ] [[package]] @@ -4927,6 +4958,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.22" @@ -5058,9 +5095,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap" @@ -6829,6 +6866,7 @@ dependencies = [ "hex", "http 1.1.0", "http-body-util", + "http-range", "hyper 1.4.1", "hyper-staticfile", "illumos-utils", @@ -6897,7 +6935,7 @@ dependencies = [ "uuid", "walkdir", "zeroize", - "zip", + "zip 2.2.0", "zone 0.3.0", ] @@ -10130,6 +10168,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "similar" version = "2.6.0" @@ -10188,6 +10232,7 @@ dependencies = [ "camino", "chrono", "dropshot", + "http 1.1.0", "nexus-sled-agent-shared", "omicron-common", "omicron-uuid-kinds", @@ -11758,7 +11803,7 @@ dependencies = [ "toml 0.8.19", "tough", "url", - "zip", + "zip 2.2.0", ] [[package]] @@ -13068,6 +13113,24 @@ dependencies = [ "flate2", ] +[[package]] +name = "zip" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" +dependencies = [ + "arbitrary", + "bzip2", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap 2.5.0", + "memchr", + "thiserror", + "zopfli", +] + [[package]] name = "zone" version = "0.1.8" @@ -13137,3 +13200,17 @@ dependencies = [ "quote", "syn 1.0.109", ] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index c237803e03..add670138c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -389,6 +389,7 @@ highway = "1.2.0" hkdf = "0.12.4" http = "1.1.0" http-body-util = "0.1.2" +http-range = "0.1.5" httpmock = "0.8.0-alpha.1" httptest = "0.16.1" hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "main" } @@ -631,7 +632,7 @@ wicket-common = { path = "wicket-common" } wicketd-api = { path = "wicketd-api" } wicketd-client = { path = "clients/wicketd-client" } zeroize = { version = "1.8.1", features = ["zeroize_derive", "std"] } -zip = { version = "0.6.6", default-features = false, features = ["deflate","bzip2"] } +zip = { version = "2.2.0", default-features = false, features = ["deflate","bzip2"] } zone = { version = "0.3", default-features = false, features = ["async"] } # newtype-uuid is set to default-features = false because we don't want to diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 31d14f0f1e..04c20bfad3 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -574,7 +574,7 @@ }, "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}": { "get": { - "summary": "Fetch a whole service bundle from a particular dataset", + "summary": "Fetch a service bundle from a particular dataset", "operationId": "support_bundle_get", "parameters": [ { @@ -616,19 +616,13 @@ "required": true }, "responses": { - "200": { + "default": { "description": "", "content": { "*/*": { "schema": {} } } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" } } }, diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 222fccecf0..ede616c7aa 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -37,6 +37,7 @@ glob.workspace = true hex.workspace = true http.workspace = true http-body-util.workspace = true +http-range.workspace = true hyper.workspace = true hyper-staticfile.workspace = true gateway-client.workspace = true diff --git a/sled-agent/api/Cargo.toml b/sled-agent/api/Cargo.toml index 17a1294f1f..8020651b15 100644 --- a/sled-agent/api/Cargo.toml +++ b/sled-agent/api/Cargo.toml @@ -11,6 +11,7 @@ workspace = true camino.workspace = true chrono.workspace = true dropshot.workspace = true +http.workspace = true nexus-sled-agent-shared.workspace = true omicron-common.workspace = true omicron-uuid-kinds.workspace = true diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index 10f6e2e980..4abe298d93 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -6,7 +6,7 @@ use std::{collections::BTreeMap, time::Duration}; use camino::Utf8PathBuf; use dropshot::{ - FreeformBody, HttpError, HttpResponseCreated, HttpResponseDeleted, + Body, FreeformBody, HttpError, HttpResponseCreated, HttpResponseDeleted, HttpResponseHeaders, HttpResponseOk, HttpResponseUpdatedNoContent, Path, Query, RequestContext, StreamingBody, TypedBody, }; @@ -188,7 +188,7 @@ pub trait SledAgentApi { rqctx: RequestContext, path_params: Path, body: TypedBody, - ) -> Result>, HttpError>; + ) -> Result, HttpError>; /// Delete a service bundle from a particular dataset #[endpoint { @@ -621,6 +621,11 @@ pub struct SupportBundleCreateQueryParams { pub hash: ArtifactHash, } +#[derive(Deserialize, Serialize, JsonSchema)] +pub struct SupportBundleGetHeaders { + range: String, +} + /// Query parameters for reading the support bundle #[derive(Deserialize, Serialize, JsonSchema)] pub struct SupportBundleGetQueryParams { diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 60580ac30e..f58e41e579 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -4,6 +4,7 @@ //! HTTP entrypoint functions for the sled agent's exposed API +use super::range::RequestContextEx; use super::sled_agent::SledAgent; use crate::sled_agent::Error as SledAgentError; use crate::zone_bundle::BundleError; @@ -16,7 +17,6 @@ use dropshot::{ HttpResponseUpdatedNoContent, Path, Query, RequestContext, StreamingBody, TypedBody, }; -use futures::TryStreamExt; use nexus_sled_agent_shared::inventory::{ Inventory, OmicronZonesConfig, SledRole, }; @@ -52,8 +52,6 @@ use sled_agent_types::zone_bundle::{ StorageLimit, ZoneBundleId, ZoneBundleMetadata, }; use std::collections::BTreeMap; -use std::io::Read; -use tokio_util::io::ReaderStream; type SledApiDescription = ApiDescription; @@ -65,60 +63,6 @@ pub fn api() -> SledApiDescription { enum SledAgentImpl {} -// TODO: Patch the error kinds? -// TODO: Maybe move this impl out of this file? - -fn stream_zip_entry_helper( - tx: &tokio::sync::mpsc::Sender, HttpError>>, - file: std::fs::File, - entry_path: String, -) -> Result<(), HttpError> { - let mut archive = zip::ZipArchive::new(file) - .map_err(|err| HttpError::for_unavail(None, err.to_string()))?; - let mut reader = archive - .by_name(&entry_path) - .map_err(|err| HttpError::for_unavail(None, err.to_string()))?; - if !reader.is_file() { - return Err(HttpError::for_unavail(None, "Not a file".to_string())); - } - - loop { - let mut buf = vec![0; 4096]; - let n = reader - .read(&mut buf) - .map_err(|err| HttpError::for_unavail(None, err.to_string()))?; - if n == 0 { - return Ok::<(), HttpError>(()); - } - buf.truncate(n); - tx.blocking_send(Ok(buf)) - .map_err(|err| HttpError::for_unavail(None, err.to_string()))?; - } -} - -// Returns a stream of bytes representing an entry within a zipfile. -// -// Q: Why does this spawn a task? -// A: Two reasons - first, the "zip" crate is synchronous, and secondly, -// it has strong opinions about the "archive" living as long as the "entry -// reader". Without a task, streaming an entry from the archive would require -// a self-referential struct, as described in: -// https://morestina.net/blog/1868/self-referential-types-for-fun-and-profit -fn stream_zip_entry( - file: std::fs::File, - entry_path: String, -) -> tokio_stream::wrappers::ReceiverStream, HttpError>> { - let (tx, rx) = tokio::sync::mpsc::channel(16); - - tokio::task::spawn_blocking(move || { - if let Err(err) = stream_zip_entry_helper(&tx, file, entry_path) { - let _ = tx.blocking_send(Err(err)); - } - }); - - tokio_stream::wrappers::ReceiverStream::new(rx) -} - impl SledAgentApi for SledAgentImpl { type Context = SledAgent; @@ -323,53 +267,22 @@ impl SledAgentApi for SledAgentImpl { rqctx: RequestContext, path_params: Path, body: TypedBody, - ) -> Result>, HttpError> - { + ) -> Result, HttpError> { let sa = rqctx.context(); let SupportBundlePathParam { zpool_id, dataset_id, support_bundle_id } = path_params.into_inner(); - let file = sa - .support_bundle_get(zpool_id, dataset_id, support_bundle_id) - .await?; - + let range = rqctx.range(); let query = body.into_inner().query_type; - let (body, content_type) = match query { - SupportBundleQueryType::Whole => { - let data_stream = ReaderStream::new(file); - let body = http_body_util::StreamBody::new( - data_stream.map_ok(|b| hyper::body::Frame::data(b)), - ); - (Body::wrap(body), "application/zip") - } - SupportBundleQueryType::Index => { - let file_std = file.into_std().await; - let archive = - zip::ZipArchive::new(file_std).map_err(|err| { - HttpError::for_unavail(None, err.to_string()) - })?; - let names: Vec<&str> = archive.file_names().collect(); - let all_names = names.join("\n"); - (Body::wrap(all_names), "text/plain") - } - SupportBundleQueryType::Path { file_path } => { - let file_std = file.into_std().await; - let streamer = http_body_util::StreamBody::new( - stream_zip_entry(file_std, file_path) - .map_ok(|b| hyper::body::Frame::data(b.into())), - ); - (Body::wrap(streamer), "text/plain") - } - }; - - let body = FreeformBody(body); - let mut response = - HttpResponseHeaders::new_unnamed(HttpResponseOk(body)); - response.headers_mut().append( - http::header::CONTENT_TYPE, - content_type.try_into().unwrap(), - ); - Ok(response) + Ok(sa + .support_bundle_get( + zpool_id, + dataset_id, + support_bundle_id, + range, + query, + ) + .await?) } async fn support_bundle_delete( diff --git a/sled-agent/src/lib.rs b/sled-agent/src/lib.rs index a2421528e2..befb00e2ba 100644 --- a/sled-agent/src/lib.rs +++ b/sled-agent/src/lib.rs @@ -31,6 +31,7 @@ pub mod params; mod probe_manager; mod profile; pub mod rack_setup; +mod range; pub mod server; pub mod services; mod sled_agent; diff --git a/sled-agent/src/range.rs b/sled-agent/src/range.rs new file mode 100644 index 0000000000..1306c786cd --- /dev/null +++ b/sled-agent/src/range.rs @@ -0,0 +1,175 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use bytes::Bytes; +use dropshot::Body; +use dropshot::HttpError; +use futures::TryStreamExt; +use hyper::{ + header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE}, + Response, StatusCode, +}; +use tokio::sync::mpsc; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Using multiple ranges is not supported")] + MultipleRangesUnsupported, + + #[error("Failed to parse range")] + Parse(http_range::HttpRangeParseError), + + #[error(transparent)] + Http(#[from] http::Error), +} + +impl From for HttpError { + fn from(err: Error) -> Self { + match err { + Error::MultipleRangesUnsupported | Error::Parse(_) => { + HttpError::for_bad_request(None, err.to_string()) + } + Error::Http(err) => err.into(), + } + } +} + +pub fn bad_range_response(file_size: u64) -> Response { + hyper::Response::builder() + .status(StatusCode::RANGE_NOT_SATISFIABLE) + .header(ACCEPT_RANGES, "bytes") + .header(CONTENT_RANGE, format!("bytes */{file_size}")) + .body(Body::empty()) + .unwrap() +} + +/// Generate a GET response, optionally for a HTTP range request. The total +/// file length should be provided, whether or not the expected Content-Length +/// for a range request is shorter. +pub fn make_get_response( + range: Option, + file_length: u64, + content_type: Option<&str>, + rx: mpsc::Receiver>, +) -> Result, Error> +where + E: Into> + + Send + + Sync + + 'static, +{ + Ok(make_response_common(range, file_length, content_type).body( + Body::wrap(http_body_util::StreamBody::new( + tokio_stream::wrappers::ReceiverStream::new(rx) + .map_ok(|b| hyper::body::Frame::data(b)), + )), + )?) +} + +/// Generate a HEAD response, optionally for a HTTP range request. The total +/// file length should be provided, whether or not the expected Content-Length +/// for a range request is shorter. +pub fn make_head_response( + range: Option, + file_length: u64, + content_type: Option<&str>, +) -> Result, Error> { + Ok(make_response_common(range, file_length, content_type) + .body(Body::empty())?) +} + +pub fn make_response_common( + range: Option, + file_length: u64, + content_type: Option<&str>, +) -> hyper::http::response::Builder { + let mut res = Response::builder(); + res = res.header(ACCEPT_RANGES, "bytes"); + res = res.header( + CONTENT_TYPE, + content_type.unwrap_or("application/octet-stream"), + ); + + if let Some(range) = range { + res = res.header(CONTENT_LENGTH, range.content_length().to_string()); + res = res.header(CONTENT_RANGE, range.to_content_range()); + res = res.status(StatusCode::PARTIAL_CONTENT); + } else { + res = res.header(CONTENT_LENGTH, file_length.to_string()); + res = res.status(StatusCode::OK); + } + + res +} + +pub struct PotentialRange(Vec); + +impl PotentialRange { + pub fn single_range(&self, len: u64) -> Result { + match http_range::HttpRange::parse_bytes(&self.0, len) { + Ok(ranges) => { + if ranges.len() != 1 || ranges[0].length < 1 { + // Right now, we don't want to deal with encoding a + // response that has multiple ranges. + Err(Error::MultipleRangesUnsupported) + } else { + Ok(SingleRange(ranges[0], len)) + } + } + Err(err) => Err(Error::Parse(err)), + } + } +} + +pub struct SingleRange(http_range::HttpRange, u64); + +impl SingleRange { + /// Return the first byte in this range for use in inclusive ranges. + pub fn start(&self) -> u64 { + self.0.start + } + + /// Return the last byte in this range for use in inclusive ranges. + pub fn end(&self) -> u64 { + assert!(self.0.length > 0); + + self.0.start.checked_add(self.0.length).unwrap().checked_sub(1).unwrap() + } + + /// Generate the Content-Range header for inclusion in a HTTP 206 partial + /// content response using this range. + pub fn to_content_range(&self) -> String { + format!("bytes {}-{}/{}", self.0.start, self.end(), self.1) + } + + /// Generate a Range header for inclusion in another HTTP request; e.g., + /// to a backend object store. + pub fn to_range(&self) -> String { + format!("bytes={}-{}", self.0.start, self.end()) + } + + pub fn content_length(&self) -> u64 { + assert!(self.0.length > 0); + + self.0.length + } +} + +pub trait RequestContextEx { + fn range(&self) -> Option; +} + +impl RequestContextEx for dropshot::RequestContext +where + T: Send + Sync + 'static, +{ + /// If there is a Range header, return it for processing during response + /// generation. + fn range(&self) -> Option { + self.request + .headers() + .get(hyper::header::RANGE) + .map(|hv| PotentialRange(hv.as_bytes().to_vec())) + } +} diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 1da17cb827..30dcd20f49 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -405,17 +405,20 @@ impl SledAgentApi for SledAgentSimImpl { rqctx: RequestContext, path_params: Path, _body: TypedBody, - ) -> Result>, HttpError> - { + ) -> Result, HttpError> { let sa = rqctx.context(); let SupportBundlePathParam { zpool_id, dataset_id, support_bundle_id } = path_params.into_inner(); sa.support_bundle_get(zpool_id, dataset_id, support_bundle_id).await?; - let body = FreeformBody("simulated support bundle; do not eat".into()); - let response = HttpResponseHeaders::new_unnamed(HttpResponseOk(body)); - Ok(response) + Ok(http::Response::builder() + .status(http::StatusCode::OK) + .header(http::header::CONTENT_TYPE, "text/html") + .body(dropshot::Body::with_content( + "simulated support bundle; do not eat", + )) + .unwrap()) } async fn support_bundle_delete( diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 1a38d8a901..765983dd72 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -23,11 +23,9 @@ use crate::vmm_reservoir::{ReservoirMode, VmmReservoirManager}; use crate::zone_bundle; use crate::zone_bundle::BundleError; use bootstore::schemes::v0 as bootstore; -use camino::Utf8Path; use camino::Utf8PathBuf; use derive_more::From; use dropshot::HttpError; -use dropshot::StreamingBody; use futures::stream::FuturesUnordered; use futures::StreamExt; use illumos_utils::opte::PortManager; @@ -54,20 +52,11 @@ use omicron_common::backoff::{ retry_notify, retry_policy_internal_service_aggressive, BackoffError, }; use omicron_common::disk::{ - CompressionAlgorithm, DatasetKind, DatasetName, DatasetsConfig, - DatasetsManagementResult, DisksManagementResult, NestedDatasetConfig, - NestedDatasetLocation, OmicronPhysicalDisksConfig, SharedDatasetConfig, + DatasetsConfig, DatasetsManagementResult, DisksManagementResult, + OmicronPhysicalDisksConfig, }; -use omicron_common::update::ArtifactHash; -use omicron_common::zpool_name::ZpoolName; use omicron_ddm_admin_client::Client as DdmAdminClient; -use omicron_uuid_kinds::{ - DatasetUuid, GenericUuid, PropolisUuid, SledUuid, SupportBundleUuid, - ZpoolUuid, -}; -use sha2::{Digest, Sha256}; -use sled_agent_api::SupportBundleMetadata; -use sled_agent_api::SupportBundleState; +use omicron_uuid_kinds::{GenericUuid, PropolisUuid, SledUuid}; use sled_agent_api::Zpool; use sled_agent_types::disk::DiskStateRequested; use sled_agent_types::early_networking::EarlyNetworkConfig; @@ -85,16 +74,12 @@ use sled_hardware::{underlay, HardwareManager}; use sled_hardware_types::underlay::BootstrapInterface; use sled_hardware_types::Baseboard; use sled_storage::dataset::{CRYPT_DATASET, ZONE_DATASET}; -use sled_storage::manager::NestedDatasetListOptions; use sled_storage::manager::StorageHandle; use slog::Logger; use sprockets_tls::keys::SprocketsConfig; use std::collections::BTreeMap; -use std::io::Write; use std::net::{Ipv6Addr, SocketAddrV6}; use std::sync::Arc; -use tokio::io::AsyncReadExt; -use tokio::io::AsyncWriteExt; use uuid::Uuid; use illumos_utils::running_zone::ZoneBuilderFactory; @@ -387,6 +372,9 @@ pub struct SledAgent { sprockets: SprocketsConfig, } +// This extends the "impl SledAgent" for support bundle functionality. +mod support_bundle; + impl SledAgent { /// Initializes a new [`SledAgent`] object. pub async fn new( @@ -860,277 +848,6 @@ impl SledAgent { Ok(datasets_result) } - pub async fn support_bundle_list( - &self, - zpool_id: ZpoolUuid, - dataset_id: DatasetUuid, - ) -> Result, Error> { - let root = DatasetName::new( - ZpoolName::new_external(zpool_id), - DatasetKind::Debug, - ); - let dataset_location = omicron_common::disk::NestedDatasetLocation { - path: String::from(""), - id: dataset_id, - root, - }; - let datasets = self - .storage() - .nested_dataset_list( - dataset_location, - NestedDatasetListOptions::ChildrenOnly, - ) - .await?; - - let mut bundles = Vec::with_capacity(datasets.len()); - for dataset in datasets { - // We should be able to parse each dataset name as a support bundle UUID - let Ok(support_bundle_id) = - dataset.name.path.parse::() - else { - warn!(self.log, "Dataset path not a UUID"; "path" => dataset.name.path); - continue; - }; - - // The dataset for a support bundle exists. - let support_bundle_path = dataset - .name - .mountpoint(illumos_utils::zpool::ZPOOL_MOUNTPOINT_ROOT.into()) - .join("bundle"); - - // Identify whether or not the final "bundle" file exists. - // - // This is a signal that the support bundle has been fully written. - let state = if tokio::fs::try_exists( - &support_bundle_path - ).await.map_err(|e| { - Error::SupportBundle(format!("Cannot check filesystem for {support_bundle_path}: {e}")) - })? { - SupportBundleState::Complete - } else { - SupportBundleState::Incomplete - }; - - let bundle = SupportBundleMetadata { support_bundle_id, state }; - bundles.push(bundle); - } - - Ok(bundles) - } - - pub async fn support_bundle_create( - &self, - zpool_id: ZpoolUuid, - dataset_id: DatasetUuid, - support_bundle_id: SupportBundleUuid, - expected_hash: ArtifactHash, - body: StreamingBody, - ) -> Result { - let log = self.log.new(o!( - "operation" => "support_bundle_create", - "zpool_id" => zpool_id.to_string(), - "dataset_id" => dataset_id.to_string(), - "bundle_id" => support_bundle_id.to_string(), - )); - - let dataset = NestedDatasetLocation { - path: support_bundle_id.to_string(), - id: dataset_id, - root: DatasetName::new( - ZpoolName::new_external(zpool_id), - DatasetKind::Debug, - ), - }; - // The mounted root of the support bundle dataset - let support_bundle_dir = dataset - .mountpoint(illumos_utils::zpool::ZPOOL_MOUNTPOINT_ROOT.into()); - let support_bundle_path = support_bundle_dir.join("bundle"); - let support_bundle_path_tmp = support_bundle_dir.join("bundle.tmp"); - - // Ensure that the dataset exists. - info!(log, "Ensuring dataset exists for bundle"); - self.storage() - .nested_dataset_ensure(NestedDatasetConfig { - name: dataset, - inner: SharedDatasetConfig { - compression: CompressionAlgorithm::On, - quota: None, - reservation: None, - }, - }) - .await?; - - // Exit early if the support bundle already exists - if tokio::fs::try_exists(&support_bundle_path).await.map_err(|e| { - Error::SupportBundle(format!("Cannot check if bundle exists: {e}")) - })? { - if !Self::sha2_checksum_matches( - &support_bundle_path, - &expected_hash, - ) - .await? - { - warn!(log, "Support bundle exists, but the hash doesn't match"); - return Err(Error::SupportBundle("Hash mismatch".to_string())); - } - - info!(log, "Support bundle already exists"); - let metadata = SupportBundleMetadata { - support_bundle_id, - state: SupportBundleState::Complete, - }; - return Ok(metadata); - } - - // Stream the file into the dataset, first as a temporary file, - // and then renaming to the final location. - info!(log, "Streaming bundle to storage"); - let tmp_file = tokio::fs::File::create(&support_bundle_path_tmp) - .await - .map_err(|err| { - Error::SupportBundle(format!("Cannot create file: {err}")) - })?; - if let Err(err) = Self::write_and_finalize_bundle( - tmp_file, - &support_bundle_path_tmp, - &support_bundle_path, - expected_hash, - body, - ) - .await - { - warn!(log, "Failed to write bundle to storage"; "error" => ?err); - if let Err(unlink_err) = - tokio::fs::remove_file(support_bundle_path_tmp).await - { - warn!(log, "Failed to unlink bundle after previous error"; "error" => ?unlink_err); - } - return Err(err); - } - - info!(log, "Bundle written successfully"); - let metadata = SupportBundleMetadata { - support_bundle_id, - state: SupportBundleState::Complete, - }; - Ok(metadata) - } - - pub async fn support_bundle_delete( - &self, - zpool_id: ZpoolUuid, - dataset_id: DatasetUuid, - support_bundle_id: SupportBundleUuid, - ) -> Result<(), Error> { - let log = self.log.new(o!( - "operation" => "support_bundle_delete", - "zpool_id" => zpool_id.to_string(), - "dataset_id" => dataset_id.to_string(), - "bundle_id" => support_bundle_id.to_string(), - )); - info!(log, "Destroying support bundle"); - self.storage() - .nested_dataset_destroy(NestedDatasetLocation { - path: support_bundle_id.to_string(), - id: dataset_id, - root: DatasetName::new( - ZpoolName::new_external(zpool_id), - DatasetKind::Debug, - ), - }) - .await?; - - Ok(()) - } - - pub async fn support_bundle_get( - &self, - zpool_id: ZpoolUuid, - dataset_id: DatasetUuid, - support_bundle_id: SupportBundleUuid, - ) -> Result { - let dataset = NestedDatasetLocation { - path: support_bundle_id.to_string(), - id: dataset_id, - root: DatasetName::new( - ZpoolName::new_external(zpool_id), - DatasetKind::Debug, - ), - }; - // The mounted root of the support bundle dataset - let support_bundle_dir = dataset - .mountpoint(illumos_utils::zpool::ZPOOL_MOUNTPOINT_ROOT.into()); - let path = support_bundle_dir.join("bundle"); - - let f = tokio::fs::File::open(&path).await.map_err(|e| { - Error::SupportBundle(format!("Failed to open {path}: {e}")) - })?; - Ok(f) - } - - /// Returns the hex, lowercase sha2 checksum of a file at `path`. - async fn sha2_checksum_matches( - path: &Utf8Path, - expected: &ArtifactHash, - ) -> Result { - let mut buf = vec![0u8; 65536]; - let mut file = tokio::fs::File::open(path).await.map_err(|e| { - Error::SupportBundle(format!("Failed to open {path}: {e}")) - })?; - let mut ctx = sha2::Sha256::new(); - loop { - let n = file.read(&mut buf).await.map_err(|e| { - Error::SupportBundle(format!("Failed to read from {path}: {e}")) - })?; - if n == 0 { - break; - } - ctx.write_all(&buf[0..n]).map_err(|e| { - Error::SupportBundle(format!("Failed to hash {path}: {e}")) - })?; - } - - let digest = ctx.finalize(); - return Ok(digest.as_slice() == expected.as_ref()); - } - - // A helper function which streams the contents of a bundle to a file. - // - // If at any point this function fails, the temporary file still exists, - // and should be removed. - async fn write_and_finalize_bundle( - mut tmp_file: tokio::fs::File, - from: &Utf8Path, - to: &Utf8Path, - expected_hash: ArtifactHash, - body: StreamingBody, - ) -> Result<(), Error> { - let stream = body.into_stream(); - futures::pin_mut!(stream); - - // Write the body to the file - let mut hasher = Sha256::new(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|e| { - Error::SupportBundle(format!("Failed to stream bundle: {e}")) - })?; - hasher.update(&chunk); - tmp_file.write_all(&chunk).await.map_err(|e| { - Error::SupportBundle(format!("Failed to write bundle: {e}")) - })?; - } - let digest = hasher.finalize(); - if digest.as_slice() != expected_hash.as_ref() { - return Err(Error::SupportBundle("Hash mismatch".to_string())); - } - - // Rename the file to indicate it's ready - tokio::fs::rename(from, to).await.map_err(|e| { - Error::SupportBundle(format!("Cannot finalize bundle: {e}")) - })?; - Ok(()) - } - /// Requests the set of physical disks currently managed by the Sled Agent. /// /// This should be contrasted by the set of disks in the inventory, which diff --git a/sled-agent/src/sled_agent/support_bundle.rs b/sled-agent/src/sled_agent/support_bundle.rs new file mode 100644 index 0000000000..2696b51614 --- /dev/null +++ b/sled-agent/src/sled_agent/support_bundle.rs @@ -0,0 +1,550 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Management of and access to Support Bundles + +use crate::range::PotentialRange; +use crate::range::SingleRange; +use crate::sled_agent::SledAgent; +use camino::Utf8Path; +use dropshot::Body; +use dropshot::HttpError; +use dropshot::StreamingBody; +use futures::StreamExt; +use futures::TryStreamExt; +use omicron_common::api::external::Error as ExternalError; +use omicron_common::disk::CompressionAlgorithm; +use omicron_common::disk::DatasetKind; +use omicron_common::disk::DatasetName; +use omicron_common::disk::NestedDatasetConfig; +use omicron_common::disk::NestedDatasetLocation; +use omicron_common::disk::SharedDatasetConfig; +use omicron_common::update::ArtifactHash; +use omicron_common::zpool_name::ZpoolName; +use omicron_uuid_kinds::DatasetUuid; +use omicron_uuid_kinds::SupportBundleUuid; +use omicron_uuid_kinds::ZpoolUuid; +use sha2::{Digest, Sha256}; +use sled_agent_api::*; +use sled_storage::manager::NestedDatasetListOptions; +use std::io::Read; +use std::io::Seek; +use std::io::Write; +use tokio::io::AsyncReadExt; +use tokio::io::AsyncSeekExt; +use tokio::io::AsyncWriteExt; +use tokio_util::io::ReaderStream; +use zip::result::ZipError; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error(transparent)] + HttpError(#[from] HttpError), + + #[error("Hash mismatch accessing bundle")] + HashMismatch, + + #[error("Not a file")] + NotAFile, + + #[error(transparent)] + Storage(#[from] sled_storage::error::Error), + + #[error(transparent)] + Io(#[from] std::io::Error), + + #[error(transparent)] + Zip(#[from] ZipError), +} + +impl From for HttpError { + fn from(err: Error) -> Self { + match err { + Error::HttpError(err) => err, + Error::HashMismatch => { + HttpError::for_internal_error("Hash mismatch".to_string()) + } + Error::NotAFile => { + HttpError::for_bad_request(None, "Not a file".to_string()) + } + Error::Storage(err) => HttpError::from(ExternalError::from(err)), + Error::Io(err) => HttpError::for_internal_error(err.to_string()), + Error::Zip(err) => match err { + ZipError::FileNotFound => HttpError::for_not_found( + None, + "Entry not found".to_string(), + ), + err => HttpError::for_internal_error(err.to_string()), + }, + } + } +} + +fn stream_zip_entry_helper( + tx: &tokio::sync::mpsc::Sender, HttpError>>, + mut archive: zip::ZipArchive, + entry_path: String, + range: Option, +) -> Result<(), Error> { + let mut reader = archive.by_name_seek(&entry_path)?; + + let mut reader: Box = match range { + Some(range) => { + reader.seek(std::io::SeekFrom::Start(range.start()))?; + Box::new(reader.take(range.content_length())) + } + None => Box::new(reader), + }; + + loop { + let mut buf = vec![0; 4096]; + let n = reader.read(&mut buf)?; + if n == 0 { + return Ok(()); + } + buf.truncate(n); + if let Err(_) = tx.blocking_send(Ok(buf)) { + // If we cannot send anything, just bail out - we also won't be able + // to send an appropriate error in this case, since we'd also be + // sending it on this borked channel + return Ok(()); + } + } +} + +struct ZipEntryStream { + stream: tokio_stream::wrappers::ReceiverStream, HttpError>>, + size: u64, +} + +// Possible responses from the success case of `stream_zip_entry` +enum ZipStreamOutput { + // Returns the zip entry, as a byte stream + // TODO: do we need to pass the size back? or are we good? + Stream(ZipEntryStream), + // Returns an HTTP response indicating the accepted ranges + RangeResponse(http::Response), +} + +// Returns a stream of bytes representing an entry within a zipfile. +// +// Q: Why does this spawn a task? +// A: Two reasons - first, the "zip" crate is synchronous, and secondly, +// it has strong opinions about the "archive" living as long as the "entry +// reader". Without a task, streaming an entry from the archive would require +// a self-referential struct, as described in: +// https://morestina.net/blog/1868/self-referential-types-for-fun-and-profit +fn stream_zip_entry( + file: std::fs::File, + entry_path: String, + pr: Option, +) -> Result { + let mut archive = zip::ZipArchive::new(file)?; + let size = { + // This is a little redundant -- we look up the same file entry within the + // helper function we spawn below -- but the ZipFile is !Send and !Sync, so + // we can't re-use it in the background task. + let zipfile = archive.by_name(&entry_path)?; + if !zipfile.is_file() { + return Err(Error::NotAFile); + } + + zipfile.size() + }; + + let range = if let Some(range) = pr { + let Ok(range) = range.single_range(size) else { + return Ok(ZipStreamOutput::RangeResponse( + crate::range::bad_range_response(size), + )); + }; + Some(range) + } else { + None + }; + + let (tx, rx) = tokio::sync::mpsc::channel(16); + tokio::task::spawn_blocking(move || { + if let Err(err) = + stream_zip_entry_helper(&tx, archive, entry_path, range) + { + let _ = tx.blocking_send(Err(err.into())); + } + }); + + Ok(ZipStreamOutput::Stream(ZipEntryStream { + stream: tokio_stream::wrappers::ReceiverStream::new(rx), + size, + })) +} + +impl SledAgent { + pub async fn support_bundle_list( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + ) -> Result, Error> { + let root = DatasetName::new( + ZpoolName::new_external(zpool_id), + DatasetKind::Debug, + ); + let dataset_location = omicron_common::disk::NestedDatasetLocation { + path: String::from(""), + id: dataset_id, + root, + }; + let datasets = self + .storage() + .nested_dataset_list( + dataset_location, + NestedDatasetListOptions::ChildrenOnly, + ) + .await?; + + let mut bundles = Vec::with_capacity(datasets.len()); + for dataset in datasets { + // We should be able to parse each dataset name as a support bundle UUID + let Ok(support_bundle_id) = + dataset.name.path.parse::() + else { + warn!(self.log, "Dataset path not a UUID"; "path" => dataset.name.path); + continue; + }; + + // The dataset for a support bundle exists. + let support_bundle_path = dataset + .name + .mountpoint(illumos_utils::zpool::ZPOOL_MOUNTPOINT_ROOT.into()) + .join("bundle"); + + // Identify whether or not the final "bundle" file exists. + // + // This is a signal that the support bundle has been fully written. + let state = if tokio::fs::try_exists(&support_bundle_path).await? { + SupportBundleState::Complete + } else { + SupportBundleState::Incomplete + }; + + let bundle = SupportBundleMetadata { support_bundle_id, state }; + bundles.push(bundle); + } + + Ok(bundles) + } + + /// Returns the hex, lowercase sha2 checksum of a file at `path`. + async fn sha2_checksum_matches( + path: &Utf8Path, + expected: &ArtifactHash, + ) -> Result { + let mut buf = vec![0u8; 65536]; + let mut file = tokio::fs::File::open(path).await?; + let mut ctx = sha2::Sha256::new(); + loop { + let n = file.read(&mut buf).await?; + if n == 0 { + break; + } + ctx.write_all(&buf[0..n])?; + } + + let digest = ctx.finalize(); + return Ok(digest.as_slice() == expected.as_ref()); + } + + // A helper function which streams the contents of a bundle to a file. + // + // If at any point this function fails, the temporary file still exists, + // and should be removed. + async fn write_and_finalize_bundle( + mut tmp_file: tokio::fs::File, + from: &Utf8Path, + to: &Utf8Path, + expected_hash: ArtifactHash, + body: StreamingBody, + ) -> Result<(), Error> { + let stream = body.into_stream(); + futures::pin_mut!(stream); + + // Write the body to the file + let mut hasher = Sha256::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + hasher.update(&chunk); + tmp_file.write_all(&chunk).await?; + } + let digest = hasher.finalize(); + if digest.as_slice() != expected_hash.as_ref() { + return Err(Error::HashMismatch); + } + + // Rename the file to indicate it's ready + tokio::fs::rename(from, to).await?; + Ok(()) + } + + pub async fn support_bundle_create( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + expected_hash: ArtifactHash, + body: StreamingBody, + ) -> Result { + let log = self.log.new(o!( + "operation" => "support_bundle_create", + "zpool_id" => zpool_id.to_string(), + "dataset_id" => dataset_id.to_string(), + "bundle_id" => support_bundle_id.to_string(), + )); + + let dataset = NestedDatasetLocation { + path: support_bundle_id.to_string(), + id: dataset_id, + root: DatasetName::new( + ZpoolName::new_external(zpool_id), + DatasetKind::Debug, + ), + }; + // The mounted root of the support bundle dataset + let support_bundle_dir = dataset + .mountpoint(illumos_utils::zpool::ZPOOL_MOUNTPOINT_ROOT.into()); + let support_bundle_path = support_bundle_dir.join("bundle"); + let support_bundle_path_tmp = support_bundle_dir.join("bundle.tmp"); + + // Ensure that the dataset exists. + info!(log, "Ensuring dataset exists for bundle"); + self.storage() + .nested_dataset_ensure(NestedDatasetConfig { + name: dataset, + inner: SharedDatasetConfig { + compression: CompressionAlgorithm::On, + quota: None, + reservation: None, + }, + }) + .await?; + + // Exit early if the support bundle already exists + if tokio::fs::try_exists(&support_bundle_path).await? { + if !Self::sha2_checksum_matches( + &support_bundle_path, + &expected_hash, + ) + .await? + { + warn!(log, "Support bundle exists, but the hash doesn't match"); + return Err(Error::HashMismatch); + } + + info!(log, "Support bundle already exists"); + let metadata = SupportBundleMetadata { + support_bundle_id, + state: SupportBundleState::Complete, + }; + return Ok(metadata); + } + + // Stream the file into the dataset, first as a temporary file, + // and then renaming to the final location. + info!(log, "Streaming bundle to storage"); + let tmp_file = + tokio::fs::File::create(&support_bundle_path_tmp).await?; + if let Err(err) = Self::write_and_finalize_bundle( + tmp_file, + &support_bundle_path_tmp, + &support_bundle_path, + expected_hash, + body, + ) + .await + { + warn!(log, "Failed to write bundle to storage"; "error" => ?err); + if let Err(unlink_err) = + tokio::fs::remove_file(support_bundle_path_tmp).await + { + warn!(log, "Failed to unlink bundle after previous error"; "error" => ?unlink_err); + } + return Err(err); + } + + info!(log, "Bundle written successfully"); + let metadata = SupportBundleMetadata { + support_bundle_id, + state: SupportBundleState::Complete, + }; + Ok(metadata) + } + + pub async fn support_bundle_delete( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + ) -> Result<(), Error> { + let log = self.log.new(o!( + "operation" => "support_bundle_delete", + "zpool_id" => zpool_id.to_string(), + "dataset_id" => dataset_id.to_string(), + "bundle_id" => support_bundle_id.to_string(), + )); + info!(log, "Destroying support bundle"); + self.storage() + .nested_dataset_destroy(NestedDatasetLocation { + path: support_bundle_id.to_string(), + id: dataset_id, + root: DatasetName::new( + ZpoolName::new_external(zpool_id), + DatasetKind::Debug, + ), + }) + .await?; + + Ok(()) + } + + async fn support_bundle_get_file( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + ) -> Result { + let dataset = NestedDatasetLocation { + path: support_bundle_id.to_string(), + id: dataset_id, + root: DatasetName::new( + ZpoolName::new_external(zpool_id), + DatasetKind::Debug, + ), + }; + // The mounted root of the support bundle dataset + let support_bundle_dir = dataset + .mountpoint(illumos_utils::zpool::ZPOOL_MOUNTPOINT_ROOT.into()); + let path = support_bundle_dir.join("bundle"); + + let f = tokio::fs::File::open(&path).await?; + Ok(f) + } + + pub async fn support_bundle_get( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + range: Option, + query: SupportBundleQueryType, + ) -> Result, Error> { + // Regardless of the type of query, we first need to access the entire + // bundle as a file. + let mut file = self + .support_bundle_get_file(zpool_id, dataset_id, support_bundle_id) + .await?; + + match query { + SupportBundleQueryType::Whole => { + let len = file.metadata().await?.len(); + + // If this has a range request, we need to validate the range + // and put bounds on the part of the file we're reading. + let (range, body) = if let Some(range) = range { + let Ok(range) = range.single_range(len) else { + return Ok(crate::range::bad_range_response(len)); + }; + + file.seek(std::io::SeekFrom::Start(range.start())).await?; + let limit = range.content_length() as usize; + ( + Some(range), + Body::wrap(http_body_util::StreamBody::new( + ReaderStream::new(file) + .take(limit) + .map_ok(|b| hyper::body::Frame::data(b)), + )), + ) + } else { + ( + None, + Body::wrap(http_body_util::StreamBody::new( + ReaderStream::new(file) + .map_ok(|b| hyper::body::Frame::data(b)), + )), + ) + }; + + return Ok(crate::range::make_response_common( + range, + len, + Some("application/zip"), + ) + .body(body) + .unwrap()); + } + SupportBundleQueryType::Index => { + let file_std = file.into_std().await; + let archive = zip::ZipArchive::new(file_std)?; + let names: Vec<&str> = archive.file_names().collect(); + let all_names = names.join("\n"); + let all_names_bytes = all_names.as_bytes(); + let len = all_names_bytes.len() as u64; + + let (range, body) = if let Some(range) = range { + let Ok(range) = range.single_range(len) else { + return Ok(crate::range::bad_range_response(len)); + }; + + let section = &all_names_bytes + [range.start() as usize..=range.end() as usize]; + (Some(range), Body::with_content(section.to_owned())) + } else { + (None, Body::with_content(all_names_bytes.to_owned())) + }; + + return Ok(crate::range::make_response_common( + range, + len, + Some("text/plain"), + ) + .body(body) + .unwrap()); + } + SupportBundleQueryType::Path { file_path } => { + let file_std = file.into_std().await; + + // TODO: Need to bound min/max of entry length + let entry_stream = + match stream_zip_entry(file_std, file_path, range)? { + // We have a valid stream + ZipStreamOutput::Stream(entry_stream) => entry_stream, + // The entry exists, but the requested range is invalid -- + // send it back as an http body. + ZipStreamOutput::RangeResponse(response) => { + return Ok(response) + } + }; + return Ok(crate::range::make_response_common( + None, + entry_stream.size, + None, + ) + .body(Body::wrap(http_body_util::StreamBody::new( + entry_stream + .stream + .map_ok(|b| hyper::body::Frame::data(b.into())), + ))) + .unwrap()); + // (Body::wrap(streamer), "text/plain") + } + }; + + // let body = FreeformBody(body); + // let mut response = + // HttpResponseHeaders::new_unnamed(HttpResponseOk(body)); + // response.headers_mut().append( + // http::header::CONTENT_TYPE, + // content_type.try_into().unwrap(), + // ); + // Ok(response) + } +} diff --git a/tufaceous-lib/src/archive.rs b/tufaceous-lib/src/archive.rs index 9440db5a4c..675cba7eee 100644 --- a/tufaceous-lib/src/archive.rs +++ b/tufaceous-lib/src/archive.rs @@ -14,7 +14,10 @@ use std::{ fmt, io::{BufReader, BufWriter, Cursor, Read, Seek}, }; -use zip::{write::FileOptions, CompressionMethod, ZipArchive, ZipWriter}; +use zip::{ + write::{FileOptions, SimpleFileOptions}, + CompressionMethod, ZipArchive, ZipWriter, +}; /// A builder for TUF repo archives. #[derive(Debug)] @@ -65,18 +68,20 @@ impl ArchiveBuilder { Ok(()) } - pub fn finish(mut self) -> Result<()> { - let zip_file = self.writer.finish().with_context(|| { - format!("error finalizing archive at `{}`", self.output_path) + pub fn finish(self) -> Result<()> { + let Self { writer, output_path } = self; + + let zip_file = writer.0.finish().with_context(|| { + format!("error finalizing archive at `{}`", output_path) })?; zip_file.into_inner().with_context(|| { - format!("error writing archive at `{}`", self.output_path) + format!("error writing archive at `{}`", output_path) })?; Ok(()) } - fn file_options() -> FileOptions { + fn file_options() -> SimpleFileOptions { // The main purpose of the zip archive is to transmit archives that are // already compressed, so there's no point trying to re-compress them. FileOptions::default().compression_method(CompressionMethod::Stored) From 0549df59614a343a92a38f7293f0b01a245ca833 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 14 Oct 2024 17:46:31 -0700 Subject: [PATCH 15/37] Share logic for ranges --- sled-agent/src/range.rs | 17 ++--- sled-agent/src/sled_agent/support_bundle.rs | 75 ++++++++------------- 2 files changed, 34 insertions(+), 58 deletions(-) diff --git a/sled-agent/src/range.rs b/sled-agent/src/range.rs index 1306c786cd..1a62a5457b 100644 --- a/sled-agent/src/range.rs +++ b/sled-agent/src/range.rs @@ -2,7 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use bytes::Bytes; use dropshot::Body; use dropshot::HttpError; use futures::TryStreamExt; @@ -10,7 +9,6 @@ use hyper::{ header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE}, Response, StatusCode, }; -use tokio::sync::mpsc; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -47,22 +45,20 @@ pub fn bad_range_response(file_size: u64) -> Response { /// Generate a GET response, optionally for a HTTP range request. The total /// file length should be provided, whether or not the expected Content-Length /// for a range request is shorter. -pub fn make_get_response( +pub fn make_get_response( range: Option, file_length: u64, content_type: Option<&str>, - rx: mpsc::Receiver>, + rx: S, ) -> Result, Error> where - E: Into> - + Send - + Sync - + 'static, + E: Send + Sync + std::error::Error + 'static, + D: Into, + S: Send + Sync + futures::stream::Stream> + 'static, { Ok(make_response_common(range, file_length, content_type).body( Body::wrap(http_body_util::StreamBody::new( - tokio_stream::wrappers::ReceiverStream::new(rx) - .map_ok(|b| hyper::body::Frame::data(b)), + rx.map_ok(|b| hyper::body::Frame::data(b.into())), )), )?) } @@ -122,6 +118,7 @@ impl PotentialRange { } } +#[derive(Clone)] pub struct SingleRange(http_range::HttpRange, u64); impl SingleRange { diff --git a/sled-agent/src/sled_agent/support_bundle.rs b/sled-agent/src/sled_agent/support_bundle.rs index 2696b51614..55288b76ec 100644 --- a/sled-agent/src/sled_agent/support_bundle.rs +++ b/sled-agent/src/sled_agent/support_bundle.rs @@ -12,7 +12,6 @@ use dropshot::Body; use dropshot::HttpError; use dropshot::StreamingBody; use futures::StreamExt; -use futures::TryStreamExt; use omicron_common::api::external::Error as ExternalError; use omicron_common::disk::CompressionAlgorithm; use omicron_common::disk::DatasetKind; @@ -54,6 +53,9 @@ pub enum Error { #[error(transparent)] Io(#[from] std::io::Error), + #[error(transparent)] + Range(#[from] crate::range::Error), + #[error(transparent)] Zip(#[from] ZipError), } @@ -70,6 +72,7 @@ impl From for HttpError { } Error::Storage(err) => HttpError::from(ExternalError::from(err)), Error::Io(err) => HttpError::for_internal_error(err.to_string()), + Error::Range(err) => HttpError::for_internal_error(err.to_string()), Error::Zip(err) => match err { ZipError::FileNotFound => HttpError::for_not_found( None, @@ -115,6 +118,7 @@ fn stream_zip_entry_helper( struct ZipEntryStream { stream: tokio_stream::wrappers::ReceiverStream, HttpError>>, + range: Option, size: u64, } @@ -165,16 +169,16 @@ fn stream_zip_entry( }; let (tx, rx) = tokio::sync::mpsc::channel(16); + let r = range.clone(); tokio::task::spawn_blocking(move || { - if let Err(err) = - stream_zip_entry_helper(&tx, archive, entry_path, range) - { + if let Err(err) = stream_zip_entry_helper(&tx, archive, entry_path, r) { let _ = tx.blocking_send(Err(err.into())); } }); Ok(ZipStreamOutput::Stream(ZipEntryStream { stream: tokio_stream::wrappers::ReceiverStream::new(rx), + range, size, })) } @@ -445,41 +449,31 @@ impl SledAgent { match query { SupportBundleQueryType::Whole => { let len = file.metadata().await?.len(); + let content_type = Some("application/zip"); - // If this has a range request, we need to validate the range - // and put bounds on the part of the file we're reading. - let (range, body) = if let Some(range) = range { + if let Some(range) = range { + // If this has a range request, we need to validate the range + // and put bounds on the part of the file we're reading. let Ok(range) = range.single_range(len) else { return Ok(crate::range::bad_range_response(len)); }; file.seek(std::io::SeekFrom::Start(range.start())).await?; let limit = range.content_length() as usize; - ( + return Ok(crate::range::make_get_response( Some(range), - Body::wrap(http_body_util::StreamBody::new( - ReaderStream::new(file) - .take(limit) - .map_ok(|b| hyper::body::Frame::data(b)), - )), - ) + len, + content_type, + ReaderStream::new(file).take(limit), + )?); } else { - ( + return Ok(crate::range::make_get_response( None, - Body::wrap(http_body_util::StreamBody::new( - ReaderStream::new(file) - .map_ok(|b| hyper::body::Frame::data(b)), - )), - ) + len, + content_type, + ReaderStream::new(file), + )?); }; - - return Ok(crate::range::make_response_common( - range, - len, - Some("application/zip"), - ) - .body(body) - .unwrap()); } SupportBundleQueryType::Index => { let file_std = file.into_std().await; @@ -512,7 +506,6 @@ impl SledAgent { SupportBundleQueryType::Path { file_path } => { let file_std = file.into_std().await; - // TODO: Need to bound min/max of entry length let entry_stream = match stream_zip_entry(file_std, file_path, range)? { // We have a valid stream @@ -523,28 +516,14 @@ impl SledAgent { return Ok(response) } }; - return Ok(crate::range::make_response_common( - None, + + return Ok(crate::range::make_get_response( + entry_stream.range, entry_stream.size, None, - ) - .body(Body::wrap(http_body_util::StreamBody::new( - entry_stream - .stream - .map_ok(|b| hyper::body::Frame::data(b.into())), - ))) - .unwrap()); - // (Body::wrap(streamer), "text/plain") + entry_stream.stream, + )?); } }; - - // let body = FreeformBody(body); - // let mut response = - // HttpResponseHeaders::new_unnamed(HttpResponseOk(body)); - // response.headers_mut().append( - // http::header::CONTENT_TYPE, - // content_type.try_into().unwrap(), - // ); - // Ok(response) } } From 4132fcbc42b138bd8aa4226f64e6eff3f12b1624 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 22 Oct 2024 14:07:15 -0700 Subject: [PATCH 16/37] hakari --- Cargo.lock | 1 + workspace-hack/Cargo.toml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 546b9aec94..e904b53a80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7165,6 +7165,7 @@ dependencies = [ "x509-cert", "zerocopy 0.7.35", "zeroize", + "zip 0.6.6", ] [[package]] diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 9f6c51e0c0..b570d9a097 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -130,6 +130,7 @@ uuid = { version = "1.10.0", features = ["serde", "v4"] } x509-cert = { version = "0.2.5" } zerocopy = { version = "0.7.35", features = ["derive", "simd"] } zeroize = { version = "1.8.1", features = ["std", "zeroize_derive"] } +zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } [build-dependencies] ahash = { version = "0.8.11" } @@ -249,6 +250,7 @@ uuid = { version = "1.10.0", features = ["serde", "v4"] } x509-cert = { version = "0.2.5" } zerocopy = { version = "0.7.35", features = ["derive", "simd"] } zeroize = { version = "1.8.1", features = ["std", "zeroize_derive"] } +zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } [target.x86_64-unknown-linux-gnu.dependencies] cookie = { version = "0.18.1", default-features = false, features = ["percent-encode"] } From fe17096f21aa1c9c7e5a849e12efa65e931f9334 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 29 Oct 2024 15:04:43 -0700 Subject: [PATCH 17/37] Working on range request crate --- range-requests/Cargo.toml | 25 ++++ range-requests/src/lib.rs | 252 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 range-requests/Cargo.toml create mode 100644 range-requests/src/lib.rs diff --git a/range-requests/Cargo.toml b/range-requests/Cargo.toml new file mode 100644 index 0000000000..df453056f9 --- /dev/null +++ b/range-requests/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "range-requests" +description = "Helpers for making and receiving range requests" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[lints] +workspace = true + +[dependencies] +bytes.workspace = true +dropshot.workspace = true +futures.workspace = true +http.workspace = true +http-range.workspace = true +http-body-util.workspace = true +hyper.workspace = true +schemars.workspace = true +serde.workspace = true +thiserror.workspace = true + +[dev-dependencies] +tokio.workspace = true +tokio-util.workspace = true diff --git a/range-requests/src/lib.rs b/range-requests/src/lib.rs new file mode 100644 index 0000000000..1260c2af47 --- /dev/null +++ b/range-requests/src/lib.rs @@ -0,0 +1,252 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use dropshot::Body; +use dropshot::HttpError; +use dropshot::HttpResponseHeaders; +use futures::TryStreamExt; +use hyper::{ + header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE}, + Response, StatusCode, +}; +use schemars::JsonSchema; +use serde::Serialize; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Using multiple ranges is not supported")] + MultipleRangesUnsupported, + + #[error("Failed to parse range")] + Parse(http_range::HttpRangeParseError), + + #[error(transparent)] + Http(#[from] http::Error), +} + +impl From for HttpError { + fn from(err: Error) -> Self { + match err { + Error::MultipleRangesUnsupported | Error::Parse(_) => { + HttpError::for_bad_request(None, err.to_string()) + } + Error::Http(err) => err.into(), + } + } +} + +pub fn bad_range_response(file_size: u64) -> Response { + hyper::Response::builder() + .status(StatusCode::RANGE_NOT_SATISFIABLE) + .header(ACCEPT_RANGES, "bytes") + .header(CONTENT_RANGE, format!("bytes */{file_size}")) + .body(Body::empty()) + .unwrap() +} + +#[derive(Serialize, JsonSchema)] +pub struct RangeHeaders { + /// The unit accepted by this range request. Typically "bytes". + #[serde(rename = "accept-ranges")] + accept_ranges: String, + /// The content being accessed by this range request. + #[serde(rename = "content-type")] + content_type: String, + /// The total length of the document accessed via range request. + #[serde(rename = "content-length")] + content_length: String, + /// The portion of this message which is accessed via range request. + #[serde(rename = "content-range")] + content_range: Option, +} + +impl Default for RangeHeaders { + fn default() -> Self { + Self { + accept_ranges: "bytes".to_string(), + content_type: "application/octet-stream".to_string(), + content_length: 0.to_string(), + content_range: None, + } + } +} + +/// Generate a GET response, optionally for a HTTP range request. The total +/// file length should be provided, whether or not the expected Content-Length +/// for a range request is shorter. +pub fn make_get_response( + range: Option, + file_length: u64, + content_type: Option<&str>, + rx: S, +) -> Result, Error> +where + E: Send + Sync + std::error::Error + 'static, + D: Into, + S: Send + Sync + futures::stream::Stream> + 'static, +{ + Ok(make_response_common(range, file_length, content_type).body( + Body::wrap(http_body_util::StreamBody::new( + rx.map_ok(|b| hyper::body::Frame::data(b.into())), + )), + )?) +} + +/// Generate a HEAD response, optionally for a HTTP range request. The total +/// file length should be provided, whether or not the expected Content-Length +/// for a range request is shorter. +pub fn make_head_response( + range: Option, + file_length: u64, + content_type: Option<&str>, +) -> Result, Error> { + Ok(make_response_common(range, file_length, content_type) + .body(Body::empty())?) +} + +fn make_response_common( + range: Option, + file_length: u64, + content_type: Option<&str>, +) -> hyper::http::response::Builder { + let mut res = Response::builder(); + res = res.header(ACCEPT_RANGES, "bytes"); + res = res.header( + CONTENT_TYPE, + content_type.unwrap_or("application/octet-stream"), + ); + + if let Some(range) = range { + res = res.header(CONTENT_LENGTH, range.content_length().to_string()); + res = res.header(CONTENT_RANGE, range.to_content_range()); + res = res.status(StatusCode::PARTIAL_CONTENT); + } else { + res = res.header(CONTENT_LENGTH, file_length.to_string()); + res = res.status(StatusCode::OK); + } + + res +} + +pub struct PotentialRange(Vec); + +impl PotentialRange { + pub fn single_range(&self, len: u64) -> Result { + match http_range::HttpRange::parse_bytes(&self.0, len) { + Ok(ranges) => { + if ranges.len() != 1 || ranges[0].length < 1 { + // Right now, we don't want to deal with encoding a + // response that has multiple ranges. + Err(Error::MultipleRangesUnsupported) + } else { + Ok(SingleRange(ranges[0], len)) + } + } + Err(err) => Err(Error::Parse(err)), + } + } +} + +#[derive(Clone)] +pub struct SingleRange(http_range::HttpRange, u64); + +impl SingleRange { + /// Return the first byte in this range for use in inclusive ranges. + pub fn start(&self) -> u64 { + self.0.start + } + + /// Return the last byte in this range for use in inclusive ranges. + pub fn end(&self) -> u64 { + assert!(self.0.length > 0); + + self.0.start.checked_add(self.0.length).unwrap().checked_sub(1).unwrap() + } + + /// Generate the Content-Range header for inclusion in a HTTP 206 partial + /// content response using this range. + pub fn to_content_range(&self) -> String { + format!("bytes {}-{}/{}", self.0.start, self.end(), self.1) + } + + /// Generate a Range header for inclusion in another HTTP request; e.g., + /// to a backend object store. + #[allow(dead_code)] + pub fn to_range(&self) -> String { + format!("bytes={}-{}", self.0.start, self.end()) + } + + pub fn content_length(&self) -> u64 { + assert!(self.0.length > 0); + + self.0.length + } +} + +pub trait RequestContextEx { + fn range(&self) -> Option; +} + +impl RequestContextEx for dropshot::RequestContext +where + T: Send + Sync + 'static, +{ + /// If there is a Range header, return it for processing during response + /// generation. + fn range(&self) -> Option { + self.request + .headers() + .get(hyper::header::RANGE) + .map(|hv| PotentialRange(hv.as_bytes().to_vec())) + } +} + +#[cfg(test)] +mod test { + use super::*; + use tokio_util::io::ReaderStream; + + #[test] + fn get_response_no_range() { + let bytes = b"Hello world"; + + let response = make_get_response( + None, + bytes.len() as u64, + None, + ReaderStream::new(bytes.as_slice()), + ).expect("Should have mader response"); + + assert_eq!(response.status(), StatusCode::OK); + + let headers = response.headers(); + println!("Headers: {headers:#?}"); + assert_eq!(headers.len(), 3); + assert_eq!(headers.get(ACCEPT_RANGES).unwrap(), "bytes"); + assert_eq!(headers.get(CONTENT_TYPE).unwrap(), "application/octet-stream"); + assert_eq!(headers.get(CONTENT_LENGTH).unwrap(), &bytes.len().to_string()); + } + + #[test] + fn get_response_with_range() { + let bytes = b"Hello world"; + + let response = make_get_response( + None, + bytes.len() as u64, + None, + ReaderStream::new(bytes.as_slice()), + ).expect("Should have mader response"); + + assert_eq!(response.status(), StatusCode::OK); + + let headers = response.headers(); + println!("Headers: {headers:#?}"); + assert_eq!(headers.len(), 3); + assert_eq!(headers.get(ACCEPT_RANGES).unwrap(), "bytes"); + assert_eq!(headers.get(CONTENT_TYPE).unwrap(), "application/octet-stream"); + assert_eq!(headers.get(CONTENT_LENGTH).unwrap(), &bytes.len().to_string()); + } + +} From 6e1b767fdb8381fb1805062bc5c372e1f115b6a6 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 29 Oct 2024 17:11:47 -0700 Subject: [PATCH 18/37] use range-requests crate --- Cargo.lock | 17 ++ Cargo.toml | 3 + range-requests/Cargo.toml | 2 - range-requests/src/lib.rs | 266 +++++++++++++++----- sled-agent/Cargo.toml | 1 + sled-agent/api/src/lib.rs | 11 + sled-agent/src/http_entrypoints.rs | 28 ++- sled-agent/src/lib.rs | 1 - sled-agent/src/range.rs | 172 ------------- sled-agent/src/sim/http_entrypoints.rs | 22 ++ sled-agent/src/sled_agent/support_bundle.rs | 75 ++++-- 11 files changed, 338 insertions(+), 260 deletions(-) delete mode 100644 sled-agent/src/range.rs diff --git a/Cargo.lock b/Cargo.lock index e904b53a80..a5ce908eb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6942,6 +6942,7 @@ dependencies = [ "propolis-mock-server", "propolis_api_types", "rand", + "range-requests", "rcgen", "reqwest 0.12.8", "schemars", @@ -8942,6 +8943,22 @@ dependencies = [ "rand_core", ] +[[package]] +name = "range-requests" +version = "0.1.0" +dependencies = [ + "bytes", + "dropshot", + "futures", + "http 1.1.0", + "http-body-util", + "http-range", + "hyper 1.4.1", + "thiserror", + "tokio", + "tokio-util", +] + [[package]] name = "ratatui" version = "0.28.1" diff --git a/Cargo.toml b/Cargo.toml index db427363d3..bf9097f376 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,6 +96,7 @@ members = [ "oximeter/types", "package", "passwords", + "range-requests", "rpaths", "sled-agent", "sled-agent/api", @@ -221,6 +222,7 @@ default-members = [ "oximeter/types", "package", "passwords", + "range-requests", "rpaths", "sled-agent", "sled-agent/api", @@ -527,6 +529,7 @@ rand = "0.8.5" rand_core = "0.6.4" rand_distr = "0.4.3" rand_seeder = "0.3.0" +range-requests = { path = "range-requests" } ratatui = "0.28.1" rayon = "1.10" rcgen = "0.12.1" diff --git a/range-requests/Cargo.toml b/range-requests/Cargo.toml index df453056f9..0d8b033258 100644 --- a/range-requests/Cargo.toml +++ b/range-requests/Cargo.toml @@ -16,8 +16,6 @@ http.workspace = true http-range.workspace = true http-body-util.workspace = true hyper.workspace = true -schemars.workspace = true -serde.workspace = true thiserror.workspace = true [dev-dependencies] diff --git a/range-requests/src/lib.rs b/range-requests/src/lib.rs index 1260c2af47..7c81c3ea85 100644 --- a/range-requests/src/lib.rs +++ b/range-requests/src/lib.rs @@ -4,15 +4,13 @@ use dropshot::Body; use dropshot::HttpError; -use dropshot::HttpResponseHeaders; use futures::TryStreamExt; use hyper::{ header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE}, Response, StatusCode, }; -use schemars::JsonSchema; -use serde::Serialize; +/// Errors which may be returned when processing range requests #[derive(thiserror::Error, Debug)] pub enum Error { #[error("Using multiple ranges is not supported")] @@ -36,7 +34,7 @@ impl From for HttpError { } } -pub fn bad_range_response(file_size: u64) -> Response { +fn bad_range_response(file_size: u64) -> Response { hyper::Response::builder() .status(StatusCode::RANGE_NOT_SATISFIABLE) .header(ACCEPT_RANGES, "bytes") @@ -45,33 +43,6 @@ pub fn bad_range_response(file_size: u64) -> Response { .unwrap() } -#[derive(Serialize, JsonSchema)] -pub struct RangeHeaders { - /// The unit accepted by this range request. Typically "bytes". - #[serde(rename = "accept-ranges")] - accept_ranges: String, - /// The content being accessed by this range request. - #[serde(rename = "content-type")] - content_type: String, - /// The total length of the document accessed via range request. - #[serde(rename = "content-length")] - content_length: String, - /// The portion of this message which is accessed via range request. - #[serde(rename = "content-range")] - content_range: Option, -} - -impl Default for RangeHeaders { - fn default() -> Self { - Self { - accept_ranges: "bytes".to_string(), - content_type: "application/octet-stream".to_string(), - content_length: 0.to_string(), - content_range: None, - } - } -} - /// Generate a GET response, optionally for a HTTP range request. The total /// file length should be provided, whether or not the expected Content-Length /// for a range request is shorter. @@ -129,10 +100,23 @@ fn make_response_common( res } +/// Represents the raw, unparsed values of "range" from a request header. pub struct PotentialRange(Vec); impl PotentialRange { - pub fn single_range(&self, len: u64) -> Result { + /// Parses a single range request out of the range request. + /// + /// `len` is the total length of the document, for the range request being made. + /// + /// On failure, returns a range response with the appropriate headers + /// to inform the caller how to make a correct range request. + pub fn parse(&self, len: u64) -> Result> { + self.single_range(len).map_err(|_err| { + bad_range_response(len) + }) + } + + fn single_range(&self, len: u64) -> Result { match http_range::HttpRange::parse_bytes(&self.0, len) { Ok(ranges) => { if ranges.len() != 1 || ranges[0].length < 1 { @@ -140,7 +124,7 @@ impl PotentialRange { // response that has multiple ranges. Err(Error::MultipleRangesUnsupported) } else { - Ok(SingleRange(ranges[0], len)) + Ok(SingleRange::new(ranges[0], len)?) } } Err(err) => Err(Error::Parse(err)), @@ -148,42 +132,80 @@ impl PotentialRange { } } -#[derive(Clone)] -pub struct SingleRange(http_range::HttpRange, u64); +/// A parsed range request, and associated "total document length". +#[derive(Clone, Debug)] +pub struct SingleRange { + range: http_range::HttpRange, + total: u64, +} + +#[cfg(test)] +impl PartialEq for SingleRange { + fn eq(&self, other: &Self) -> bool { + self.range.start == other.range.start && + self.range.length == other.range.length && + self.total == other.total + } +} impl SingleRange { + pub fn new( + range: http_range::HttpRange, + total: u64, + ) -> Result { + let http_range::HttpRange { start, mut length } = range; + + const INVALID_RANGE: Error = Error::Parse(http_range::HttpRangeParseError::InvalidRange); + + // Clip the length to avoid going beyond the end of the total range + if start.checked_add(length).ok_or(INVALID_RANGE)? >= total { + length = total.checked_sub(start).ok_or(INVALID_RANGE)?; + } + // If the length is zero, we cannot satisfy the range request + if length == 0 { + return Err(INVALID_RANGE); + } + + Ok(Self { + range: http_range::HttpRange { start, length }, + total + }) + } + /// Return the first byte in this range for use in inclusive ranges. pub fn start(&self) -> u64 { - self.0.start + self.range.start } /// Return the last byte in this range for use in inclusive ranges. - pub fn end(&self) -> u64 { - assert!(self.0.length > 0); + pub fn end_inclusive(&self) -> u64 { + assert!(self.range.length > 0); - self.0.start.checked_add(self.0.length).unwrap().checked_sub(1).unwrap() + self.range.start.checked_add(self.range.length).unwrap().checked_sub(1).unwrap() } /// Generate the Content-Range header for inclusion in a HTTP 206 partial /// content response using this range. pub fn to_content_range(&self) -> String { - format!("bytes {}-{}/{}", self.0.start, self.end(), self.1) + format!("bytes {}-{}/{}", self.range.start, self.end_inclusive(), self.total) } /// Generate a Range header for inclusion in another HTTP request; e.g., /// to a backend object store. #[allow(dead_code)] pub fn to_range(&self) -> String { - format!("bytes={}-{}", self.0.start, self.end()) + format!("bytes={}-{}", self.range.start, self.end_inclusive()) } pub fn content_length(&self) -> u64 { - assert!(self.0.length > 0); + assert!(self.range.length > 0); - self.0.length + self.range.length } } +/// A trait, implemented for [dropshot::RequestContext], to pull a range header +/// out of the request headers. pub trait RequestContextEx { fn range(&self) -> Option; } @@ -205,8 +227,62 @@ where #[cfg(test)] mod test { use super::*; + use futures::stream::once; + use std::convert::Infallible; use tokio_util::io::ReaderStream; + #[test] + fn parse_range_valid() { + // Whole range + let pr = PotentialRange(b"bytes=0-100".to_vec()); + assert_eq!( + pr.single_range(100).unwrap(), + SingleRange { + range: http_range::HttpRange { start: 0, length: 100 }, + total: 100 + } + ); + + // Clipped + let pr = PotentialRange(b"bytes=0-100".to_vec()); + assert_eq!( + pr.single_range(50).unwrap(), + SingleRange { + range: http_range::HttpRange { start: 0, length: 50 }, + total: 50 + } + ); + + // Single byte + let pr = PotentialRange(b"bytes=49-49".to_vec()); + assert_eq!( + pr.single_range(50).unwrap(), + SingleRange { + range: http_range::HttpRange { start: 49, length: 1 }, + total: 50 + } + ); + } + + #[test] + fn parse_range_invalid() { + let pr = PotentialRange(b"bytes=50-50".to_vec()); + assert!( + matches!( + pr.single_range(50).expect_err("Range should be invalid"), + Error::Parse(http_range::HttpRangeParseError::NoOverlap), + ) + ); + + let pr = PotentialRange(b"bytes=20-1".to_vec()); + assert!( + matches!( + pr.single_range(50).expect_err("Range should be invalid"), + Error::Parse(http_range::HttpRangeParseError::InvalidRange), + ) + ); + } + #[test] fn get_response_no_range() { let bytes = b"Hello world"; @@ -216,7 +292,8 @@ mod test { bytes.len() as u64, None, ReaderStream::new(bytes.as_slice()), - ).expect("Should have mader response"); + ) + .expect("Should have made response"); assert_eq!(response.status(), StatusCode::OK); @@ -224,29 +301,96 @@ mod test { println!("Headers: {headers:#?}"); assert_eq!(headers.len(), 3); assert_eq!(headers.get(ACCEPT_RANGES).unwrap(), "bytes"); - assert_eq!(headers.get(CONTENT_TYPE).unwrap(), "application/octet-stream"); - assert_eq!(headers.get(CONTENT_LENGTH).unwrap(), &bytes.len().to_string()); + assert_eq!( + headers.get(CONTENT_TYPE).unwrap(), + "application/octet-stream" + ); + assert_eq!( + headers.get(CONTENT_LENGTH).unwrap(), + &bytes.len().to_string() + ); } #[test] fn get_response_with_range() { - let bytes = b"Hello world"; - - let response = make_get_response( - None, - bytes.len() as u64, - None, - ReaderStream::new(bytes.as_slice()), - ).expect("Should have mader response"); - - assert_eq!(response.status(), StatusCode::OK); - + let ranged_get_request = |start, length, total_length| { + let range = SingleRange::new( + http_range::HttpRange { start, length }, + total_length, + ) + .unwrap(); + + let b = vec![0; length as usize]; + let response = make_get_response( + Some(range.clone()), + total_length, + None, + once(async move { Ok::<_, Infallible>(b) }), + ) + .expect("Should have made response"); + + response + }; + + // First half + let response = ranged_get_request(0, 512, 1024); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); let headers = response.headers(); println!("Headers: {headers:#?}"); - assert_eq!(headers.len(), 3); + assert_eq!(headers.len(), 4); assert_eq!(headers.get(ACCEPT_RANGES).unwrap(), "bytes"); - assert_eq!(headers.get(CONTENT_TYPE).unwrap(), "application/octet-stream"); - assert_eq!(headers.get(CONTENT_LENGTH).unwrap(), &bytes.len().to_string()); + assert_eq!( + headers.get(CONTENT_TYPE).unwrap(), + "application/octet-stream" + ); + assert_eq!(headers.get(CONTENT_LENGTH).unwrap(), "512"); + assert_eq!( + headers.get(CONTENT_RANGE).unwrap(), + &format!("bytes 0-511/1024") + ); + + // Second half + let response = ranged_get_request(512, 512, 1024); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + let headers = response.headers(); + println!("Headers: {headers:#?}"); + assert_eq!(headers.len(), 4); + assert_eq!(headers.get(ACCEPT_RANGES).unwrap(), "bytes"); + assert_eq!( + headers.get(CONTENT_TYPE).unwrap(), + "application/octet-stream" + ); + assert_eq!(headers.get(CONTENT_LENGTH).unwrap(), "512"); + assert_eq!( + headers.get(CONTENT_RANGE).unwrap(), + &format!("bytes 512-1023/1024") + ); + + // Partially out of bounds + let response = ranged_get_request(1000, 512, 1024); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + let headers = response.headers(); + println!("Headers: {headers:#?}"); + assert_eq!(headers.len(), 4); + assert_eq!(headers.get(ACCEPT_RANGES).unwrap(), "bytes"); + assert_eq!( + headers.get(CONTENT_TYPE).unwrap(), + "application/octet-stream" + ); + assert_eq!(headers.get(CONTENT_LENGTH).unwrap(), "24"); + assert_eq!( + headers.get(CONTENT_RANGE).unwrap(), + &format!("bytes 1000-1023/1024") + ); + + // Fully out of bounds + assert!(matches!( + SingleRange::new( + http_range::HttpRange { start: 1024, length: 512 }, + 1024 + ) + .expect_err("Should have thrown an error"), + Error::Parse(http_range::HttpRangeParseError::InvalidRange) + )); } - } diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 37d894bdef..31c6feff3a 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -67,6 +67,7 @@ propolis_api_types.workspace = true propolis-client.workspace = true propolis-mock-server.workspace = true # Only used by the simulated sled agent rand = { workspace = true, features = ["getrandom"] } +range-requests.workspace = true reqwest = { workspace = true, features = ["rustls-tls", "stream"] } schemars = { workspace = true, features = ["chrono", "uuid1"] } semver.workspace = true diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index 4267229ffa..fcffaad8b5 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -190,6 +190,17 @@ pub trait SledAgentApi { body: TypedBody, ) -> Result, HttpError>; + /// Fetch a service bundle from a particular dataset + #[endpoint { + method = HEAD, + path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}" + }] + async fn support_bundle_head( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result, HttpError>; + /// Delete a service bundle from a particular dataset #[endpoint { method = DELETE, diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 76d5ff06c0..dfafb375f2 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -4,7 +4,6 @@ //! HTTP entrypoint functions for the sled agent's exposed API -use super::range::RequestContextEx; use super::sled_agent::SledAgent; use crate::sled_agent::Error as SledAgentError; use crate::zone_bundle::BundleError; @@ -32,6 +31,7 @@ use omicron_common::disk::{ DatasetsConfig, DatasetsManagementResult, DiskVariant, DisksManagementResult, M2Slot, OmicronPhysicalDisksConfig, }; +use range_requests::RequestContextEx; use sled_agent_api::*; use sled_agent_types::boot_disk::{ BootDiskOsWriteStatus, BootDiskPathParams, BootDiskUpdatePathParams, @@ -274,6 +274,7 @@ impl SledAgentApi for SledAgentImpl { let range = rqctx.range(); let query = body.into_inner().query_type; + let head_only = false; Ok(sa .support_bundle_get( zpool_id, @@ -281,6 +282,31 @@ impl SledAgentApi for SledAgentImpl { support_bundle_id, range, query, + head_only, + ) + .await?) + } + + async fn support_bundle_head( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result, HttpError> { + let sa = rqctx.context(); + let SupportBundlePathParam { zpool_id, dataset_id, support_bundle_id } = + path_params.into_inner(); + + let range = rqctx.range(); + let query = body.into_inner().query_type; + let head_only = true; + Ok(sa + .support_bundle_get( + zpool_id, + dataset_id, + support_bundle_id, + range, + query, + head_only, ) .await?) } diff --git a/sled-agent/src/lib.rs b/sled-agent/src/lib.rs index befb00e2ba..a2421528e2 100644 --- a/sled-agent/src/lib.rs +++ b/sled-agent/src/lib.rs @@ -31,7 +31,6 @@ pub mod params; mod probe_manager; mod profile; pub mod rack_setup; -mod range; pub mod server; pub mod services; mod sled_agent; diff --git a/sled-agent/src/range.rs b/sled-agent/src/range.rs deleted file mode 100644 index 1a62a5457b..0000000000 --- a/sled-agent/src/range.rs +++ /dev/null @@ -1,172 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use dropshot::Body; -use dropshot::HttpError; -use futures::TryStreamExt; -use hyper::{ - header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE}, - Response, StatusCode, -}; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("Using multiple ranges is not supported")] - MultipleRangesUnsupported, - - #[error("Failed to parse range")] - Parse(http_range::HttpRangeParseError), - - #[error(transparent)] - Http(#[from] http::Error), -} - -impl From for HttpError { - fn from(err: Error) -> Self { - match err { - Error::MultipleRangesUnsupported | Error::Parse(_) => { - HttpError::for_bad_request(None, err.to_string()) - } - Error::Http(err) => err.into(), - } - } -} - -pub fn bad_range_response(file_size: u64) -> Response { - hyper::Response::builder() - .status(StatusCode::RANGE_NOT_SATISFIABLE) - .header(ACCEPT_RANGES, "bytes") - .header(CONTENT_RANGE, format!("bytes */{file_size}")) - .body(Body::empty()) - .unwrap() -} - -/// Generate a GET response, optionally for a HTTP range request. The total -/// file length should be provided, whether or not the expected Content-Length -/// for a range request is shorter. -pub fn make_get_response( - range: Option, - file_length: u64, - content_type: Option<&str>, - rx: S, -) -> Result, Error> -where - E: Send + Sync + std::error::Error + 'static, - D: Into, - S: Send + Sync + futures::stream::Stream> + 'static, -{ - Ok(make_response_common(range, file_length, content_type).body( - Body::wrap(http_body_util::StreamBody::new( - rx.map_ok(|b| hyper::body::Frame::data(b.into())), - )), - )?) -} - -/// Generate a HEAD response, optionally for a HTTP range request. The total -/// file length should be provided, whether or not the expected Content-Length -/// for a range request is shorter. -pub fn make_head_response( - range: Option, - file_length: u64, - content_type: Option<&str>, -) -> Result, Error> { - Ok(make_response_common(range, file_length, content_type) - .body(Body::empty())?) -} - -pub fn make_response_common( - range: Option, - file_length: u64, - content_type: Option<&str>, -) -> hyper::http::response::Builder { - let mut res = Response::builder(); - res = res.header(ACCEPT_RANGES, "bytes"); - res = res.header( - CONTENT_TYPE, - content_type.unwrap_or("application/octet-stream"), - ); - - if let Some(range) = range { - res = res.header(CONTENT_LENGTH, range.content_length().to_string()); - res = res.header(CONTENT_RANGE, range.to_content_range()); - res = res.status(StatusCode::PARTIAL_CONTENT); - } else { - res = res.header(CONTENT_LENGTH, file_length.to_string()); - res = res.status(StatusCode::OK); - } - - res -} - -pub struct PotentialRange(Vec); - -impl PotentialRange { - pub fn single_range(&self, len: u64) -> Result { - match http_range::HttpRange::parse_bytes(&self.0, len) { - Ok(ranges) => { - if ranges.len() != 1 || ranges[0].length < 1 { - // Right now, we don't want to deal with encoding a - // response that has multiple ranges. - Err(Error::MultipleRangesUnsupported) - } else { - Ok(SingleRange(ranges[0], len)) - } - } - Err(err) => Err(Error::Parse(err)), - } - } -} - -#[derive(Clone)] -pub struct SingleRange(http_range::HttpRange, u64); - -impl SingleRange { - /// Return the first byte in this range for use in inclusive ranges. - pub fn start(&self) -> u64 { - self.0.start - } - - /// Return the last byte in this range for use in inclusive ranges. - pub fn end(&self) -> u64 { - assert!(self.0.length > 0); - - self.0.start.checked_add(self.0.length).unwrap().checked_sub(1).unwrap() - } - - /// Generate the Content-Range header for inclusion in a HTTP 206 partial - /// content response using this range. - pub fn to_content_range(&self) -> String { - format!("bytes {}-{}/{}", self.0.start, self.end(), self.1) - } - - /// Generate a Range header for inclusion in another HTTP request; e.g., - /// to a backend object store. - pub fn to_range(&self) -> String { - format!("bytes={}-{}", self.0.start, self.end()) - } - - pub fn content_length(&self) -> u64 { - assert!(self.0.length > 0); - - self.0.length - } -} - -pub trait RequestContextEx { - fn range(&self) -> Option; -} - -impl RequestContextEx for dropshot::RequestContext -where - T: Send + Sync + 'static, -{ - /// If there is a Range header, return it for processing during response - /// generation. - fn range(&self) -> Option { - self.request - .headers() - .get(hyper::header::RANGE) - .map(|hv| PotentialRange(hv.as_bytes().to_vec())) - } -} diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 1ddf29379b..8d6d940a28 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -415,6 +415,28 @@ impl SledAgentApi for SledAgentSimImpl { .unwrap()) } + async fn support_bundle_head( + rqctx: RequestContext, + path_params: Path, + _body: TypedBody, + ) -> Result, HttpError> { + let sa = rqctx.context(); + let SupportBundlePathParam { zpool_id, dataset_id, support_bundle_id } = + path_params.into_inner(); + + sa.support_bundle_get(zpool_id, dataset_id, support_bundle_id).await?; + + let fictional_length = 10000; + + Ok(http::Response::builder() + .status(http::StatusCode::OK) + .header(http::header::CONTENT_TYPE, "text/html") + .header(hyper::header::ACCEPT_RANGES, "bytes") + .header(hyper::header::CONTENT_LENGTH, fictional_length) + .body(dropshot::Body::empty()) + .unwrap()) + } + async fn support_bundle_delete( rqctx: RequestContext, path_params: Path, diff --git a/sled-agent/src/sled_agent/support_bundle.rs b/sled-agent/src/sled_agent/support_bundle.rs index 55288b76ec..99470c8ae7 100644 --- a/sled-agent/src/sled_agent/support_bundle.rs +++ b/sled-agent/src/sled_agent/support_bundle.rs @@ -4,8 +4,6 @@ //! Management of and access to Support Bundles -use crate::range::PotentialRange; -use crate::range::SingleRange; use crate::sled_agent::SledAgent; use camino::Utf8Path; use dropshot::Body; @@ -24,6 +22,8 @@ use omicron_common::zpool_name::ZpoolName; use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::SupportBundleUuid; use omicron_uuid_kinds::ZpoolUuid; +use range_requests::PotentialRange; +use range_requests::SingleRange; use sha2::{Digest, Sha256}; use sled_agent_api::*; use sled_storage::manager::NestedDatasetListOptions; @@ -54,7 +54,7 @@ pub enum Error { Io(#[from] std::io::Error), #[error(transparent)] - Range(#[from] crate::range::Error), + Range(#[from] range_requests::Error), #[error(transparent)] Zip(#[from] ZipError), @@ -158,10 +158,9 @@ fn stream_zip_entry( }; let range = if let Some(range) = pr { - let Ok(range) = range.single_range(size) else { - return Ok(ZipStreamOutput::RangeResponse( - crate::range::bad_range_response(size), - )); + let range = match range.parse(size) { + Ok(range) => range, + Err(err) => return Ok(ZipStreamOutput::RangeResponse(err)), }; Some(range) } else { @@ -439,6 +438,7 @@ impl SledAgent { support_bundle_id: SupportBundleUuid, range: Option, query: SupportBundleQueryType, + head_only: bool, ) -> Result, Error> { // Regardless of the type of query, we first need to access the entire // bundle as a file. @@ -451,23 +451,32 @@ impl SledAgent { let len = file.metadata().await?.len(); let content_type = Some("application/zip"); + if head_only { + return Ok(range_requests::make_head_response( + None, + len, + content_type, + )?); + } + if let Some(range) = range { // If this has a range request, we need to validate the range // and put bounds on the part of the file we're reading. - let Ok(range) = range.single_range(len) else { - return Ok(crate::range::bad_range_response(len)); + let range = match range.parse(len) { + Ok(range) => range, + Err(err_response) => return Ok(err_response), }; file.seek(std::io::SeekFrom::Start(range.start())).await?; let limit = range.content_length() as usize; - return Ok(crate::range::make_get_response( + return Ok(range_requests::make_get_response( Some(range), len, content_type, ReaderStream::new(file).take(limit), )?); } else { - return Ok(crate::range::make_get_response( + return Ok(range_requests::make_get_response( None, len, content_type, @@ -482,26 +491,38 @@ impl SledAgent { let all_names = names.join("\n"); let all_names_bytes = all_names.as_bytes(); let len = all_names_bytes.len() as u64; + let content_type = Some("text/plain"); - let (range, body) = if let Some(range) = range { - let Ok(range) = range.single_range(len) else { - return Ok(crate::range::bad_range_response(len)); + if head_only { + return Ok(range_requests::make_head_response( + None, + len, + content_type, + )?); + } + + let (range, bytes) = if let Some(range) = range { + let range = match range.parse(len) { + Ok(range) => range, + Err(err_response) => return Ok(err_response), }; let section = &all_names_bytes - [range.start() as usize..=range.end() as usize]; - (Some(range), Body::with_content(section.to_owned())) + [range.start() as usize..=range.end_inclusive() as usize]; + (Some(range), section.to_owned()) } else { - (None, Body::with_content(all_names_bytes.to_owned())) + (None, all_names_bytes.to_owned()) }; - return Ok(crate::range::make_response_common( + let stream = futures::stream::once(async { + Ok::<_, std::convert::Infallible>(bytes) + }); + return Ok(range_requests::make_get_response( range, len, - Some("text/plain"), - ) - .body(body) - .unwrap()); + content_type, + stream, + )?); } SupportBundleQueryType::Path { file_path } => { let file_std = file.into_std().await; @@ -517,7 +538,15 @@ impl SledAgent { } }; - return Ok(crate::range::make_get_response( + if head_only { + return Ok(range_requests::make_head_response( + None, + entry_stream.size, + None, + )?); + } + + return Ok(range_requests::make_get_response( entry_stream.range, entry_stream.size, None, From c7ee752cc62ef6558f40428c7cf7ed2703624ea8 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 30 Oct 2024 13:50:09 -0700 Subject: [PATCH 19/37] fmt --- range-requests/src/lib.rs | 52 ++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/range-requests/src/lib.rs b/range-requests/src/lib.rs index 7c81c3ea85..4ba2518710 100644 --- a/range-requests/src/lib.rs +++ b/range-requests/src/lib.rs @@ -111,9 +111,7 @@ impl PotentialRange { /// On failure, returns a range response with the appropriate headers /// to inform the caller how to make a correct range request. pub fn parse(&self, len: u64) -> Result> { - self.single_range(len).map_err(|_err| { - bad_range_response(len) - }) + self.single_range(len).map_err(|_err| bad_range_response(len)) } fn single_range(&self, len: u64) -> Result { @@ -142,9 +140,9 @@ pub struct SingleRange { #[cfg(test)] impl PartialEq for SingleRange { fn eq(&self, other: &Self) -> bool { - self.range.start == other.range.start && - self.range.length == other.range.length && - self.total == other.total + self.range.start == other.range.start + && self.range.length == other.range.length + && self.total == other.total } } @@ -155,7 +153,8 @@ impl SingleRange { ) -> Result { let http_range::HttpRange { start, mut length } = range; - const INVALID_RANGE: Error = Error::Parse(http_range::HttpRangeParseError::InvalidRange); + const INVALID_RANGE: Error = + Error::Parse(http_range::HttpRangeParseError::InvalidRange); // Clip the length to avoid going beyond the end of the total range if start.checked_add(length).ok_or(INVALID_RANGE)? >= total { @@ -166,10 +165,7 @@ impl SingleRange { return Err(INVALID_RANGE); } - Ok(Self { - range: http_range::HttpRange { start, length }, - total - }) + Ok(Self { range: http_range::HttpRange { start, length }, total }) } /// Return the first byte in this range for use in inclusive ranges. @@ -181,13 +177,23 @@ impl SingleRange { pub fn end_inclusive(&self) -> u64 { assert!(self.range.length > 0); - self.range.start.checked_add(self.range.length).unwrap().checked_sub(1).unwrap() + self.range + .start + .checked_add(self.range.length) + .unwrap() + .checked_sub(1) + .unwrap() } /// Generate the Content-Range header for inclusion in a HTTP 206 partial /// content response using this range. pub fn to_content_range(&self) -> String { - format!("bytes {}-{}/{}", self.range.start, self.end_inclusive(), self.total) + format!( + "bytes {}-{}/{}", + self.range.start, + self.end_inclusive(), + self.total + ) } /// Generate a Range header for inclusion in another HTTP request; e.g., @@ -267,20 +273,16 @@ mod test { #[test] fn parse_range_invalid() { let pr = PotentialRange(b"bytes=50-50".to_vec()); - assert!( - matches!( - pr.single_range(50).expect_err("Range should be invalid"), - Error::Parse(http_range::HttpRangeParseError::NoOverlap), - ) - ); + assert!(matches!( + pr.single_range(50).expect_err("Range should be invalid"), + Error::Parse(http_range::HttpRangeParseError::NoOverlap), + )); let pr = PotentialRange(b"bytes=20-1".to_vec()); - assert!( - matches!( - pr.single_range(50).expect_err("Range should be invalid"), - Error::Parse(http_range::HttpRangeParseError::InvalidRange), - ) - ); + assert!(matches!( + pr.single_range(50).expect_err("Range should be invalid"), + Error::Parse(http_range::HttpRangeParseError::InvalidRange), + )); } #[test] From 915563bf4d41b9d5c8c4f3f710d795b32c15154f Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 30 Oct 2024 14:06:17 -0700 Subject: [PATCH 20/37] remove dead code warning --- range-requests/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/range-requests/src/lib.rs b/range-requests/src/lib.rs index 4ba2518710..432fd134f1 100644 --- a/range-requests/src/lib.rs +++ b/range-requests/src/lib.rs @@ -198,7 +198,6 @@ impl SingleRange { /// Generate a Range header for inclusion in another HTTP request; e.g., /// to a backend object store. - #[allow(dead_code)] pub fn to_range(&self) -> String { format!("bytes={}-{}", self.range.start, self.end_inclusive()) } From 313113737e3e162e9a46290c2dcf25491b87827b Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 30 Oct 2024 14:24:44 -0700 Subject: [PATCH 21/37] hakari --- Cargo.lock | 1 + range-requests/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 2c5fe45777..497129624b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8955,6 +8955,7 @@ dependencies = [ "http-body-util", "http-range", "hyper 1.4.1", + "omicron-workspace-hack", "thiserror", "tokio", "tokio-util", diff --git a/range-requests/Cargo.toml b/range-requests/Cargo.toml index 0d8b033258..5a9a526258 100644 --- a/range-requests/Cargo.toml +++ b/range-requests/Cargo.toml @@ -17,6 +17,7 @@ http-range.workspace = true http-body-util.workspace = true hyper.workspace = true thiserror.workspace = true +omicron-workspace-hack.workspace = true [dev-dependencies] tokio.workspace = true From 2bb6ac044cc3985c49e32407fd5e90d1d8e556f2 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 30 Oct 2024 14:25:07 -0700 Subject: [PATCH 22/37] clippy --- range-requests/src/lib.rs | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/range-requests/src/lib.rs b/range-requests/src/lib.rs index 432fd134f1..4af1bbf941 100644 --- a/range-requests/src/lib.rs +++ b/range-requests/src/lib.rs @@ -345,10 +345,7 @@ mod test { "application/octet-stream" ); assert_eq!(headers.get(CONTENT_LENGTH).unwrap(), "512"); - assert_eq!( - headers.get(CONTENT_RANGE).unwrap(), - &format!("bytes 0-511/1024") - ); + assert_eq!(headers.get(CONTENT_RANGE).unwrap(), "bytes 0-511/1024",); // Second half let response = ranged_get_request(512, 512, 1024); @@ -362,10 +359,7 @@ mod test { "application/octet-stream" ); assert_eq!(headers.get(CONTENT_LENGTH).unwrap(), "512"); - assert_eq!( - headers.get(CONTENT_RANGE).unwrap(), - &format!("bytes 512-1023/1024") - ); + assert_eq!(headers.get(CONTENT_RANGE).unwrap(), "bytes 512-1023/1024",); // Partially out of bounds let response = ranged_get_request(1000, 512, 1024); @@ -379,10 +373,7 @@ mod test { "application/octet-stream" ); assert_eq!(headers.get(CONTENT_LENGTH).unwrap(), "24"); - assert_eq!( - headers.get(CONTENT_RANGE).unwrap(), - &format!("bytes 1000-1023/1024") - ); + assert_eq!(headers.get(CONTENT_RANGE).unwrap(), "bytes 1000-1023/1024",); // Fully out of bounds assert!(matches!( From f918cdccc7b47599ff9837e0bb2aa5b8cd6a924c Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 30 Oct 2024 17:53:44 -0700 Subject: [PATCH 23/37] Actually check dataset_id, stop using DatasetKind::Update, dataset refactors --- common/src/disk.rs | 3 - sled-agent/src/sled_agent.rs | 27 +---- sled-agent/src/sled_agent/support_bundle.rs | 68 ++++++----- sled-storage/src/manager.rs | 119 +++++++++++++------- 4 files changed, 117 insertions(+), 100 deletions(-) diff --git a/common/src/disk.rs b/common/src/disk.rs index 684ae7a821..bd4642ca57 100644 --- a/common/src/disk.rs +++ b/common/src/disk.rs @@ -319,9 +319,6 @@ pub struct NestedDatasetLocation { /// A path, within the dataset root, which is being requested. pub path: String, - /// The UUID within which the dataset being requested - pub id: DatasetUuid, - /// The root in which this dataset is being requested pub root: DatasetName, } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index df886a85bc..6b294e3dbd 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -73,7 +73,6 @@ use sled_agent_types::zone_bundle::{ use sled_hardware::{underlay, HardwareManager}; use sled_hardware_types::underlay::BootstrapInterface; use sled_hardware_types::Baseboard; -use sled_storage::dataset::{CRYPT_DATASET, ZONE_DATASET}; use sled_storage::manager::StorageHandle; use slog::Logger; use sprockets_tls::keys::SprocketsConfig; @@ -1309,32 +1308,8 @@ impl SledAgent { total_size: ByteCount::try_from(info.size())?, }); - // We do care about the total space usage within zpools, but mapping - // the layering back to "datasets we care about" is a little - // awkward. - // - // We could query for all datasets within a pool, but the sled agent - // doesn't really care about the children of datasets that it - // allocates. As an example: Sled Agent might provision a "crucible" - // dataset, but how region allocation occurs within that dataset - // is a detail for Crucible to care about, not the Sled Agent. - // - // To balance this effort, we ask for information about datasets - // that the Sled Agent is directly resopnsible for managing. - let datasets_of_interest = [ - // We care about the zpool itself, and all direct children. - zpool.to_string(), - // Likewise, we care about the encrypted dataset, and all - // direct children. - format!("{zpool}/{CRYPT_DATASET}"), - // The zone dataset gives us additional context on "what zones - // have datasets provisioned". - format!("{zpool}/{ZONE_DATASET}"), - ]; let inv_props = - match illumos_utils::zfs::Zfs::get_dataset_properties( - datasets_of_interest.as_slice(), - ) { + match self.storage().datasets_list(zpool.clone()).await { Ok(props) => props .into_iter() .map(|prop| InventoryDataset::from(prop)), diff --git a/sled-agent/src/sled_agent/support_bundle.rs b/sled-agent/src/sled_agent/support_bundle.rs index 4d758d0b60..f310da117f 100644 --- a/sled-agent/src/sled_agent/support_bundle.rs +++ b/sled-agent/src/sled_agent/support_bundle.rs @@ -12,13 +12,11 @@ use dropshot::StreamingBody; use futures::StreamExt; use omicron_common::api::external::Error as ExternalError; use omicron_common::disk::CompressionAlgorithm; -use omicron_common::disk::DatasetKind; -use omicron_common::disk::DatasetName; +use omicron_common::disk::DatasetConfig; use omicron_common::disk::NestedDatasetConfig; use omicron_common::disk::NestedDatasetLocation; use omicron_common::disk::SharedDatasetConfig; use omicron_common::update::ArtifactHash; -use omicron_common::zpool_name::ZpoolName; use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::SupportBundleUuid; use omicron_uuid_kinds::ZpoolUuid; @@ -47,6 +45,9 @@ pub enum Error { #[error("Not a file")] NotAFile, + #[error("Dataset not found")] + DatasetNotFound, + #[error(transparent)] Storage(#[from] sled_storage::error::Error), @@ -67,6 +68,9 @@ impl From for HttpError { Error::HashMismatch => { HttpError::for_internal_error("Hash mismatch".to_string()) } + Error::DatasetNotFound => { + HttpError::for_not_found(None, "Dataset not found".to_string()) + } Error::NotAFile => { HttpError::for_bad_request(None, "Not a file".to_string()) } @@ -183,18 +187,33 @@ fn stream_zip_entry( } impl SledAgent { + /// Returns a dataset that the sled has been explicitly configured to use. + pub async fn get_configured_dataset( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + ) -> Result { + let datasets_config = self.storage().datasets_config_list().await?; + let dataset = datasets_config + .datasets + .get(&dataset_id) + .ok_or_else(|| Error::DatasetNotFound)?; + + if dataset.id != dataset_id || dataset.name.pool().id() != zpool_id { + return Err(Error::DatasetNotFound); + } + Ok(dataset.clone()) + } + pub async fn support_bundle_list( &self, zpool_id: ZpoolUuid, dataset_id: DatasetUuid, ) -> Result, Error> { - let root = DatasetName::new( - ZpoolName::new_external(zpool_id), - DatasetKind::Debug, - ); + let root = + self.get_configured_dataset(zpool_id, dataset_id).await?.name; let dataset_location = omicron_common::disk::NestedDatasetLocation { path: String::from(""), - id: dataset_id, root, }; let datasets = self @@ -302,15 +321,10 @@ impl SledAgent { "dataset_id" => dataset_id.to_string(), "bundle_id" => support_bundle_id.to_string(), )); - - let dataset = NestedDatasetLocation { - path: support_bundle_id.to_string(), - id: dataset_id, - root: DatasetName::new( - ZpoolName::new_external(zpool_id), - DatasetKind::Debug, - ), - }; + let root = + self.get_configured_dataset(zpool_id, dataset_id).await?.name; + let dataset = + NestedDatasetLocation { path: support_bundle_id.to_string(), root }; // The mounted root of the support bundle dataset let support_bundle_dir = dataset .mountpoint(illumos_utils::zpool::ZPOOL_MOUNTPOINT_ROOT.into()); @@ -394,14 +408,12 @@ impl SledAgent { "bundle_id" => support_bundle_id.to_string(), )); info!(log, "Destroying support bundle"); + let root = + self.get_configured_dataset(zpool_id, dataset_id).await?.name; self.storage() .nested_dataset_destroy(NestedDatasetLocation { path: support_bundle_id.to_string(), - id: dataset_id, - root: DatasetName::new( - ZpoolName::new_external(zpool_id), - DatasetKind::Debug, - ), + root, }) .await?; @@ -414,14 +426,10 @@ impl SledAgent { dataset_id: DatasetUuid, support_bundle_id: SupportBundleUuid, ) -> Result { - let dataset = NestedDatasetLocation { - path: support_bundle_id.to_string(), - id: dataset_id, - root: DatasetName::new( - ZpoolName::new_external(zpool_id), - DatasetKind::Debug, - ), - }; + let root = + self.get_configured_dataset(zpool_id, dataset_id).await?.name; + let dataset = + NestedDatasetLocation { path: support_bundle_id.to_string(), root }; // The mounted root of the support bundle dataset let support_bundle_dir = dataset .mountpoint(illumos_utils::zpool::ZPOOL_MOUNTPOINT_ROOT.into()); diff --git a/sled-storage/src/manager.rs b/sled-storage/src/manager.rs index 0fcceec360..df023fb50a 100644 --- a/sled-storage/src/manager.rs +++ b/sled-storage/src/manager.rs @@ -5,7 +5,7 @@ //! The storage manager task use crate::config::MountConfig; -use crate::dataset::CONFIG_DATASET; +use crate::dataset::{CONFIG_DATASET, CRYPT_DATASET, ZONE_DATASET}; use crate::disk::RawDisk; use crate::error::Error; use crate::resources::{AllDisks, StorageResources}; @@ -13,7 +13,7 @@ use anyhow::anyhow; use camino::Utf8PathBuf; use debug_ignore::DebugIgnore; use futures::future::FutureExt; -use illumos_utils::zfs::{Mountpoint, Zfs}; +use illumos_utils::zfs::{DatasetProperties, Mountpoint, Zfs}; use illumos_utils::zpool::{ZpoolName, ZPOOL_MOUNTPOINT_ROOT}; use key_manager::StorageKeyRequester; use omicron_common::disk::{ @@ -135,9 +135,13 @@ pub(crate) enum StorageRequest { oneshot::Sender>, >, }, - DatasetsList { + DatasetsConfigList { tx: DebugIgnore>>, }, + DatasetsList { + zpool: ZpoolName, + tx: DebugIgnore, Error>>>, + }, NestedDatasetEnsure { config: NestedDatasetConfig, @@ -306,7 +310,24 @@ impl StorageHandle { pub async fn datasets_config_list(&self) -> Result { let (tx, rx) = oneshot::channel(); self.tx - .send(StorageRequest::DatasetsList { tx: tx.into() }) + .send(StorageRequest::DatasetsConfigList { tx: tx.into() }) + .await + .unwrap(); + + rx.await.unwrap() + } + + /// Lists the datasets contained within a zpool. + /// + /// Note that this might be distinct from the last configuration + /// the Sled Agent was told to use. + pub async fn datasets_list( + &self, + zpool: ZpoolName, + ) -> Result, Error> { + let (tx, rx) = oneshot::channel(); + self.tx + .send(StorageRequest::DatasetsList { zpool, tx: tx.into() }) .await .unwrap(); @@ -552,9 +573,12 @@ impl StorageManager { StorageRequest::DatasetsEnsure { config, tx } => { let _ = tx.0.send(self.datasets_ensure(config).await); } - StorageRequest::DatasetsList { tx } => { + StorageRequest::DatasetsConfigList { tx } => { let _ = tx.0.send(self.datasets_config_list().await); } + StorageRequest::DatasetsList { zpool, tx } => { + let _ = tx.0.send(self.datasets_list(&zpool).await); + } StorageRequest::NestedDatasetEnsure { config, tx } => { let _ = tx.0.send(self.nested_dataset_ensure(config).await); } @@ -917,6 +941,31 @@ impl StorageManager { } } + // Lists datasets that this zpool contains + async fn datasets_list( + &self, + zpool: &ZpoolName, + ) -> Result, Error> { + let log = self.log.new(o!("request" => "datasets_list")); + + let datasets_of_interest = [ + // We care about the zpool itself, and all direct children. + zpool.to_string(), + // Likewise, we care about the encrypted dataset, and all + // direct children. + format!("{zpool}/{CRYPT_DATASET}"), + // The zone dataset gives us additional context on "what zones + // have datasets provisioned". + format!("{zpool}/{ZONE_DATASET}"), + ]; + + info!(log, "Listing datasets within zpool"; "zpool" => zpool.to_string()); + illumos_utils::zfs::Zfs::get_dataset_properties( + datasets_of_interest.as_slice(), + ) + .map_err(Error::Other) + } + // Ensures that a dataset exists, nested somewhere arbitrary within // a Nexus-controlled dataset. async fn nested_dataset_ensure( @@ -1005,7 +1054,6 @@ impl StorageManager { // root. name: NestedDatasetLocation { path, - id: name.id, root: name.root.clone(), }, inner: SharedDatasetConfig { @@ -1242,7 +1290,8 @@ impl StorageManager { .iter_managed() .any(|(_, disk)| disk.zpool_name() == zpool) { - return Err(Error::ZpoolNotFound(format!("{}", zpool,))); + warn!(self.log, "Failed to find zpool"); + return Err(Error::ZpoolNotFound(format!("{}", zpool))); } let DatasetCreationDetails { zoned, mountpoint, full_name } = details; @@ -1340,6 +1389,7 @@ mod tests { use omicron_test_utils::dev::test_setup_log; use sled_hardware::DiskFirmware; use std::collections::BTreeMap; + use std::str::FromStr; use std::sync::atomic::Ordering; use uuid::Uuid; @@ -1919,39 +1969,27 @@ mod tests { .expect("Ensuring disks should work after key manager is ready"); assert!(!result.has_error(), "{:?}", result); - // Create a dataset on the newly formatted U.2 - // - // NOTE: The choice of "Update" dataset here is kinda arbitrary, - // with a couple caveats: - // - // - This dataset must not be "zoned", as if it is, the nested datasets - // (which are opinionated about being mountable) do not work. - // - Calling "omicron_physical_disks_ensure" automatically creates - // some datasets (see: U2_EXPECTED_DATASETS). We want to avoid - // colliding with those, though in practice it would be fine to re-use - // them. - let id = DatasetUuid::new_v4(); - let zpool_name = ZpoolName::new_external(config.disks[0].pool_id); - let name = DatasetName::new(zpool_name.clone(), DatasetKind::Update); - let root_config = SharedDatasetConfig { - compression: CompressionAlgorithm::Off, - quota: None, - reservation: None, - }; + // Use the dataset on the newly formatted U.2 + let all_disks = harness.handle().get_latest_disks().await; + let zpool = all_disks.all_u2_zpools()[0].clone(); + let datasets = harness.handle().datasets_list(zpool).await.unwrap(); + + let dataset = datasets + .iter() + .find_map(|dataset| { + if dataset.name.contains(&DatasetKind::Debug.to_string()) { + return Some(dataset); + } + None + }) + .expect("Debug dataset not found"); - let datasets = BTreeMap::from([( - id, - DatasetConfig { - id, - name: name.clone(), - inner: root_config.clone(), - }, - )]); - let generation = Generation::new().next(); - let config = DatasetsConfig { generation, datasets }; - let status = - harness.handle().datasets_ensure(config.clone()).await.unwrap(); - assert!(!status.has_error(), "{:?}", status); + // This is a little magic; we can infer the zpool name from the "raw + // string" dataset name. + let zpool = + ZpoolName::from_str(dataset.name.split('/').next().unwrap()) + .unwrap(); + let dataset_name = DatasetName::new(zpool, DatasetKind::Debug); // Start querying the state of nested datasets. // @@ -1959,8 +1997,7 @@ mod tests { // about the dataset we're asking for. let root_location = NestedDatasetLocation { path: String::new(), - id, - root: name.clone(), + root: dataset_name.clone(), }; let nested_datasets = harness .handle() From 347b396e02c8e525cfee64ed87ebdf15b36d1cd5 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Wed, 30 Oct 2024 18:02:01 -0700 Subject: [PATCH 24/37] schemas --- openapi/sled-agent.json | 53 +++++++++++++++++++++++++++++++++ schema/rss-service-plan-v5.json | 2 +- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index ccc17cddc7..d716e1d127 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -746,6 +746,59 @@ "$ref": "#/components/responses/Error" } } + }, + "head": { + "summary": "Fetch a service bundle from a particular dataset", + "operationId": "support_bundle_head", + "parameters": [ + { + "in": "path", + "name": "dataset_id", + "description": "The dataset on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForDatasetKind" + } + }, + { + "in": "path", + "name": "support_bundle_id", + "description": "The ID of the support bundle itself", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForSupportBundleKind" + } + }, + { + "in": "path", + "name": "zpool_id", + "description": "The zpool on which this support bundle was provisioned", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForZpoolKind" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SupportBundleGetQueryParams" + } + } + }, + "required": true + }, + "responses": { + "default": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + } + } } }, "/switch-ports": { diff --git a/schema/rss-service-plan-v5.json b/schema/rss-service-plan-v5.json index 132b27c10e..93257e8540 100644 --- a/schema/rss-service-plan-v5.json +++ b/schema/rss-service-plan-v5.json @@ -546,7 +546,7 @@ ] }, "DatasetConfig": { - "description": "Configuration information necessary to request a single dataset", + "description": "Configuration information necessary to request a single dataset.\n\nThese datasets are tracked directly by Nexus.", "type": "object", "required": [ "compression", From fffabb274745ec37ff713203e5a5a2967a305b6c Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 31 Oct 2024 11:41:06 -0700 Subject: [PATCH 25/37] a lesser zip --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 1b33dce70b..81a5592ae5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -636,7 +636,7 @@ wicket-common = { path = "wicket-common" } wicketd-api = { path = "wicketd-api" } wicketd-client = { path = "clients/wicketd-client" } zeroize = { version = "1.8.1", features = ["zeroize_derive", "std"] } -zip = { version = "2.2.0", default-features = false, features = ["deflate","bzip2"] } +zip = { version = "2.1.2", default-features = false, features = ["deflate","bzip2"] } zone = { version = "0.3", default-features = false, features = ["async"] } # newtype-uuid is set to default-features = false because we don't want to From b9c15fe3248c10f56285084632d14ee5c3ab3376 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 31 Oct 2024 14:31:53 -0700 Subject: [PATCH 26/37] Fix merge --- sled-agent/src/artifact_store.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sled-agent/src/artifact_store.rs b/sled-agent/src/artifact_store.rs index fc0dc4a20a..adf1885b43 100644 --- a/sled-agent/src/artifact_store.rs +++ b/sled-agent/src/artifact_store.rs @@ -717,6 +717,7 @@ mod test { use hex_literal::hex; use omicron_common::disk::{ DatasetConfig, DatasetKind, DatasetName, DatasetsConfig, + SharedDatasetConfig, }; use omicron_common::update::ArtifactHash; use omicron_common::zpool_name::ZpoolName; @@ -747,9 +748,11 @@ mod test { ZpoolName::new_external(ZpoolUuid::new_v4()), DatasetKind::Update, ), - compression: Default::default(), - quota: None, - reservation: None, + inner: SharedDatasetConfig { + compression: Default::default(), + quota: None, + reservation: None, + }, }; let mountpoint = dataset.name.mountpoint(mountpoint_root.path()); From cc80a70f3cb366dae4907c652f381bf5a99558aa Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 31 Oct 2024 14:34:15 -0700 Subject: [PATCH 27/37] hakari of course --- Cargo.lock | 1 + workspace-hack/Cargo.toml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 3c4c8c5bfe..bc9e1bf099 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7170,6 +7170,7 @@ dependencies = [ "x509-cert", "zerocopy 0.7.35", "zeroize", + "zip 0.6.6", ] [[package]] diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index fb23570232..4e51dccf53 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -131,6 +131,7 @@ uuid = { version = "1.10.0", features = ["serde", "v4"] } x509-cert = { version = "0.2.5" } zerocopy = { version = "0.7.35", features = ["derive", "simd"] } zeroize = { version = "1.8.1", features = ["std", "zeroize_derive"] } +zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } [build-dependencies] ahash = { version = "0.8.11" } @@ -250,6 +251,7 @@ uuid = { version = "1.10.0", features = ["serde", "v4"] } x509-cert = { version = "0.2.5" } zerocopy = { version = "0.7.35", features = ["derive", "simd"] } zeroize = { version = "1.8.1", features = ["std", "zeroize_derive"] } +zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } [target.x86_64-unknown-linux-gnu.dependencies] cookie = { version = "0.18.1", default-features = false, features = ["percent-encode"] } From b86fc4bd13cd17642d8fac7082ed84fcd72701bc Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 31 Oct 2024 15:16:33 -0700 Subject: [PATCH 28/37] Refactor structs, fix clippy --- common/src/disk.rs | 75 ------------------------------------- sled-storage/src/manager.rs | 62 ++++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 82 deletions(-) diff --git a/common/src/disk.rs b/common/src/disk.rs index bd4642ca57..7ae943d08c 100644 --- a/common/src/disk.rs +++ b/common/src/disk.rs @@ -303,81 +303,6 @@ pub struct SharedDatasetConfig { pub reservation: Option, } -#[derive( - Clone, - Debug, - Deserialize, - Serialize, - JsonSchema, - PartialEq, - Eq, - Hash, - PartialOrd, - Ord, -)] -pub struct NestedDatasetLocation { - /// A path, within the dataset root, which is being requested. - pub path: String, - - /// The root in which this dataset is being requested - pub root: DatasetName, -} - -impl NestedDatasetLocation { - pub fn mountpoint(&self, root: &Utf8Path) -> Utf8PathBuf { - let mut path = Utf8Path::new(&self.path); - - // This path must be nested, so we need it to be relative to - // "self.root". However, joining paths in Rust is quirky, - // as it chooses to replace the path entirely if the argument - // to `.join(...)` is absolute. - // - // Here, we "fix" the path to make non-absolute before joining - // the paths. - while path.is_absolute() { - path = path - .strip_prefix("/") - .expect("Path is absolute, but we cannot strip '/' character"); - } - - self.root.mountpoint(root).join(path) - } - - pub fn full_name(&self) -> String { - if self.path.is_empty() { - self.root.full_name().to_string() - } else { - format!("{}/{}", self.root.full_name(), self.path) - } - } -} - -// TODO: Does this need to be here? Feels a little like an internal detail... -/// Configuration information necessary to request a single nested dataset. -/// -/// These datasets must be placed within one of the top-level datasets -/// managed directly by Nexus. -#[derive( - Clone, - Debug, - Deserialize, - Serialize, - JsonSchema, - PartialEq, - Eq, - Hash, - PartialOrd, - Ord, -)] -pub struct NestedDatasetConfig { - /// Location of this nested dataset - pub name: NestedDatasetLocation, - - /// Configuration of this dataset - #[serde(flatten)] - pub inner: SharedDatasetConfig, -} - /// Configuration information necessary to request a single dataset. /// /// These datasets are tracked directly by Nexus. diff --git a/sled-storage/src/manager.rs b/sled-storage/src/manager.rs index df023fb50a..ab91505470 100644 --- a/sled-storage/src/manager.rs +++ b/sled-storage/src/manager.rs @@ -10,6 +10,7 @@ use crate::disk::RawDisk; use crate::error::Error; use crate::resources::{AllDisks, StorageResources}; use anyhow::anyhow; +use camino::Utf8Path; use camino::Utf8PathBuf; use debug_ignore::DebugIgnore; use futures::future::FutureExt; @@ -19,8 +20,7 @@ use key_manager::StorageKeyRequester; use omicron_common::disk::{ DatasetConfig, DatasetManagementStatus, DatasetName, DatasetsConfig, DatasetsManagementResult, DiskIdentity, DiskVariant, DisksManagementResult, - NestedDatasetConfig, NestedDatasetLocation, OmicronPhysicalDisksConfig, - SharedDatasetConfig, + OmicronPhysicalDisksConfig, SharedDatasetConfig, }; use omicron_common::ledger::Ledger; use omicron_uuid_kinds::DatasetUuid; @@ -108,6 +108,57 @@ pub enum NestedDatasetListOptions { SelfAndChildren, } +/// Configuration information necessary to request a single nested dataset. +/// +/// These datasets must be placed within one of the top-level datasets +/// managed directly by Nexus. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct NestedDatasetConfig { + /// Location of this nested dataset + pub name: NestedDatasetLocation, + + /// Configuration of this dataset + pub inner: SharedDatasetConfig, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct NestedDatasetLocation { + /// A path, within the dataset root, which is being requested. + pub path: String, + + /// The root in which this dataset is being requested + pub root: DatasetName, +} + +impl NestedDatasetLocation { + pub fn mountpoint(&self, root: &Utf8Path) -> Utf8PathBuf { + let mut path = Utf8Path::new(&self.path); + + // This path must be nested, so we need it to be relative to + // "self.root". However, joining paths in Rust is quirky, + // as it chooses to replace the path entirely if the argument + // to `.join(...)` is absolute. + // + // Here, we "fix" the path to make non-absolute before joining + // the paths. + while path.is_absolute() { + path = path + .strip_prefix("/") + .expect("Path is absolute, but we cannot strip '/' character"); + } + + self.root.mountpoint(root).join(path) + } + + pub fn full_name(&self) -> String { + if self.path.is_empty() { + self.root.full_name().to_string() + } else { + format!("{}/{}", self.root.full_name(), self.path) + } + } +} + #[derive(Debug)] pub(crate) enum StorageRequest { // Requests to manage which devices the sled considers active. @@ -1976,11 +2027,8 @@ mod tests { let dataset = datasets .iter() - .find_map(|dataset| { - if dataset.name.contains(&DatasetKind::Debug.to_string()) { - return Some(dataset); - } - None + .find(|dataset| { + dataset.name.contains(&DatasetKind::Debug.to_string()) }) .expect("Debug dataset not found"); From d11d6ca993b73d6ee9cbd44cce05a2f6a35d7c06 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 31 Oct 2024 15:26:28 -0700 Subject: [PATCH 29/37] Patch paths moving in b86fc4bd13 --- sled-agent/src/sled_agent/support_bundle.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sled-agent/src/sled_agent/support_bundle.rs b/sled-agent/src/sled_agent/support_bundle.rs index f310da117f..72d6d0e086 100644 --- a/sled-agent/src/sled_agent/support_bundle.rs +++ b/sled-agent/src/sled_agent/support_bundle.rs @@ -13,8 +13,6 @@ use futures::StreamExt; use omicron_common::api::external::Error as ExternalError; use omicron_common::disk::CompressionAlgorithm; use omicron_common::disk::DatasetConfig; -use omicron_common::disk::NestedDatasetConfig; -use omicron_common::disk::NestedDatasetLocation; use omicron_common::disk::SharedDatasetConfig; use omicron_common::update::ArtifactHash; use omicron_uuid_kinds::DatasetUuid; @@ -24,7 +22,9 @@ use range_requests::PotentialRange; use range_requests::SingleRange; use sha2::{Digest, Sha256}; use sled_agent_api::*; +use sled_storage::manager::NestedDatasetConfig; use sled_storage::manager::NestedDatasetListOptions; +use sled_storage::manager::NestedDatasetLocation; use std::io::Read; use std::io::Seek; use std::io::Write; @@ -212,10 +212,8 @@ impl SledAgent { ) -> Result, Error> { let root = self.get_configured_dataset(zpool_id, dataset_id).await?.name; - let dataset_location = omicron_common::disk::NestedDatasetLocation { - path: String::from(""), - root, - }; + let dataset_location = + NestedDatasetLocation { path: String::from(""), root }; let datasets = self .storage() .nested_dataset_list( From d03fdf8749fcfbac596522addcc0658fe6cae57a Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 4 Nov 2024 15:02:07 -0800 Subject: [PATCH 30/37] Update range requests API --- Cargo.lock | 2 + range-requests/Cargo.toml | 2 + range-requests/src/lib.rs | 354 ++++++++++++++------ sled-agent/src/sled_agent/support_bundle.rs | 18 +- 4 files changed, 265 insertions(+), 111 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 509e58346d..2c34208a33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8957,10 +8957,12 @@ dependencies = [ "dropshot", "futures", "http 1.1.0", + "http-body 1.0.1", "http-body-util", "http-range", "hyper 1.4.1", "omicron-workspace-hack", + "proptest", "thiserror", "tokio", "tokio-util", diff --git a/range-requests/Cargo.toml b/range-requests/Cargo.toml index 5a9a526258..ab9972180d 100644 --- a/range-requests/Cargo.toml +++ b/range-requests/Cargo.toml @@ -20,5 +20,7 @@ thiserror.workspace = true omicron-workspace-hack.workspace = true [dev-dependencies] +http-body.workspace = true +proptest.workspace = true tokio.workspace = true tokio-util.workspace = true diff --git a/range-requests/src/lib.rs b/range-requests/src/lib.rs index 4af1bbf941..ccd250d949 100644 --- a/range-requests/src/lib.rs +++ b/range-requests/src/lib.rs @@ -3,53 +3,81 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use dropshot::Body; -use dropshot::HttpError; use futures::TryStreamExt; +use http::HeaderValue; use hyper::{ header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE}, Response, StatusCode, }; +const ACCEPT_RANGES_BYTES: http::HeaderValue = + http::HeaderValue::from_static("bytes"); +const CONTENT_TYPE_OCTET_STREAM: http::HeaderValue = + http::HeaderValue::from_static("application/octet-stream"); + /// Errors which may be returned when processing range requests #[derive(thiserror::Error, Debug)] pub enum Error { #[error("Using multiple ranges is not supported")] MultipleRangesUnsupported, - #[error("Failed to parse range")] + #[error("Range would overflow (start + length is too large)")] + RangeOverflow, + + #[error("Range would underflow (total content length < start)")] + RangeUnderflow, + + #[error("Empty Range")] + EmptyRange, + + #[error("Failed to parse range: {0:?}")] Parse(http_range::HttpRangeParseError), #[error(transparent)] Http(#[from] http::Error), } -impl From for HttpError { - fn from(err: Error) -> Self { - match err { - Error::MultipleRangesUnsupported | Error::Parse(_) => { - HttpError::for_bad_request(None, err.to_string()) - } - Error::Http(err) => err.into(), - } - } +// TODO(https://github.com/oxidecomputer/dropshot/issues/39): Return a dropshot +// type here (HttpError?) to e.g. include the RequestID in the response. +// +// Same for the other functions returning "Response" below - we're doing +// this so the "RANGE_NOT_SATISFIABLE" response can attach extra info, but it's +// currently happening at the expense of headers that Dropshot wants to supply. + +fn bad_request_response() -> Response { + hyper::Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(Body::empty()) + .expect("'bad request response' creation should be infallible") } -fn bad_range_response(file_size: u64) -> Response { +fn internal_error_response() -> Response { + hyper::Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::empty()) + .expect("'internal error response' creation should be infallible") +} + +fn not_satisfiable_response(file_size: u64) -> Response { hyper::Response::builder() .status(StatusCode::RANGE_NOT_SATISFIABLE) - .header(ACCEPT_RANGES, "bytes") + .header(ACCEPT_RANGES, ACCEPT_RANGES_BYTES) .header(CONTENT_RANGE, format!("bytes */{file_size}")) .body(Body::empty()) - .unwrap() + .expect("'not satisfiable response' creation should be infallible") } /// Generate a GET response, optionally for a HTTP range request. The total /// file length should be provided, whether or not the expected Content-Length /// for a range request is shorter. +/// +/// It is the responsibility of the caller to ensure that `rx` is a stream of +/// data matching the requested range in the `range` argument, if it is +/// supplied. pub fn make_get_response( range: Option, file_length: u64, - content_type: Option<&str>, + content_type: Option>, rx: S, ) -> Result, Error> where @@ -70,7 +98,7 @@ where pub fn make_head_response( range: Option, file_length: u64, - content_type: Option<&str>, + content_type: Option>, ) -> Result, Error> { Ok(make_response_common(range, file_length, content_type) .body(Body::empty())?) @@ -79,13 +107,13 @@ pub fn make_head_response( fn make_response_common( range: Option, file_length: u64, - content_type: Option<&str>, + content_type: Option>, ) -> hyper::http::response::Builder { let mut res = Response::builder(); - res = res.header(ACCEPT_RANGES, "bytes"); + res = res.header(ACCEPT_RANGES, ACCEPT_RANGES_BYTES); res = res.header( CONTENT_TYPE, - content_type.unwrap_or("application/octet-stream"), + content_type.map(|t| t.into()).unwrap_or(CONTENT_TYPE_OCTET_STREAM), ); if let Some(range) = range { @@ -111,7 +139,15 @@ impl PotentialRange { /// On failure, returns a range response with the appropriate headers /// to inform the caller how to make a correct range request. pub fn parse(&self, len: u64) -> Result> { - self.single_range(len).map_err(|_err| bad_range_response(len)) + self.single_range(len).map_err(|err| match err { + Error::MultipleRangesUnsupported | Error::Parse(_) => { + bad_request_response() + } + Error::RangeOverflow + | Error::RangeUnderflow + | Error::EmptyRange => not_satisfiable_response(len), + Error::Http(_err) => internal_error_response(), + }) } fn single_range(&self, len: u64) -> Result { @@ -147,22 +183,16 @@ impl PartialEq for SingleRange { } impl SingleRange { - pub fn new( - range: http_range::HttpRange, - total: u64, - ) -> Result { + fn new(range: http_range::HttpRange, total: u64) -> Result { let http_range::HttpRange { start, mut length } = range; - const INVALID_RANGE: Error = - Error::Parse(http_range::HttpRangeParseError::InvalidRange); - // Clip the length to avoid going beyond the end of the total range - if start.checked_add(length).ok_or(INVALID_RANGE)? >= total { - length = total.checked_sub(start).ok_or(INVALID_RANGE)?; + if start.checked_add(length).ok_or(Error::RangeOverflow)? >= total { + length = total.checked_sub(start).ok_or(Error::RangeUnderflow)?; } // If the length is zero, we cannot satisfy the range request if length == 0 { - return Err(INVALID_RANGE); + return Err(Error::EmptyRange); } Ok(Self { range: http_range::HttpRange { start, length }, total }) @@ -180,32 +210,39 @@ impl SingleRange { self.range .start .checked_add(self.range.length) - .unwrap() + .expect("start + length overflowed, but should have been checked in 'SingleRange::new'") .checked_sub(1) - .unwrap() + .expect("start + length underflowed, but should have been checked in 'SingleRange::new'") } /// Generate the Content-Range header for inclusion in a HTTP 206 partial /// content response using this range. - pub fn to_content_range(&self) -> String { - format!( + pub fn to_content_range(&self) -> HeaderValue { + HeaderValue::from_str(&format!( "bytes {}-{}/{}", self.range.start, self.end_inclusive(), self.total - ) + )) + .expect("Content-Range value should have been ASCII string") } /// Generate a Range header for inclusion in another HTTP request; e.g., /// to a backend object store. - pub fn to_range(&self) -> String { - format!("bytes={}-{}", self.range.start, self.end_inclusive()) + pub fn to_range(&self) -> HeaderValue { + HeaderValue::from_str(&format!( + "bytes={}-{}", + self.range.start, + self.end_inclusive() + )) + .expect("Range bounds should have been ASCII string") } - pub fn content_length(&self) -> u64 { - assert!(self.range.length > 0); - - self.range.length + /// Returns the content length for this range + pub fn content_length(&self) -> std::num::NonZeroU64 { + self.range.length.try_into().expect( + "Length should be more than zero, validated in SingleRange::new", + ) } } @@ -232,10 +269,71 @@ where #[cfg(test)] mod test { use super::*; + + use bytes::Bytes; use futures::stream::once; + use http_body_util::BodyExt; + use proptest::prelude::*; use std::convert::Infallible; use tokio_util::io::ReaderStream; + proptest! { + #[test] + fn potential_range_parsing_does_not_crash( + bytes: Vec, + len in 0_u64..=u64::MAX, + ) { + let result = PotentialRange(bytes).parse(len); + let Ok(range) = result else { return Ok(()); }; + let _ = range.start(); + let _ = range.end_inclusive(); + let _ = range.to_content_range(); + let _ = range.to_range(); + } + + #[test] + fn single_range_parsing_does_not_crash( + start in 0_u64..=u64::MAX, + length in 0_u64..=u64::MAX, + total in 0_u64..=u64::MAX + ) { + let result = SingleRange::new(http_range::HttpRange { + start, length + }, total); + + let Ok(range) = result else { return Ok(()); }; + + assert_eq!(range.start(), start); + let _ = range.end_inclusive(); + let _ = range.to_content_range(); + let _ = range.to_range(); + } + } + + #[test] + fn invalid_ranges() { + assert!(matches!( + SingleRange::new( + http_range::HttpRange { start: u64::MAX, length: 1 }, + 1 + ), + Err(Error::RangeOverflow) + )); + + assert!(matches!( + SingleRange::new( + http_range::HttpRange { start: 100, length: 0 }, + 10 + ), + Err(Error::RangeUnderflow) + )); + + assert!(matches!( + SingleRange::new(http_range::HttpRange { start: 0, length: 0 }, 1), + Err(Error::EmptyRange) + )); + } + #[test] fn parse_range_valid() { // Whole range @@ -291,98 +389,144 @@ mod test { let response = make_get_response( None, bytes.len() as u64, - None, + None::, ReaderStream::new(bytes.as_slice()), ) .expect("Should have made response"); assert_eq!(response.status(), StatusCode::OK); - let headers = response.headers(); - println!("Headers: {headers:#?}"); - assert_eq!(headers.len(), 3); - assert_eq!(headers.get(ACCEPT_RANGES).unwrap(), "bytes"); - assert_eq!( - headers.get(CONTENT_TYPE).unwrap(), - "application/octet-stream" - ); - assert_eq!( - headers.get(CONTENT_LENGTH).unwrap(), - &bytes.len().to_string() + expect_headers( + response.headers(), + &[ + (ACCEPT_RANGES, "bytes"), + (CONTENT_TYPE, "application/octet-stream"), + (CONTENT_LENGTH, &bytes.len().to_string()), + ], ); } - #[test] - fn get_response_with_range() { - let ranged_get_request = |start, length, total_length| { - let range = SingleRange::new( - http_range::HttpRange { start, length }, - total_length, - ) - .unwrap(); - - let b = vec![0; length as usize]; - let response = make_get_response( - Some(range.clone()), - total_length, - None, - once(async move { Ok::<_, Infallible>(b) }), - ) - .expect("Should have made response"); + // Makes a get response with a Vec of bytes that counts from zero. + // + // The u8s aren't normal bounds on the length, but they make the mapping + // of "the data is the index" easy. + fn ranged_get_request( + start: u8, + length: u8, + total_length: u8, + ) -> Response { + let range = SingleRange::new( + http_range::HttpRange { + start: start.into(), + length: length.into(), + }, + total_length.into(), + ) + .unwrap(); - response - }; + let b: Vec<_> = (u8::try_from(range.start()).unwrap() + ..=u8::try_from(range.end_inclusive()).unwrap()) + .collect(); + let response = make_get_response( + Some(range.clone()), + total_length.into(), + None::, + once(async move { Ok::<_, Infallible>(b) }), + ) + .expect("Should have made response"); + + response + } + + // Validates the headers exactly match the map + fn expect_headers( + headers: &http::HeaderMap, + expected: &[(http::HeaderName, &str)], + ) { + println!("Headers: {headers:#?}"); + assert_eq!(headers.len(), expected.len()); + for (k, v) in expected { + assert_eq!(headers.get(k).unwrap(), v); + } + } + + // Validates the data matches an incrementing Vec of u8 values + async fn expect_data( + body: &mut (dyn http_body::Body< + Data = Bytes, + Error = Box, + > + Unpin), + start: u8, + length: u8, + ) { + println!("Checking data from {start}, with length {length}"); + let frame = body + .frame() + .await + .expect("Error reading frame") + .expect("Should have one frame") + .into_data() + .expect("Should be a DATA frame"); + assert_eq!(frame.len(), usize::from(length),); + + for i in 0..length { + assert_eq!(frame[i as usize], i + start); + } + } + + #[tokio::test] + async fn get_response_with_range() { // First half - let response = ranged_get_request(0, 512, 1024); + let mut response = ranged_get_request(0, 32, 64); assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - let headers = response.headers(); - println!("Headers: {headers:#?}"); - assert_eq!(headers.len(), 4); - assert_eq!(headers.get(ACCEPT_RANGES).unwrap(), "bytes"); - assert_eq!( - headers.get(CONTENT_TYPE).unwrap(), - "application/octet-stream" + expect_data(response.body_mut(), 0, 32).await; + expect_headers( + response.headers(), + &[ + (ACCEPT_RANGES, "bytes"), + (CONTENT_TYPE, "application/octet-stream"), + (CONTENT_LENGTH, "32"), + (CONTENT_RANGE, "bytes 0-31/64"), + ], ); - assert_eq!(headers.get(CONTENT_LENGTH).unwrap(), "512"); - assert_eq!(headers.get(CONTENT_RANGE).unwrap(), "bytes 0-511/1024",); // Second half - let response = ranged_get_request(512, 512, 1024); + let mut response = ranged_get_request(32, 32, 64); assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - let headers = response.headers(); - println!("Headers: {headers:#?}"); - assert_eq!(headers.len(), 4); - assert_eq!(headers.get(ACCEPT_RANGES).unwrap(), "bytes"); - assert_eq!( - headers.get(CONTENT_TYPE).unwrap(), - "application/octet-stream" + expect_data(response.body_mut(), 32, 32).await; + expect_headers( + response.headers(), + &[ + (ACCEPT_RANGES, "bytes"), + (CONTENT_TYPE, "application/octet-stream"), + (CONTENT_LENGTH, "32"), + (CONTENT_RANGE, "bytes 32-63/64"), + ], ); - assert_eq!(headers.get(CONTENT_LENGTH).unwrap(), "512"); - assert_eq!(headers.get(CONTENT_RANGE).unwrap(), "bytes 512-1023/1024",); // Partially out of bounds - let response = ranged_get_request(1000, 512, 1024); + let mut response = ranged_get_request(60, 32, 64); assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); - let headers = response.headers(); - println!("Headers: {headers:#?}"); - assert_eq!(headers.len(), 4); - assert_eq!(headers.get(ACCEPT_RANGES).unwrap(), "bytes"); - assert_eq!( - headers.get(CONTENT_TYPE).unwrap(), - "application/octet-stream" + expect_data(response.body_mut(), 60, 4).await; + expect_headers( + response.headers(), + &[ + (ACCEPT_RANGES, "bytes"), + (CONTENT_TYPE, "application/octet-stream"), + (CONTENT_LENGTH, "4"), + (CONTENT_RANGE, "bytes 60-63/64"), + ], ); - assert_eq!(headers.get(CONTENT_LENGTH).unwrap(), "24"); - assert_eq!(headers.get(CONTENT_RANGE).unwrap(), "bytes 1000-1023/1024",); // Fully out of bounds assert!(matches!( SingleRange::new( - http_range::HttpRange { start: 1024, length: 512 }, - 1024 + http_range::HttpRange { start: 64, length: 32 }, + 64 ) .expect_err("Should have thrown an error"), - Error::Parse(http_range::HttpRangeParseError::InvalidRange) + Error::EmptyRange, )); } } diff --git a/sled-agent/src/sled_agent/support_bundle.rs b/sled-agent/src/sled_agent/support_bundle.rs index 72d6d0e086..b103097f7b 100644 --- a/sled-agent/src/sled_agent/support_bundle.rs +++ b/sled-agent/src/sled_agent/support_bundle.rs @@ -99,7 +99,7 @@ fn stream_zip_entry_helper( let mut reader: Box = match range { Some(range) => { reader.seek(std::io::SeekFrom::Start(range.start()))?; - Box::new(reader.take(range.content_length())) + Box::new(reader.take(range.content_length().get())) } None => Box::new(reader), }; @@ -455,7 +455,9 @@ impl SledAgent { match query { SupportBundleQueryType::Whole => { let len = file.metadata().await?.len(); - let content_type = Some("application/zip"); + const CONTENT_TYPE: http::HeaderValue = + http::HeaderValue::from_static("application/zip"); + let content_type = Some(CONTENT_TYPE); if head_only { return Ok(range_requests::make_head_response( @@ -474,7 +476,9 @@ impl SledAgent { }; file.seek(std::io::SeekFrom::Start(range.start())).await?; - let limit = range.content_length() as usize; + let limit: usize = std::num::NonZeroUsize::try_from( + range.content_length() + ).expect("Cannot convert u64 to usize; are you on a 64-bit machine?").get(); return Ok(range_requests::make_get_response( Some(range), len, @@ -497,7 +501,9 @@ impl SledAgent { let all_names = names.join("\n"); let all_names_bytes = all_names.as_bytes(); let len = all_names_bytes.len() as u64; - let content_type = Some("text/plain"); + const CONTENT_TYPE: http::HeaderValue = + http::HeaderValue::from_static("text/plain"); + let content_type = Some(CONTENT_TYPE); if head_only { return Ok(range_requests::make_head_response( @@ -548,14 +554,14 @@ impl SledAgent { return Ok(range_requests::make_head_response( None, entry_stream.size, - None, + None::, )?); } return Ok(range_requests::make_get_response( entry_stream.range, entry_stream.size, - None, + None::, entry_stream.stream, )?); } From 0a688c234ea830ee6be50d716ea900ba3409b499 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 14 Nov 2024 15:08:15 -0800 Subject: [PATCH 31/37] hakari --- Cargo.lock | 1 - nexus/tests/integration_tests/volume_management.rs | 10 ++++++++-- sled-agent/src/sled_agent.rs | 5 ++--- sled-agent/src/support_bundle/mod.rs | 2 +- workspace-hack/Cargo.toml | 2 -- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7a464ae84c..f7d6ac718d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7341,7 +7341,6 @@ dependencies = [ "getrandom", "group", "hashbrown 0.15.0", - "heck 0.4.1", "hex", "hickory-proto", "hmac", diff --git a/nexus/tests/integration_tests/volume_management.rs b/nexus/tests/integration_tests/volume_management.rs index 19f3ba0af0..249f8938db 100644 --- a/nexus/tests/integration_tests/volume_management.rs +++ b/nexus/tests/integration_tests/volume_management.rs @@ -2364,7 +2364,10 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { assert!(datasets_and_regions.is_empty()); assert_eq!(datasets_and_snapshots.len(), 3); -// Now, let's say we're at a spot where the running snapshots have been deleted, but before volume_hard_delete or region_snapshot_remove are called. Pretend another snapshot-create and snapshot-delete snuck in + + // Now, let's say we're at a spot where the running snapshots have been + // deleted, but before volume_hard_delete or region_snapshot_remove are + // called. Pretend another snapshot-create and snapshot-delete snuck in // here, and the second snapshot hits a agent that reuses the first target. for i in 3..6 { @@ -5505,7 +5508,10 @@ async fn test_migrate_to_ref_count_with_records_region_snapshot_deleting( let region_snapshot_to_delete = &snapshots_to_delete[0].1; - assert_eq!(region_snapshot_to_delete.dataset_id, region_snapshots[0].0.into_untyped_uuid()); + assert_eq!( + region_snapshot_to_delete.dataset_id, + region_snapshots[0].0.into_untyped_uuid() + ); assert_eq!(region_snapshot_to_delete.region_id, region_snapshots[0].1); assert_eq!(region_snapshot_to_delete.snapshot_id, region_snapshots[0].2); assert_eq!(region_snapshot_to_delete.snapshot_addr, region_snapshots[0].3); diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 5b8bf1b398..112443bf59 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -20,13 +20,12 @@ use crate::probe_manager::ProbeManager; use crate::services::{self, ServiceManager}; use crate::storage_monitor::StorageMonitorHandle; use crate::support_bundle::queries::{ - SupportBundleCmdError, SupportBundleCmdOutput, - zoneadm_info, ipadm_info, + ipadm_info, zoneadm_info, SupportBundleCmdError, SupportBundleCmdOutput, }; use crate::updates::{ConfigUpdates, UpdateManager}; use crate::vmm_reservoir::{ReservoirMode, VmmReservoirManager}; -use crate::zone_bundle::BundleError; use crate::zone_bundle; +use crate::zone_bundle::BundleError; use bootstore::schemes::v0 as bootstore; use camino::Utf8PathBuf; use derive_more::From; diff --git a/sled-agent/src/support_bundle/mod.rs b/sled-agent/src/support_bundle/mod.rs index ac69e6c65e..314edfaec8 100644 --- a/sled-agent/src/support_bundle/mod.rs +++ b/sled-agent/src/support_bundle/mod.rs @@ -2,5 +2,5 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -pub mod storage; pub mod queries; +pub mod storage; diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 9d1345bb7c..7152bd0177 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -63,7 +63,6 @@ generic-array = { version = "0.14.7", default-features = false, features = ["mor getrandom = { version = "0.2.15", default-features = false, features = ["js", "rdrand", "std"] } group = { version = "0.13.0", default-features = false, features = ["alloc"] } hashbrown = { version = "0.15.0" } -heck = { version = "0.4.1" } hex = { version = "0.4.3", features = ["serde"] } hickory-proto = { version = "0.24.1", features = ["text-parsing"] } hmac = { version = "0.12.1", default-features = false, features = ["reset"] } @@ -184,7 +183,6 @@ generic-array = { version = "0.14.7", default-features = false, features = ["mor getrandom = { version = "0.2.15", default-features = false, features = ["js", "rdrand", "std"] } group = { version = "0.13.0", default-features = false, features = ["alloc"] } hashbrown = { version = "0.15.0" } -heck = { version = "0.4.1" } hex = { version = "0.4.3", features = ["serde"] } hickory-proto = { version = "0.24.1", features = ["text-parsing"] } hmac = { version = "0.12.1", default-features = false, features = ["reset"] } From 43299931c9b6cb63eb45410242a986d86d21c443 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 18 Nov 2024 13:27:51 -0800 Subject: [PATCH 32/37] Downgrade zip, manually seek --- Cargo.lock | 8 ++--- Cargo.toml | 4 +-- sled-agent/src/support_bundle/storage.rs | 37 ++++++++++++++++++++---- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f7d6ac718d..0764297eb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7235,7 +7235,7 @@ dependencies = [ "uuid", "walkdir", "zeroize", - "zip 2.1.4", + "zip 2.1.3", "zone 0.3.0", ] @@ -12183,7 +12183,7 @@ dependencies = [ "toml 0.8.19", "tough", "url", - "zip 2.1.4", + "zip 2.1.3", ] [[package]] @@ -13629,9 +13629,9 @@ dependencies = [ [[package]] name = "zip" -version = "2.1.4" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e29ab4097989787b2029a5981c41b7bfb427b5a601e23f455daacb4d0360a9e9" +checksum = "775a2b471036342aa69bc5a602bc889cb0a06cda00477d0c69566757d5553d39" dependencies = [ "arbitrary", "bzip2", diff --git a/Cargo.toml b/Cargo.toml index fb25ca04df..c33c253581 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -658,9 +658,7 @@ wicketd-api = { path = "wicketd-api" } wicketd-client = { path = "clients/wicketd-client" } zeroize = { version = "1.8.1", features = ["zeroize_derive", "std"] } # NOTE: Avoid upgrading zip until https://github.com/zip-rs/zip2/issues/231 is resolved -# XXX XXX THIS PR IS DOING THIS UPGRADE TO GET ACCESS TO SEEKABLE APIS -# XXX CONSIDER USING https://github.com/bearcove/rc-zip FOR READING -zip = { version = "=2.1.4", default-features = false, features = ["deflate","bzip2"] } +zip = { version = "=2.1.3", default-features = false, features = ["deflate","bzip2"] } zone = { version = "0.3", default-features = false, features = ["async"] } # newtype-uuid is set to default-features = false because we don't want to diff --git a/sled-agent/src/support_bundle/storage.rs b/sled-agent/src/support_bundle/storage.rs index b103097f7b..401481c44b 100644 --- a/sled-agent/src/support_bundle/storage.rs +++ b/sled-agent/src/support_bundle/storage.rs @@ -26,7 +26,6 @@ use sled_storage::manager::NestedDatasetConfig; use sled_storage::manager::NestedDatasetListOptions; use sled_storage::manager::NestedDatasetLocation; use std::io::Read; -use std::io::Seek; use std::io::Write; use tokio::io::AsyncReadExt; use tokio::io::AsyncSeekExt; @@ -88,19 +87,47 @@ impl From for HttpError { } } +// Implements "seeking" and "putting a capacity on a file" manually. +// +// TODO: When https://github.com/zip-rs/zip2/issues/231 is resolved, +// this method should be replaced by calling "seek" directly, +// via the "by_name_seek" method from the zip crate. +fn skip_and_limit( + mut reader: impl std::io::Read, + skip: usize, + limit: usize, +) -> std::io::Result { + const BUF_SIZE: usize = 4096; + let mut buf = vec![0; BUF_SIZE]; + let mut skip_left = skip; + + while skip_left > 0 { + let to_read = std::cmp::min(skip_left, BUF_SIZE); + reader.read_exact(&mut buf[0..to_read])?; + skip_left -= to_read; + } + + Ok(reader.take(limit as u64)) +} + fn stream_zip_entry_helper( tx: &tokio::sync::mpsc::Sender, HttpError>>, mut archive: zip::ZipArchive, entry_path: String, range: Option, ) -> Result<(), Error> { - let mut reader = archive.by_name_seek(&entry_path)?; + // TODO: When https://github.com/zip-rs/zip2/issues/231 is resolved, + // this should call "by_name_seek" instead. + let reader = archive.by_name(&entry_path)?; let mut reader: Box = match range { Some(range) => { - reader.seek(std::io::SeekFrom::Start(range.start()))?; - Box::new(reader.take(range.content_length().get())) - } + Box::new(skip_and_limit( + reader, + range.start() as usize, + range.content_length().get() as usize + )?) + }, None => Box::new(reader), }; From 56807ceae3fe1dc2489a4037fc505372f31037ed Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Mon, 18 Nov 2024 17:59:54 -0800 Subject: [PATCH 33/37] Start working on tests --- sled-agent/api/src/lib.rs | 2 +- sled-agent/src/http_entrypoints.rs | 33 +-- sled-agent/src/sled_agent.rs | 6 + sled-agent/src/support_bundle/storage.rs | 326 +++++++++++++++++++++-- sled-storage/src/manager.rs | 4 +- 5 files changed, 322 insertions(+), 49 deletions(-) diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index 8962e5f3af..b8d365c54e 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -712,7 +712,7 @@ pub enum SupportBundleQueryType { Path { file_path: String }, } -#[derive(Deserialize, Serialize, JsonSchema, PartialEq)] +#[derive(Deserialize, Debug, Serialize, JsonSchema, PartialEq)] #[serde(rename_all = "snake_case")] pub enum SupportBundleState { Complete, diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 927c8846da..ef8399f706 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -235,7 +235,8 @@ impl SledAgentApi for SledAgentImpl { let SupportBundleListPathParam { zpool_id, dataset_id } = path_params.into_inner(); - let bundles = sa.support_bundle_list(zpool_id, dataset_id).await?; + let bundles = + sa.as_support_bundle_storage().list(zpool_id, dataset_id).await?; Ok(HttpResponseOk(bundles)) } @@ -253,12 +254,13 @@ impl SledAgentApi for SledAgentImpl { let SupportBundleCreateQueryParams { hash } = query_params.into_inner(); let metadata = sa - .support_bundle_create( + .as_support_bundle_storage() + .create( zpool_id, dataset_id, support_bundle_id, hash, - body, + body.into_stream(), ) .await?; @@ -276,16 +278,9 @@ impl SledAgentApi for SledAgentImpl { let range = rqctx.range(); let query = body.into_inner().query_type; - let head_only = false; Ok(sa - .support_bundle_get( - zpool_id, - dataset_id, - support_bundle_id, - range, - query, - head_only, - ) + .as_support_bundle_storage() + .get(zpool_id, dataset_id, support_bundle_id, range, query) .await?) } @@ -300,16 +295,9 @@ impl SledAgentApi for SledAgentImpl { let range = rqctx.range(); let query = body.into_inner().query_type; - let head_only = true; Ok(sa - .support_bundle_get( - zpool_id, - dataset_id, - support_bundle_id, - range, - query, - head_only, - ) + .as_support_bundle_storage() + .head(zpool_id, dataset_id, support_bundle_id, range, query) .await?) } @@ -322,7 +310,8 @@ impl SledAgentApi for SledAgentImpl { let SupportBundlePathParam { zpool_id, dataset_id, support_bundle_id } = path_params.into_inner(); - sa.support_bundle_delete(zpool_id, dataset_id, support_bundle_id) + sa.as_support_bundle_storage() + .delete(zpool_id, dataset_id, support_bundle_id) .await?; Ok(HttpResponseDeleted()) } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 112443bf59..3b5ce53ab7 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -22,6 +22,7 @@ use crate::storage_monitor::StorageMonitorHandle; use crate::support_bundle::queries::{ ipadm_info, zoneadm_info, SupportBundleCmdError, SupportBundleCmdOutput, }; +use crate::support_bundle::storage::SupportBundleManager; use crate::updates::{ConfigUpdates, UpdateManager}; use crate::vmm_reservoir::{ReservoirMode, VmmReservoirManager}; use crate::zone_bundle; @@ -697,6 +698,11 @@ impl SledAgent { .unwrap(); // we retry forever, so this can't fail } + /// Accesses the [SupportBundleManager] API. + pub(crate) fn as_support_bundle_storage(&self) -> SupportBundleManager<'_> { + SupportBundleManager::new(&self.log, self.storage()) + } + pub(crate) fn switch_zone_underlay_info( &self, ) -> (Ipv6Addr, Option<&RackNetworkConfig>) { diff --git a/sled-agent/src/support_bundle/storage.rs b/sled-agent/src/support_bundle/storage.rs index 401481c44b..9d280c41bf 100644 --- a/sled-agent/src/support_bundle/storage.rs +++ b/sled-agent/src/support_bundle/storage.rs @@ -4,11 +4,11 @@ //! Management of and access to Support Bundles -use crate::sled_agent::SledAgent; +use bytes::Bytes; use camino::Utf8Path; use dropshot::Body; use dropshot::HttpError; -use dropshot::StreamingBody; +use futures::Stream; use futures::StreamExt; use omicron_common::api::external::Error as ExternalError; use omicron_common::disk::CompressionAlgorithm; @@ -25,6 +25,8 @@ use sled_agent_api::*; use sled_storage::manager::NestedDatasetConfig; use sled_storage::manager::NestedDatasetListOptions; use sled_storage::manager::NestedDatasetLocation; +use sled_storage::manager::StorageHandle; +use slog::Logger; use std::io::Read; use std::io::Write; use tokio::io::AsyncReadExt; @@ -121,13 +123,11 @@ fn stream_zip_entry_helper( let reader = archive.by_name(&entry_path)?; let mut reader: Box = match range { - Some(range) => { - Box::new(skip_and_limit( - reader, - range.start() as usize, - range.content_length().get() as usize - )?) - }, + Some(range) => Box::new(skip_and_limit( + reader, + range.start() as usize, + range.content_length().get() as usize, + )?), None => Box::new(reader), }; @@ -213,14 +213,29 @@ fn stream_zip_entry( })) } -impl SledAgent { - /// Returns a dataset that the sled has been explicitly configured to use. - pub async fn get_configured_dataset( +/// APIs to manage support bundle storage. +pub struct SupportBundleManager<'a> { + log: &'a Logger, + storage: &'a StorageHandle, +} + +impl<'a> SupportBundleManager<'a> { + /// Creates a new SupportBundleManager, which provides access + /// to support bundle CRUD APIs. + pub fn new( + log: &'a Logger, + storage: &'a StorageHandle, + ) -> SupportBundleManager<'a> { + Self { log, storage } + } + + // Returns a dataset that the sled has been explicitly configured to use. + async fn get_configured_dataset( &self, zpool_id: ZpoolUuid, dataset_id: DatasetUuid, ) -> Result { - let datasets_config = self.storage().datasets_config_list().await?; + let datasets_config = self.storage.datasets_config_list().await?; let dataset = datasets_config .datasets .get(&dataset_id) @@ -232,7 +247,8 @@ impl SledAgent { Ok(dataset.clone()) } - pub async fn support_bundle_list( + /// Lists all support bundles on a particular dataset. + pub async fn list( &self, zpool_id: ZpoolUuid, dataset_id: DatasetUuid, @@ -242,7 +258,7 @@ impl SledAgent { let dataset_location = NestedDatasetLocation { path: String::from(""), root }; let datasets = self - .storage() + .storage .nested_dataset_list( dataset_location, NestedDatasetListOptions::ChildrenOnly, @@ -310,9 +326,8 @@ impl SledAgent { from: &Utf8Path, to: &Utf8Path, expected_hash: ArtifactHash, - body: StreamingBody, + stream: impl Stream>, ) -> Result<(), Error> { - let stream = body.into_stream(); futures::pin_mut!(stream); // Write the body to the file @@ -332,13 +347,14 @@ impl SledAgent { Ok(()) } - pub async fn support_bundle_create( + /// Creates a new support bundle on a dataset. + pub async fn create( &self, zpool_id: ZpoolUuid, dataset_id: DatasetUuid, support_bundle_id: SupportBundleUuid, expected_hash: ArtifactHash, - body: StreamingBody, + stream: impl Stream>, ) -> Result { let log = self.log.new(o!( "operation" => "support_bundle_create", @@ -358,7 +374,7 @@ impl SledAgent { // Ensure that the dataset exists. info!(log, "Ensuring dataset exists for bundle"); - self.storage() + self.storage .nested_dataset_ensure(NestedDatasetConfig { name: dataset, inner: SharedDatasetConfig { @@ -399,7 +415,7 @@ impl SledAgent { &support_bundle_path_tmp, &support_bundle_path, expected_hash, - body, + stream, ) .await { @@ -420,7 +436,8 @@ impl SledAgent { Ok(metadata) } - pub async fn support_bundle_delete( + /// Destroys a support bundle that exists on a dataset. + pub async fn delete( &self, zpool_id: ZpoolUuid, dataset_id: DatasetUuid, @@ -435,7 +452,7 @@ impl SledAgent { info!(log, "Destroying support bundle"); let root = self.get_configured_dataset(zpool_id, dataset_id).await?.name; - self.storage() + self.storage .nested_dataset_destroy(NestedDatasetLocation { path: support_bundle_id.to_string(), root, @@ -464,7 +481,48 @@ impl SledAgent { Ok(f) } - pub async fn support_bundle_get( + /// Streams a support bundle (or portion of a support bundle) from a + /// dataset. + pub async fn get( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + range: Option, + query: SupportBundleQueryType, + ) -> Result, Error> { + self.get_inner( + zpool_id, + dataset_id, + support_bundle_id, + range, + query, + false, + ) + .await + } + + /// Returns metadata about a support bundle. + pub async fn head( + &self, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid, + support_bundle_id: SupportBundleUuid, + range: Option, + query: SupportBundleQueryType, + ) -> Result, Error> { + self.get_inner( + zpool_id, + dataset_id, + support_bundle_id, + range, + query, + true, + ) + .await + } + + async fn get_inner( &self, zpool_id: ZpoolUuid, dataset_id: DatasetUuid, @@ -595,3 +653,223 @@ impl SledAgent { }; } } + +// #[cfg(all(test, target_os = "illumos"))] +#[cfg(test)] +mod tests { + use super::*; + + use futures::stream; + use hyper::header::{ACCCPT_RANGES, CONTENT_LENGTH, CONTENT_TYPE}; + use omicron_common::disk::DatasetConfig; + use omicron_common::disk::DatasetKind; + use omicron_common::disk::DatasetName; + use omicron_common::disk::DatasetsConfig; + use omicron_common::zpool_name::ZpoolName; + use omicron_test_utils::dev::test_setup_log; + use sled_storage::manager_test_harness::StorageManagerTestHarness; + use std::collections::BTreeMap; + use zip::write::SimpleFileOptions; + use zip::ZipWriter; + + struct SingleU2StorageHarness { + storage_test_harness: StorageManagerTestHarness, + zpool_id: ZpoolUuid, + } + + impl SingleU2StorageHarness { + async fn new(log: &Logger) -> Self { + let mut harness = StorageManagerTestHarness::new(log).await; + harness.handle().key_manager_ready().await; + let _raw_internal_disks = + harness.add_vdevs(&["m2_left.vdev", "m2_right.vdev"]).await; + + let raw_disks = harness.add_vdevs(&["u2_0.vdev"]).await; + + let config = harness.make_config(1, &raw_disks); + let result = harness + .handle() + .omicron_physical_disks_ensure(config.clone()) + .await + .expect("Failed to ensure disks"); + assert!(!result.has_error(), "{result:?}"); + + let zpool_id = config.disks[0].pool_id; + Self { storage_test_harness: harness, zpool_id } + } + + async fn configure_dataset( + &self, + dataset_id: DatasetUuid, + kind: DatasetKind, + ) { + let result = self + .storage_test_harness + .handle() + .datasets_ensure(DatasetsConfig { + datasets: BTreeMap::from([( + dataset_id, + DatasetConfig { + id: dataset_id, + name: DatasetName::new( + ZpoolName::new_external(self.zpool_id), + kind, + ), + inner: Default::default(), + }, + )]), + ..Default::default() + }) + .await + .expect("Failed to ensure datasets"); + assert!(!result.has_error(), "{result:?}"); + } + + async fn cleanup(mut self) { + self.storage_test_harness.cleanup().await + } + } + + enum Data { + File(&'static [u8]), + Directory, + } + type NamedFile = (&'static str, Data); + fn example_files() -> [NamedFile; 5] { + [ + ("greeting.txt", Data::File(b"Hello around the world!")), + ("english/", Data::Directory), + ("english/hello.txt", Data::File(b"Hello world!")), + ("spanish/", Data::Directory), + ("spanish/hello.txt", Data::File(b"Hola mundo!")), + ] + } + + fn example_zipfile() -> Vec { + let mut buf = vec![0u8; 65536]; + let len = { + let mut zip = ZipWriter::new(std::io::Cursor::new(&mut buf[..])); + let options = SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored); + + for (name, data) in example_files() { + match data { + Data::File(data) => { + zip.start_file(name, options).unwrap(); + zip.write(data).unwrap(); + } + Data::Directory => { + zip.add_directory(name, options).unwrap(); + } + } + } + zip.finish().unwrap().position() + }; + buf.truncate(len as usize); + buf + } + + async fn read_body(response: &mut http::Response) -> Vec { + use http_body_util::BodyExt; + let mut data = vec![]; + while let Some(frame) = response.body_mut().frame().await { + data.append(&mut frame.unwrap().into_data().unwrap().to_vec()); + } + data + } + + #[tokio::test] + async fn basic_crud() { + let logctx = test_setup_log("basic_crud"); + let log = &logctx.log; + + // Set up storage + let harness = SingleU2StorageHarness::new(log).await; + + // For this test, we'll add a dataset that can contain our bundles. + let dataset_id = DatasetUuid::new_v4(); + harness.configure_dataset(dataset_id, DatasetKind::Debug).await; + + // Access the Support Bundle API + let mgr = SupportBundleManager::new( + log, + harness.storage_test_harness.handle(), + ); + + // Create a fake support bundle -- really, just a zipfile. + let support_bundle_id = SupportBundleUuid::new_v4(); + let data = example_zipfile(); + let hash = ArtifactHash( + Sha256::digest(data.as_slice()).as_slice().try_into().unwrap(), + ); + + // Create a new bundle + let bundle = mgr + .create( + harness.zpool_id, + dataset_id, + support_bundle_id, + hash, + stream::once(async { + Ok(Bytes::copy_from_slice(data.as_slice())) + }), + ) + .await + .expect("Should have created support bundle"); + assert_eq!(bundle.support_bundle_id, support_bundle_id); + assert_eq!(bundle.state, SupportBundleState::Complete); + + // List the bundle we just created + let bundles = mgr + .list(harness.zpool_id, dataset_id) + .await + .expect("Should have been able to read bundles"); + assert_eq!(bundles.len(), 1); + assert_eq!(bundles[0].support_bundle_id, support_bundle_id); + + // "Head" the bundle we created - we can see it's a zipfile with the + // expected length, even without reading anything. + let mut response = mgr + .head( + harness.zpool_id, + dataset_id, + support_bundle_id, + None, + SupportBundleQueryType::Whole, + ) + .await + .expect("Should have been able to HEAD bundle"); + assert_eq!(read_body(&mut response).await, Vec::::new()); + assert_eq!(response.headers().len(), 3); + assert_eq!(response.headers()[CONTENT_LENGTH], data.len().to_string()); + assert_eq!(response.headers()[CONTENT_TYPE], "application/zip"); + assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); + + let mut response = mgr + .head( + harness.zpool_id, + dataset_id, + support_bundle_id, + None, + SupportBundleQueryType::Index, + ) + .await + .expect("Should have been able to HEAD bundle index"); + assert_eq!(read_body(&mut response).await, Vec::::new()); + + let expected_index = example_files() + .into_iter() + .map(|(name, _)| name) + .collect::>() + .join("\n"); + let expected_len = expected_index.len().to_string(); + + assert_eq!(response.headers().len(), 3); + assert_eq!(response.headers()[CONTENT_LENGTH], expected_len); + assert_eq!(response.headers()[CONTENT_TYPE], "text/plain"); + assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); + + harness.cleanup().await; + logctx.cleanup_successful(); + } +} diff --git a/sled-storage/src/manager.rs b/sled-storage/src/manager.rs index 806ca50a3c..a760285d3f 100644 --- a/sled-storage/src/manager.rs +++ b/sled-storage/src/manager.rs @@ -952,8 +952,8 @@ impl StorageManager { err: None, }; - let mountpoint_path = - config.name.mountpoint(ZPOOL_MOUNTPOINT_ROOT.into()); + let mountpoint_root = &self.resources.disks().mount_config().root; + let mountpoint_path = config.name.mountpoint(mountpoint_root); let details = DatasetCreationDetails { zoned: config.name.dataset().zoned(), mountpoint: Mountpoint::Path(mountpoint_path), From a6e0c885b49cf0cb1af0f9d766a57168f133ec70 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 19 Nov 2024 16:27:12 -0800 Subject: [PATCH 34/37] Full test suite --- range-requests/src/lib.rs | 4 + sled-agent/api/src/lib.rs | 2 +- sled-agent/src/support_bundle/storage.rs | 552 ++++++++++++++++++++++- sled-storage/src/manager_test_harness.rs | 51 ++- 4 files changed, 577 insertions(+), 32 deletions(-) diff --git a/range-requests/src/lib.rs b/range-requests/src/lib.rs index ccd250d949..dffb2e0859 100644 --- a/range-requests/src/lib.rs +++ b/range-requests/src/lib.rs @@ -132,6 +132,10 @@ fn make_response_common( pub struct PotentialRange(Vec); impl PotentialRange { + pub fn new(bytes: &[u8]) -> Self { + Self(Vec::from(bytes)) + } + /// Parses a single range request out of the range request. /// /// `len` is the total length of the document, for the range request being made. diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index b8d365c54e..9083941ca4 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -720,7 +720,7 @@ pub enum SupportBundleState { } /// Metadata about a support bundle -#[derive(Deserialize, Serialize, JsonSchema)] +#[derive(Debug, Deserialize, Serialize, JsonSchema)] pub struct SupportBundleMetadata { pub support_bundle_id: SupportBundleUuid, pub state: SupportBundleState, diff --git a/sled-agent/src/support_bundle/storage.rs b/sled-agent/src/support_bundle/storage.rs index 9d280c41bf..94a5f80791 100644 --- a/sled-agent/src/support_bundle/storage.rs +++ b/sled-agent/src/support_bundle/storage.rs @@ -67,7 +67,7 @@ impl From for HttpError { match err { Error::HttpError(err) => err, Error::HashMismatch => { - HttpError::for_internal_error("Hash mismatch".to_string()) + HttpError::for_bad_request(None, "Hash mismatch".to_string()) } Error::DatasetNotFound => { HttpError::for_not_found(None, "Dataset not found".to_string()) @@ -156,7 +156,6 @@ struct ZipEntryStream { // Possible responses from the success case of `stream_zip_entry` enum ZipStreamOutput { // Returns the zip entry, as a byte stream - // TODO: do we need to pass the size back? or are we good? Stream(ZipEntryStream), // Returns an HTTP response indicating the accepted ranges RangeResponse(http::Response), @@ -560,15 +559,21 @@ impl<'a> SupportBundleManager<'a> { Err(err_response) => return Ok(err_response), }; + info!( + &self.log, + "SupportBundle GET whole file (ranged)"; + "bundle_id" => %support_bundle_id, + "start" => range.start(), + "limit" => range.content_length().get(), + ); + file.seek(std::io::SeekFrom::Start(range.start())).await?; - let limit: usize = std::num::NonZeroUsize::try_from( - range.content_length() - ).expect("Cannot convert u64 to usize; are you on a 64-bit machine?").get(); + let limit = range.content_length().get(); return Ok(range_requests::make_get_response( Some(range), len, content_type, - ReaderStream::new(file).take(limit), + ReaderStream::new(file.take(limit)), )?); } else { return Ok(range_requests::make_get_response( @@ -654,13 +659,15 @@ impl<'a> SupportBundleManager<'a> { } } -// #[cfg(all(test, target_os = "illumos"))] -#[cfg(test)] +#[cfg(all(test, target_os = "illumos"))] mod tests { use super::*; use futures::stream; - use hyper::header::{ACCCPT_RANGES, CONTENT_LENGTH, CONTENT_TYPE}; + use http::status::StatusCode; + use hyper::header::{ + ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, + }; use omicron_common::disk::DatasetConfig; use omicron_common::disk::DatasetKind; use omicron_common::disk::DatasetName; @@ -735,9 +742,13 @@ mod tests { Directory, } type NamedFile = (&'static str, Data); + + const GREET_PATH: &'static str = "greeting.txt"; + const GREET_DATA: &'static [u8] = b"Hello around the world!"; + fn example_files() -> [NamedFile; 5] { [ - ("greeting.txt", Data::File(b"Hello around the world!")), + (GREET_PATH, Data::File(GREET_DATA)), ("english/", Data::Directory), ("english/hello.txt", Data::File(b"Hello world!")), ("spanish/", Data::Directory), @@ -756,7 +767,7 @@ mod tests { match data { Data::File(data) => { zip.start_file(name, options).unwrap(); - zip.write(data).unwrap(); + zip.write_all(data).unwrap(); } Data::Directory => { zip.add_directory(name, options).unwrap(); @@ -798,9 +809,12 @@ mod tests { // Create a fake support bundle -- really, just a zipfile. let support_bundle_id = SupportBundleUuid::new_v4(); - let data = example_zipfile(); + let zipfile_data = example_zipfile(); let hash = ArtifactHash( - Sha256::digest(data.as_slice()).as_slice().try_into().unwrap(), + Sha256::digest(zipfile_data.as_slice()) + .as_slice() + .try_into() + .unwrap(), ); // Create a new bundle @@ -811,7 +825,7 @@ mod tests { support_bundle_id, hash, stream::once(async { - Ok(Bytes::copy_from_slice(data.as_slice())) + Ok(Bytes::copy_from_slice(zipfile_data.as_slice())) }), ) .await @@ -827,7 +841,7 @@ mod tests { assert_eq!(bundles.len(), 1); assert_eq!(bundles[0].support_bundle_id, support_bundle_id); - // "Head" the bundle we created - we can see it's a zipfile with the + // HEAD the bundle we created - we can see it's a zipfile with the // expected length, even without reading anything. let mut response = mgr .head( @@ -841,10 +855,35 @@ mod tests { .expect("Should have been able to HEAD bundle"); assert_eq!(read_body(&mut response).await, Vec::::new()); assert_eq!(response.headers().len(), 3); - assert_eq!(response.headers()[CONTENT_LENGTH], data.len().to_string()); + assert_eq!( + response.headers()[CONTENT_LENGTH], + zipfile_data.len().to_string() + ); + assert_eq!(response.headers()[CONTENT_TYPE], "application/zip"); + assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); + + // GET the bundle we created, and observe the contents of the bundle + let mut response = mgr + .get( + harness.zpool_id, + dataset_id, + support_bundle_id, + None, + SupportBundleQueryType::Whole, + ) + .await + .expect("Should have been able to GET bundle"); + assert_eq!(read_body(&mut response).await, zipfile_data); + assert_eq!(response.headers().len(), 3); + assert_eq!( + response.headers()[CONTENT_LENGTH], + zipfile_data.len().to_string() + ); assert_eq!(response.headers()[CONTENT_TYPE], "application/zip"); assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); + // HEAD the index of the bundle - it should report the size of all + // files. let mut response = mgr .head( harness.zpool_id, @@ -856,19 +895,498 @@ mod tests { .await .expect("Should have been able to HEAD bundle index"); assert_eq!(read_body(&mut response).await, Vec::::new()); - let expected_index = example_files() .into_iter() .map(|(name, _)| name) .collect::>() .join("\n"); let expected_len = expected_index.len().to_string(); + assert_eq!(response.headers().len(), 3); + assert_eq!(response.headers()[CONTENT_LENGTH], expected_len); + assert_eq!(response.headers()[CONTENT_TYPE], "text/plain"); + assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); + // GET the index of the bundle. + let mut response = mgr + .get( + harness.zpool_id, + dataset_id, + support_bundle_id, + None, + SupportBundleQueryType::Index, + ) + .await + .expect("Should have been able to GET bundle index"); + assert_eq!(read_body(&mut response).await, expected_index.as_bytes()); assert_eq!(response.headers().len(), 3); assert_eq!(response.headers()[CONTENT_LENGTH], expected_len); assert_eq!(response.headers()[CONTENT_TYPE], "text/plain"); assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); + // HEAD a single file from within the bundle. + let mut response = mgr + .head( + harness.zpool_id, + dataset_id, + support_bundle_id, + None, + SupportBundleQueryType::Path { + file_path: GREET_PATH.to_string(), + }, + ) + .await + .expect("Should have been able to HEAD single file"); + assert_eq!(read_body(&mut response).await, Vec::::new()); + assert_eq!(response.headers().len(), 3); + assert_eq!( + response.headers()[CONTENT_LENGTH], + GREET_DATA.len().to_string() + ); + assert_eq!( + response.headers()[CONTENT_TYPE], + "application/octet-stream" + ); + assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); + + // GET a single file within the bundle + let mut response = mgr + .get( + harness.zpool_id, + dataset_id, + support_bundle_id, + None, + SupportBundleQueryType::Path { + file_path: GREET_PATH.to_string(), + }, + ) + .await + .expect("Should have been able to GET single file"); + assert_eq!(read_body(&mut response).await, GREET_DATA); + assert_eq!(response.headers().len(), 3); + assert_eq!( + response.headers()[CONTENT_LENGTH], + GREET_DATA.len().to_string() + ); + assert_eq!( + response.headers()[CONTENT_TYPE], + "application/octet-stream" + ); + assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); + + // DELETE the bundle on the dataset + mgr.delete(harness.zpool_id, dataset_id, support_bundle_id) + .await + .expect("Should have been able to DELETE bundle"); + + harness.cleanup().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn creation_without_dataset() { + let logctx = test_setup_log("creation_without_dataset"); + let log = &logctx.log; + + // Set up storage (zpool, but not dataset!) + let harness = SingleU2StorageHarness::new(log).await; + + // Access the Support Bundle API + let mgr = SupportBundleManager::new( + log, + harness.storage_test_harness.handle(), + ); + + // Get a support bundle that we're ready to store... + let support_bundle_id = SupportBundleUuid::new_v4(); + let zipfile_data = example_zipfile(); + let hash = ArtifactHash( + Sha256::digest(zipfile_data.as_slice()) + .as_slice() + .try_into() + .unwrap(), + ); + + // ... storing a bundle without a dataset should throw an error. + let dataset_id = DatasetUuid::new_v4(); + let err = mgr + .create( + harness.zpool_id, + dataset_id, + support_bundle_id, + hash, + stream::once(async { + Ok(Bytes::copy_from_slice(zipfile_data.as_slice())) + }), + ) + .await + .expect_err("Bundle creation should fail without dataset"); + assert!(matches!(err, Error::Storage(_)), "Unexpected error: {err:?}"); + assert_eq!(HttpError::from(err).status_code, StatusCode::NOT_FOUND); + + // Configure the dataset now, so it'll exist for future requests. + harness.configure_dataset(dataset_id, DatasetKind::Debug).await; + + mgr.create( + harness.zpool_id, + dataset_id, + support_bundle_id, + hash, + stream::once(async { + Ok(Bytes::copy_from_slice(zipfile_data.as_slice())) + }), + ) + .await + .expect("Should have created support bundle"); + + harness.cleanup().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn creation_bad_hash() { + let logctx = test_setup_log("creation_bad_hash"); + let log = &logctx.log; + + // Set up storage (zpool, but not dataset!) + let harness = SingleU2StorageHarness::new(log).await; + + // Access the Support Bundle API + let mgr = SupportBundleManager::new( + log, + harness.storage_test_harness.handle(), + ); + + // Get a support bundle that we're ready to store... + let support_bundle_id = SupportBundleUuid::new_v4(); + let zipfile_data = example_zipfile(); + let hash = ArtifactHash( + Sha256::digest(zipfile_data.as_slice()) + .as_slice() + .try_into() + .unwrap(), + ); + + // Configure the dataset now, so it'll exist for future requests. + let dataset_id = DatasetUuid::new_v4(); + harness.configure_dataset(dataset_id, DatasetKind::Debug).await; + + let bad_hash = ArtifactHash( + Sha256::digest(b"Hey, this ain't right") + .as_slice() + .try_into() + .unwrap(), + ); + + // Creating the bundle with a bad hash should fail. + let err = mgr + .create( + harness.zpool_id, + dataset_id, + support_bundle_id, + bad_hash, + stream::once(async { + Ok(Bytes::copy_from_slice(zipfile_data.as_slice())) + }), + ) + .await + .expect_err("Bundle creation should fail with bad hash"); + assert!( + matches!(err, Error::HashMismatch), + "Unexpected error: {err:?}" + ); + assert_eq!(HttpError::from(err).status_code, StatusCode::BAD_REQUEST); + + // As long as the dataset exists, we'll make storage for it, which means + // the bundle will be visible, but incomplete. + let bundles = mgr.list(harness.zpool_id, dataset_id).await.unwrap(); + assert_eq!(bundles.len(), 1); + assert_eq!(bundles[0].support_bundle_id, support_bundle_id); + assert_eq!(bundles[0].state, SupportBundleState::Incomplete); + + // Creating the bundle with bad data should fail + let err = mgr + .create( + harness.zpool_id, + dataset_id, + support_bundle_id, + hash, + stream::once(async { + Ok(Bytes::from_static(b"Not a zipfile")) + }), + ) + .await + .expect_err("Bundle creation should fail with bad hash"); + assert!( + matches!(err, Error::HashMismatch), + "Unexpected error: {err:?}" + ); + assert_eq!(HttpError::from(err).status_code, StatusCode::BAD_REQUEST); + + let bundles = mgr.list(harness.zpool_id, dataset_id).await.unwrap(); + assert_eq!(bundles.len(), 1); + assert_eq!(bundles[0].support_bundle_id, support_bundle_id); + assert_eq!(bundles[0].state, SupportBundleState::Incomplete); + + // Good hash + Good data -> creation should succeed + mgr.create( + harness.zpool_id, + dataset_id, + support_bundle_id, + hash, + stream::once(async { + Ok(Bytes::copy_from_slice(zipfile_data.as_slice())) + }), + ) + .await + .expect("Should have created support bundle"); + + // The bundle should now appear "Complete" + let bundles = mgr.list(harness.zpool_id, dataset_id).await.unwrap(); + assert_eq!(bundles.len(), 1); + assert_eq!(bundles[0].support_bundle_id, support_bundle_id); + assert_eq!(bundles[0].state, SupportBundleState::Complete); + + harness.cleanup().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn creation_idempotency() { + let logctx = test_setup_log("creation_idempotency"); + let log = &logctx.log; + + // Set up storage (zpool, but not dataset!) + let harness = SingleU2StorageHarness::new(log).await; + + // Access the Support Bundle API + let mgr = SupportBundleManager::new( + log, + harness.storage_test_harness.handle(), + ); + + // Get a support bundle that we're ready to store... + let support_bundle_id = SupportBundleUuid::new_v4(); + let zipfile_data = example_zipfile(); + let hash = ArtifactHash( + Sha256::digest(zipfile_data.as_slice()) + .as_slice() + .try_into() + .unwrap(), + ); + + // Configure the dataset now, so it'll exist for future requests. + let dataset_id = DatasetUuid::new_v4(); + harness.configure_dataset(dataset_id, DatasetKind::Debug).await; + + // Create the bundle + mgr.create( + harness.zpool_id, + dataset_id, + support_bundle_id, + hash, + stream::once(async { + Ok(Bytes::copy_from_slice(zipfile_data.as_slice())) + }), + ) + .await + .expect("Should have created support bundle"); + + // Creating the dataset again should work. + mgr.create( + harness.zpool_id, + dataset_id, + support_bundle_id, + hash, + stream::once(async { + Ok(Bytes::copy_from_slice(zipfile_data.as_slice())) + }), + ) + .await + .expect("Support bundle should already exist"); + + // This is an edge-case, but just to make sure the behavior + // is codified: If we are creating a bundle that already exists, + // we'll skip reading the body. + mgr.create( + harness.zpool_id, + dataset_id, + support_bundle_id, + hash, + stream::once(async { + // NOTE: This is different from the call above. + Ok(Bytes::from_static(b"Ignored")) + }), + ) + .await + .expect("Support bundle should already exist"); + + harness.cleanup().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn ranges() { + let logctx = test_setup_log("ranges"); + let log = &logctx.log; + + // Set up storage + let harness = SingleU2StorageHarness::new(log).await; + + // For this test, we'll add a dataset that can contain our bundles. + let dataset_id = DatasetUuid::new_v4(); + harness.configure_dataset(dataset_id, DatasetKind::Debug).await; + + // Access the Support Bundle API + let mgr = SupportBundleManager::new( + log, + harness.storage_test_harness.handle(), + ); + + // Create a fake support bundle -- really, just a zipfile. + let support_bundle_id = SupportBundleUuid::new_v4(); + let zipfile_data = example_zipfile(); + let hash = ArtifactHash( + Sha256::digest(zipfile_data.as_slice()) + .as_slice() + .try_into() + .unwrap(), + ); + + // Create a new bundle + let bundle = mgr + .create( + harness.zpool_id, + dataset_id, + support_bundle_id, + hash, + stream::once(async { + Ok(Bytes::copy_from_slice(zipfile_data.as_slice())) + }), + ) + .await + .expect("Should have created support bundle"); + assert_eq!(bundle.support_bundle_id, support_bundle_id); + assert_eq!(bundle.state, SupportBundleState::Complete); + + // GET the bundle we created, and observe the contents of the bundle + let ranges = [ + (0, 5), + (5, 100), + (0, 100), + (1000, 1000), + (1000, 1001), + (1000, zipfile_data.len() - 1), + ]; + + for (first, last) in ranges.into_iter() { + eprintln!("Trying whole-file range: {first}-{last}"); + let range = + PotentialRange::new(format!("bytes={first}-{last}").as_bytes()); + let expected_data = &zipfile_data[first..=last]; + + let mut response = mgr + .get( + harness.zpool_id, + dataset_id, + support_bundle_id, + Some(range), + SupportBundleQueryType::Whole, + ) + .await + .expect("Should have been able to GET bundle"); + assert_eq!(read_body(&mut response).await, expected_data); + assert_eq!(response.headers().len(), 4); + assert_eq!( + response.headers()[CONTENT_RANGE], + format!("bytes {first}-{last}/{}", zipfile_data.len()) + ); + assert_eq!( + response.headers()[CONTENT_LENGTH], + ((last + 1) - first).to_string() + ); + assert_eq!(response.headers()[CONTENT_TYPE], "application/zip"); + assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); + } + + // GET the index of the bundle. + let expected_index_str = example_files() + .into_iter() + .map(|(name, _)| name) + .collect::>() + .join("\n"); + let expected_index = expected_index_str.as_bytes(); + let ranges = [(0, 5), (5, 10), (10, expected_index.len() - 1)]; + + for (first, last) in ranges.into_iter() { + eprintln!("Trying index range: {first}-{last}"); + let range = + PotentialRange::new(format!("bytes={first}-{last}").as_bytes()); + let expected_data = &expected_index[first..=last]; + let mut response = mgr + .get( + harness.zpool_id, + dataset_id, + support_bundle_id, + Some(range), + SupportBundleQueryType::Index, + ) + .await + .expect("Should have been able to GET bundle index"); + assert_eq!(read_body(&mut response).await, expected_data); + assert_eq!(response.headers().len(), 4); + assert_eq!( + response.headers()[CONTENT_RANGE], + format!("bytes {first}-{last}/{}", expected_index.len()) + ); + assert_eq!( + response.headers()[CONTENT_LENGTH], + ((last + 1) - first).to_string(), + ); + assert_eq!(response.headers()[CONTENT_TYPE], "text/plain"); + assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); + } + + // GET a single file within the bundle + let ranges = [(0, 5), (5, 10), (5, GREET_DATA.len() - 1)]; + for (first, last) in ranges.into_iter() { + eprintln!("Trying single file range: {first}-{last}"); + let range = + PotentialRange::new(format!("bytes={first}-{last}").as_bytes()); + let expected_data = &GREET_DATA[first..=last]; + let mut response = mgr + .get( + harness.zpool_id, + dataset_id, + support_bundle_id, + Some(range), + SupportBundleQueryType::Path { + file_path: GREET_PATH.to_string(), + }, + ) + .await + .expect("Should have been able to GET single file"); + assert_eq!(read_body(&mut response).await, expected_data); + assert_eq!(response.headers().len(), 4); + assert_eq!( + response.headers()[CONTENT_RANGE], + format!("bytes {first}-{last}/{}", GREET_DATA.len()) + ); + assert_eq!( + response.headers()[CONTENT_LENGTH], + ((last + 1) - first).to_string(), + ); + assert_eq!( + response.headers()[CONTENT_TYPE], + "application/octet-stream" + ); + assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); + } + + // DELETE the bundle on the dataset + mgr.delete(harness.zpool_id, dataset_id, support_bundle_id) + .await + .expect("Should have been able to DELETE bundle"); + harness.cleanup().await; logctx.cleanup_successful(); } diff --git a/sled-storage/src/manager_test_harness.rs b/sled-storage/src/manager_test_harness.rs index 40c7c5fbed..068816ffa7 100644 --- a/sled-storage/src/manager_test_harness.rs +++ b/sled-storage/src/manager_test_harness.rs @@ -83,10 +83,9 @@ pub struct StorageManagerTestHarness { impl Drop for StorageManagerTestHarness { fn drop(&mut self) { if let Some(vdev_dir) = self.vdev_dir.take() { - eprintln!( + eprint!( "WARNING: StorageManagerTestHarness called without 'cleanup()'.\n\ - We may have leaked zpools, and not correctly deleted {}", - vdev_dir.path() + Attempting automated cleanup ... ", ); let pools = [ @@ -100,25 +99,49 @@ impl Drop for StorageManagerTestHarness { ), ]; - eprintln!( - "The following commands may need to be run to clean up state:" - ); - eprintln!("---"); + let mut failed_commands = vec![]; + for (prefix, pool) in pools { let Ok(entries) = pool.read_dir_utf8() else { continue; }; for entry in entries.flatten() { - eprintln!( - " pfexec zpool destroy {prefix}{} ", - entry.file_name() - ); + let pool_name = format!("{prefix}{}", entry.file_name()); + if let Err(_) = + std::process::Command::new(illumos_utils::PFEXEC) + .args(["zpool", "destroy", &pool_name]) + .status() + { + failed_commands + .push(format!("pfexec zpool destroy {pool_name}")); + } } } - eprintln!(" pfexec rm -rf {}", vdev_dir.path()); - eprintln!("---"); - panic!("Dropped without cleanup. See stderr for cleanup advice"); + let vdev_path = vdev_dir.path(); + if let Err(_) = std::process::Command::new(illumos_utils::PFEXEC) + .args(["rm", "-rf", vdev_path.as_str()]) + .status() + { + failed_commands.push(format!("pfexec rm -rf {vdev_path}")); + } + + if !failed_commands.is_empty() { + eprintln!("FAILED"); + eprintln!( + "The following commands may need to be run to clean up state:" + ); + eprintln!("---"); + for cmd in failed_commands { + eprintln!("{cmd}"); + } + eprintln!("---"); + panic!( + "Dropped without cleanup. See stderr for cleanup advice" + ); + } else { + eprintln!("OK"); + } } } } From 2df85cacffd977d8cb4d79eddf4c874d0ec18e92 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 19 Nov 2024 16:58:23 -0800 Subject: [PATCH 35/37] minimizing diff --- Cargo.lock | 1 - .../tests/integration_tests/volume_management.rs | 15 ++++++++++----- sled-agent/api/Cargo.toml | 1 - 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6708f1814d..f2f907dc6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10628,7 +10628,6 @@ name = "sled-agent-api" version = "0.1.0" dependencies = [ "camino", - "chrono", "dropshot 0.13.0", "http", "nexus-sled-agent-shared", diff --git a/nexus/tests/integration_tests/volume_management.rs b/nexus/tests/integration_tests/volume_management.rs index aa4a8d7a4e..6a9ce28389 100644 --- a/nexus/tests/integration_tests/volume_management.rs +++ b/nexus/tests/integration_tests/volume_management.rs @@ -12,6 +12,7 @@ use dropshot::test_util::ClientTestContext; use http::method::Method; use http::StatusCode; use nexus_config::RegionAllocationStrategy; +use nexus_db_model::to_db_typed_uuid; use nexus_db_model::RegionSnapshotReplacement; use nexus_db_model::RegionSnapshotReplacementState; use nexus_db_model::Volume; @@ -61,7 +62,6 @@ use omicron_uuid_kinds::TypedUuid; use omicron_uuid_kinds::UpstairsKind; use omicron_uuid_kinds::UpstairsRepairKind; use omicron_uuid_kinds::UpstairsSessionKind; -use omicron_uuid_kinds::ZpoolUuid; use rand::prelude::SliceRandom; use rand::{rngs::StdRng, SeedableRng}; use sled_agent_client::types::{CrucibleOpts, VolumeConstructionRequest}; @@ -2321,6 +2321,7 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { for i in 0..3 { let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; + let usage = datastore .volume_usage_records_for_resource( VolumeResourceUsage::RegionSnapshot { @@ -2343,6 +2344,7 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { for i in 0..3 { let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; + let usage = datastore .volume_usage_records_for_resource( VolumeResourceUsage::RegionSnapshot { @@ -2440,6 +2442,7 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { for i in 0..3 { let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; + let usage = datastore .volume_usage_records_for_resource( VolumeResourceUsage::RegionSnapshot { @@ -2456,6 +2459,7 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { for i in 3..6 { let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; + let usage = datastore .volume_usage_records_for_resource( VolumeResourceUsage::RegionSnapshot { @@ -2480,6 +2484,7 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { for i in 0..6 { let (dataset_id, region_id, snapshot_id, _) = region_snapshots[i]; + let usage = datastore .volume_usage_records_for_resource( VolumeResourceUsage::RegionSnapshot { @@ -4217,7 +4222,7 @@ async fn test_read_only_region_reference_counting( .sled_agent .sled_agent .get_crucible_dataset( - ZpoolUuid::from_untyped_uuid(db_read_only_dataset.pool_id), + TypedUuid::from_untyped_uuid(db_read_only_dataset.pool_id), db_read_only_dataset.id(), ) .await @@ -4289,7 +4294,7 @@ async fn test_read_only_region_reference_counting( .sled_agent .sled_agent .get_crucible_dataset( - ZpoolUuid::from_untyped_uuid(db_read_only_dataset.pool_id), + TypedUuid::from_untyped_uuid(db_read_only_dataset.pool_id), db_read_only_dataset.id(), ) .await @@ -5404,7 +5409,7 @@ async fn test_migrate_to_ref_count_with_records_region_snapshot_deleting( datastore .region_snapshot_create(nexus_db_model::RegionSnapshot { - dataset_id: (*dataset_id).into(), + dataset_id: to_db_typed_uuid(*dataset_id), region_id: *region_id, snapshot_id: *snapshot_id, snapshot_addr: snapshot_addr.clone(), @@ -5509,7 +5514,7 @@ async fn test_migrate_to_ref_count_with_records_region_snapshot_deleting( assert_eq!( region_snapshot_to_delete.dataset_id, - region_snapshots[0].0.into() + to_db_typed_uuid(region_snapshots[0].0) ); assert_eq!(region_snapshot_to_delete.region_id, region_snapshots[0].1); assert_eq!(region_snapshot_to_delete.snapshot_id, region_snapshots[0].2); diff --git a/sled-agent/api/Cargo.toml b/sled-agent/api/Cargo.toml index 8020651b15..95e9552f53 100644 --- a/sled-agent/api/Cargo.toml +++ b/sled-agent/api/Cargo.toml @@ -9,7 +9,6 @@ workspace = true [dependencies] camino.workspace = true -chrono.workspace = true dropshot.workspace = true http.workspace = true nexus-sled-agent-shared.workspace = true From e1e185bbc3640cb99a1c0b9049ad520981b6b283 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 21 Nov 2024 16:52:57 -0800 Subject: [PATCH 36/37] review feedback --- openapi/sled-agent.json | 8 +- sled-agent/api/src/lib.rs | 8 +- sled-agent/src/support_bundle/storage.rs | 200 +++++++++++++++++++++-- 3 files changed, 190 insertions(+), 26 deletions(-) diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 55c695a01a..8015691f4e 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -765,7 +765,7 @@ }, "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}": { "get": { - "summary": "Fetch a service bundle from a particular dataset", + "summary": "Fetch a support bundle from a particular dataset", "operationId": "support_bundle_get", "parameters": [ { @@ -818,7 +818,7 @@ } }, "post": { - "summary": "Create a service bundle within a particular dataset", + "summary": "Create a support bundle within a particular dataset", "operationId": "support_bundle_create", "parameters": [ { @@ -889,7 +889,7 @@ } }, "delete": { - "summary": "Delete a service bundle from a particular dataset", + "summary": "Delete a support bundle from a particular dataset", "operationId": "support_bundle_delete", "parameters": [ { @@ -933,7 +933,7 @@ } }, "head": { - "summary": "Fetch a service bundle from a particular dataset", + "summary": "Fetch a support bundle from a particular dataset", "operationId": "support_bundle_head", "parameters": [ { diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index 9083941ca4..f60eac1177 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -169,7 +169,7 @@ pub trait SledAgentApi { path_params: Path, ) -> Result>, HttpError>; - /// Create a service bundle within a particular dataset + /// Create a support bundle within a particular dataset #[endpoint { method = POST, path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}" @@ -181,7 +181,7 @@ pub trait SledAgentApi { body: StreamingBody, ) -> Result, HttpError>; - /// Fetch a service bundle from a particular dataset + /// Fetch a support bundle from a particular dataset #[endpoint { method = GET, path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}" @@ -192,7 +192,7 @@ pub trait SledAgentApi { body: TypedBody, ) -> Result, HttpError>; - /// Fetch a service bundle from a particular dataset + /// Fetch a support bundle from a particular dataset #[endpoint { method = HEAD, path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}" @@ -203,7 +203,7 @@ pub trait SledAgentApi { body: TypedBody, ) -> Result, HttpError>; - /// Delete a service bundle from a particular dataset + /// Delete a support bundle from a particular dataset #[endpoint { method = DELETE, path = "/support-bundles/{zpool_id}/{dataset_id}/{support_bundle_id}" diff --git a/sled-agent/src/support_bundle/storage.rs b/sled-agent/src/support_bundle/storage.rs index 94a5f80791..cc9e931f3e 100644 --- a/sled-agent/src/support_bundle/storage.rs +++ b/sled-agent/src/support_bundle/storage.rs @@ -18,6 +18,8 @@ use omicron_common::update::ArtifactHash; use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::SupportBundleUuid; use omicron_uuid_kinds::ZpoolUuid; +use rand::distributions::Alphanumeric; +use rand::{thread_rng, Rng}; use range_requests::PotentialRange; use range_requests::SingleRange; use sha2::{Digest, Sha256}; @@ -27,7 +29,7 @@ use sled_storage::manager::NestedDatasetListOptions; use sled_storage::manager::NestedDatasetLocation; use sled_storage::manager::StorageHandle; use slog::Logger; -use std::io::Read; +use slog_error_chain::InlineErrorChain; use std::io::Write; use tokio::io::AsyncReadExt; use tokio::io::AsyncSeekExt; @@ -49,6 +51,12 @@ pub enum Error { #[error("Dataset not found")] DatasetNotFound, + #[error("Dataset exists, but has an invalid configuration: (wanted {wanted}, saw {actual})")] + DatasetExistsBadConfig { wanted: DatasetUuid, actual: DatasetUuid }, + + #[error("Dataset exists, but appears on the wrong zpool (wanted {wanted}, saw {actual})")] + DatasetExistsOnWrongZpool { wanted: ZpoolUuid, actual: ZpoolUuid }, + #[error(transparent)] Storage(#[from] sled_storage::error::Error), @@ -62,6 +70,10 @@ pub enum Error { Zip(#[from] ZipError), } +fn err_str(err: &dyn std::error::Error) -> String { + InlineErrorChain::new(err).to_string() +} + impl From for HttpError { fn from(err: Error) -> Self { match err { @@ -76,15 +88,14 @@ impl From for HttpError { HttpError::for_bad_request(None, "Not a file".to_string()) } Error::Storage(err) => HttpError::from(ExternalError::from(err)), - Error::Io(err) => HttpError::for_internal_error(err.to_string()), - Error::Range(err) => HttpError::for_internal_error(err.to_string()), Error::Zip(err) => match err { ZipError::FileNotFound => HttpError::for_not_found( None, "Entry not found".to_string(), ), - err => HttpError::for_internal_error(err.to_string()), + err => HttpError::for_internal_error(err_str(&err)), }, + err => HttpError::for_internal_error(err_str(&err)), } } } @@ -120,15 +131,15 @@ fn stream_zip_entry_helper( ) -> Result<(), Error> { // TODO: When https://github.com/zip-rs/zip2/issues/231 is resolved, // this should call "by_name_seek" instead. - let reader = archive.by_name(&entry_path)?; + let mut reader = archive.by_name(&entry_path)?; - let mut reader: Box = match range { - Some(range) => Box::new(skip_and_limit( + let reader: &mut dyn std::io::Read = match range { + Some(range) => &mut skip_and_limit( reader, range.start() as usize, range.content_length().get() as usize, - )?), - None => Box::new(reader), + )?, + None => &mut reader, }; loop { @@ -240,8 +251,18 @@ impl<'a> SupportBundleManager<'a> { .get(&dataset_id) .ok_or_else(|| Error::DatasetNotFound)?; - if dataset.id != dataset_id || dataset.name.pool().id() != zpool_id { - return Err(Error::DatasetNotFound); + if dataset.id != dataset_id { + return Err(Error::DatasetExistsBadConfig { + wanted: dataset_id, + actual: dataset.id, + }); + } + let actual = dataset.name.pool().id(); + if actual != zpool_id { + return Err(Error::DatasetExistsOnWrongZpool { + wanted: zpool_id, + actual, + }); } Ok(dataset.clone()) } @@ -296,7 +317,8 @@ impl<'a> SupportBundleManager<'a> { Ok(bundles) } - /// Returns the hex, lowercase sha2 checksum of a file at `path`. + /// Validates that the sha2 checksum of the file at `path` matches the + /// expected value. async fn sha2_checksum_matches( path: &Utf8Path, expected: &ArtifactHash, @@ -369,7 +391,14 @@ impl<'a> SupportBundleManager<'a> { let support_bundle_dir = dataset .mountpoint(illumos_utils::zpool::ZPOOL_MOUNTPOINT_ROOT.into()); let support_bundle_path = support_bundle_dir.join("bundle"); - let support_bundle_path_tmp = support_bundle_dir.join("bundle.tmp"); + let support_bundle_path_tmp = support_bundle_dir.join(format!( + "bundle-{}.tmp", + thread_rng() + .sample_iter(Alphanumeric) + .take(6) + .map(char::from) + .collect::() + )); // Ensure that the dataset exists. info!(log, "Ensuring dataset exists for bundle"); @@ -745,9 +774,11 @@ mod tests { const GREET_PATH: &'static str = "greeting.txt"; const GREET_DATA: &'static [u8] = b"Hello around the world!"; + const ARBITRARY_DIRECTORY: &'static str = "look-a-directory/"; - fn example_files() -> [NamedFile; 5] { + fn example_files() -> [NamedFile; 6] { [ + (ARBITRARY_DIRECTORY, Data::Directory), (GREET_PATH, Data::File(GREET_DATA)), ("english/", Data::Directory), ("english/hello.txt", Data::File(b"Hello world!")), @@ -973,6 +1004,35 @@ mod tests { ); assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); + // Cannot GET nor HEAD a directory + let err = mgr + .get( + harness.zpool_id, + dataset_id, + support_bundle_id, + None, + SupportBundleQueryType::Path { + file_path: ARBITRARY_DIRECTORY.to_string(), + }, + ) + .await + .expect_err("Should not be able to GET directory"); + assert!(matches!(err, Error::NotAFile), "Unexpected error: {err:?}"); + + let err = mgr + .head( + harness.zpool_id, + dataset_id, + support_bundle_id, + None, + SupportBundleQueryType::Path { + file_path: ARBITRARY_DIRECTORY.to_string(), + }, + ) + .await + .expect_err("Should not be able to HEAD directory"); + assert!(matches!(err, Error::NotAFile), "Unexpected error: {err:?}"); + // DELETE the bundle on the dataset mgr.delete(harness.zpool_id, dataset_id, support_bundle_id) .await @@ -996,7 +1056,7 @@ mod tests { harness.storage_test_harness.handle(), ); - // Get a support bundle that we're ready to store... + // Get a support bundle that we're ready to store. let support_bundle_id = SupportBundleUuid::new_v4(); let zipfile_data = example_zipfile(); let hash = ArtifactHash( @@ -1006,7 +1066,7 @@ mod tests { .unwrap(), ); - // ... storing a bundle without a dataset should throw an error. + // Storing a bundle without a dataset should throw an error. let dataset_id = DatasetUuid::new_v4(); let err = mgr .create( @@ -1056,7 +1116,7 @@ mod tests { harness.storage_test_harness.handle(), ); - // Get a support bundle that we're ready to store... + // Get a support bundle that we're ready to store. let support_bundle_id = SupportBundleUuid::new_v4(); let zipfile_data = example_zipfile(); let hash = ArtifactHash( @@ -1146,6 +1206,79 @@ mod tests { assert_eq!(bundles[0].support_bundle_id, support_bundle_id); assert_eq!(bundles[0].state, SupportBundleState::Complete); + // We can delete the bundle, and it should no longer appear. + mgr.delete(harness.zpool_id, dataset_id, support_bundle_id) + .await + .expect("Should have been able to DELETE bundle"); + let bundles = mgr.list(harness.zpool_id, dataset_id).await.unwrap(); + assert_eq!(bundles.len(), 0); + + harness.cleanup().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn creation_bad_hash_still_deleteable() { + let logctx = test_setup_log("creation_bad_hash_still_deleteable"); + let log = &logctx.log; + + // Set up storage (zpool, but not dataset!) + let harness = SingleU2StorageHarness::new(log).await; + + // Access the Support Bundle API + let mgr = SupportBundleManager::new( + log, + harness.storage_test_harness.handle(), + ); + + // Get a support bundle that we're ready to store + let support_bundle_id = SupportBundleUuid::new_v4(); + let zipfile_data = example_zipfile(); + + // Configure the dataset now, so it'll exist for future requests. + let dataset_id = DatasetUuid::new_v4(); + harness.configure_dataset(dataset_id, DatasetKind::Debug).await; + + let bad_hash = ArtifactHash( + Sha256::digest(b"Hey, this ain't right") + .as_slice() + .try_into() + .unwrap(), + ); + + // Creating the bundle with a bad hash should fail. + let err = mgr + .create( + harness.zpool_id, + dataset_id, + support_bundle_id, + bad_hash, + stream::once(async { + Ok(Bytes::copy_from_slice(zipfile_data.as_slice())) + }), + ) + .await + .expect_err("Bundle creation should fail with bad hash"); + assert!( + matches!(err, Error::HashMismatch), + "Unexpected error: {err:?}" + ); + assert_eq!(HttpError::from(err).status_code, StatusCode::BAD_REQUEST); + + // The bundle still appears to exist, as storage gets allocated after + // the "create" call. + let bundles = mgr.list(harness.zpool_id, dataset_id).await.unwrap(); + assert_eq!(bundles.len(), 1); + assert_eq!(bundles[0].support_bundle_id, support_bundle_id); + assert_eq!(bundles[0].state, SupportBundleState::Incomplete); + + // We can delete the bundle, and it should no longer appear. + mgr.delete(harness.zpool_id, dataset_id, support_bundle_id) + .await + .expect("Should have been able to DELETE bundle"); + let bundles = mgr.list(harness.zpool_id, dataset_id).await.unwrap(); + assert_eq!(bundles.len(), 0); + harness.cleanup().await; logctx.cleanup_successful(); } @@ -1164,7 +1297,7 @@ mod tests { harness.storage_test_harness.handle(), ); - // Get a support bundle that we're ready to store... + // Get a support bundle that we're ready to store. let support_bundle_id = SupportBundleUuid::new_v4(); let zipfile_data = example_zipfile(); let hash = ArtifactHash( @@ -1382,6 +1515,37 @@ mod tests { assert_eq!(response.headers()[ACCEPT_RANGES], "bytes"); } + // Cannot GET nor HEAD a directory, even with range requests + let range = PotentialRange::new(format!("bytes=0-1").as_bytes()); + let err = mgr + .get( + harness.zpool_id, + dataset_id, + support_bundle_id, + Some(range), + SupportBundleQueryType::Path { + file_path: ARBITRARY_DIRECTORY.to_string(), + }, + ) + .await + .expect_err("Should not be able to GET directory"); + assert!(matches!(err, Error::NotAFile), "Unexpected error: {err:?}"); + + let range = PotentialRange::new(format!("bytes=0-1").as_bytes()); + let err = mgr + .head( + harness.zpool_id, + dataset_id, + support_bundle_id, + Some(range), + SupportBundleQueryType::Path { + file_path: ARBITRARY_DIRECTORY.to_string(), + }, + ) + .await + .expect_err("Should not be able to HEAD directory"); + assert!(matches!(err, Error::NotAFile), "Unexpected error: {err:?}"); + // DELETE the bundle on the dataset mgr.delete(harness.zpool_id, dataset_id, support_bundle_id) .await From 37f0900d4b7df82f30cc5e8868ec8ae0fb7b3ac9 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 21 Nov 2024 17:27:02 -0800 Subject: [PATCH 37/37] clippy --- sled-agent/src/support_bundle/storage.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sled-agent/src/support_bundle/storage.rs b/sled-agent/src/support_bundle/storage.rs index cc9e931f3e..e51f35e146 100644 --- a/sled-agent/src/support_bundle/storage.rs +++ b/sled-agent/src/support_bundle/storage.rs @@ -1516,7 +1516,7 @@ mod tests { } // Cannot GET nor HEAD a directory, even with range requests - let range = PotentialRange::new(format!("bytes=0-1").as_bytes()); + let range = PotentialRange::new(b"bytes=0-1"); let err = mgr .get( harness.zpool_id, @@ -1531,7 +1531,7 @@ mod tests { .expect_err("Should not be able to GET directory"); assert!(matches!(err, Error::NotAFile), "Unexpected error: {err:?}"); - let range = PotentialRange::new(format!("bytes=0-1").as_bytes()); + let range = PotentialRange::new(b"bytes=0-1"); let err = mgr .head( harness.zpool_id,