diff --git a/Cargo.lock b/Cargo.lock index b41abce1f4..a1c3585233 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -568,6 +568,21 @@ dependencies = [ "constant_time_eq 0.2.6", ] +[[package]] +name = "blake3" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq 0.3.0", + "memmap2", + "rayon", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -4025,6 +4040,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "memmap2" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -4817,6 +4841,7 @@ name = "omicron-deploy" version = "0.1.0" dependencies = [ "anyhow", + "camino", "clap 4.4.3", "crossbeam", "omicron-package", @@ -5071,6 +5096,7 @@ name = "omicron-package" version = "0.1.0" dependencies = [ "anyhow", + "camino", "clap 4.4.3", "expectorate", "futures", @@ -5088,6 +5114,7 @@ dependencies = [ "sled-hardware", "slog", "slog-async", + "slog-bunyan", "slog-term", "smf", "strum", @@ -5262,7 +5289,6 @@ dependencies = [ "bstr 1.6.0", "byteorder", "bytes", - "camino", "chrono", "cipher", "clap 4.4.3", @@ -5368,15 +5394,19 @@ dependencies = [ [[package]] name = "omicron-zone-package" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdfd257b7067e7a6aa9fba896a89b0f625bac7660213bb830db36e543bd3cdb8" +checksum = "e75ad9eb79bb6a1ec78d2eecf36c67fffcf56264d779c456c6e2cd4b257ee9fe" dependencies = [ "anyhow", "async-trait", + "blake3", + "camino", + "camino-tempfile", "chrono", "filetime", "flate2", + "futures", "futures-util", "hex", "reqwest", @@ -5384,8 +5414,9 @@ dependencies = [ "semver 1.0.21", "serde", "serde_derive", + "serde_json", + "slog", "tar", - "tempfile", "thiserror", "tokio", "toml 0.7.8", diff --git a/Cargo.toml b/Cargo.toml index b83c8a44bb..ef49321358 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -166,7 +166,7 @@ bootstrap-agent-client = { path = "clients/bootstrap-agent-client" } buf-list = { version = "1.0.3", features = ["tokio1"] } byteorder = "1.5.0" bytes = "1.5.0" -camino = "1.1" +camino = { version = "1.1", features = ["serde1"] } camino-tempfile = "1.1.1" cancel-safe-futures = "0.1.5" chacha20poly1305 = "0.10.1" @@ -269,7 +269,7 @@ omicron-package = { path = "package" } omicron-rpaths = { path = "rpaths" } omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } -omicron-zone-package = "0.10.1" +omicron-zone-package = "0.11.0" oxide-client = { path = "clients/oxide-client" } oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "1d29ef60a18179babfb44f0f7a3c2fe71034a2c1", features = [ "api", "std" ] } once_cell = "1.19.0" @@ -350,6 +350,7 @@ sled-hardware = { path = "sled-hardware" } sled-storage = { path = "sled-storage" } slog = { version = "2.7", features = [ "dynamic-keys", "max_level_trace", "release_max_level_debug" ] } slog-async = "2.8" +slog-bunyan = "2.5" slog-dtrace = "0.3" slog-envlogger = "2.2" slog-error-chain = { git = "https://github.com/oxidecomputer/slog-error-chain", branch = "main", features = ["derive"] } diff --git a/dev-tools/thing-flinger/Cargo.toml b/dev-tools/thing-flinger/Cargo.toml index 2acbaf5659..a427685871 100644 --- a/dev-tools/thing-flinger/Cargo.toml +++ b/dev-tools/thing-flinger/Cargo.toml @@ -7,6 +7,7 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true +camino.workspace = true clap.workspace = true crossbeam.workspace = true omicron-package.workspace = true diff --git a/dev-tools/thing-flinger/src/bin/thing-flinger.rs b/dev-tools/thing-flinger/src/bin/thing-flinger.rs index a13d5cfef7..43b137790d 100644 --- a/dev-tools/thing-flinger/src/bin/thing-flinger.rs +++ b/dev-tools/thing-flinger/src/bin/thing-flinger.rs @@ -6,8 +6,8 @@ use omicron_package::{parse, BuildCommand, DeployCommand}; +use camino::{Utf8Path, Utf8PathBuf}; use std::collections::{BTreeMap, BTreeSet}; -use std::path::{Path, PathBuf}; use std::process::Command; use anyhow::{Context, Result}; @@ -20,7 +20,7 @@ use thiserror::Error; #[derive(Deserialize, Debug)] struct Builder { server: String, - omicron_path: PathBuf, + omicron_path: Utf8PathBuf, } // A server on which an omicron package is deployed. @@ -33,19 +33,19 @@ struct Server { #[derive(Deserialize, Debug)] struct Deployment { rss_server: String, - staging_dir: PathBuf, + staging_dir: Utf8PathBuf, servers: BTreeSet, } #[derive(Debug, Deserialize)] struct Config { - omicron_path: PathBuf, + omicron_path: Utf8PathBuf, builder: Builder, servers: BTreeMap, deployment: Deployment, #[serde(default)] - rss_config_path: Option, + rss_config_path: Option, #[serde(default)] debug: bool, @@ -129,7 +129,7 @@ struct Args { help = "Path to deployment manifest toml file", action )] - config: PathBuf, + config: Utf8PathBuf, #[clap( short, @@ -140,7 +140,7 @@ struct Args { /// The output directory, where artifacts should be built and staged #[clap(long = "artifacts", default_value = "out/")] - artifact_dir: PathBuf, + artifact_dir: Utf8PathBuf, #[clap(subcommand)] subcommand: SubCommand, @@ -152,11 +152,6 @@ enum FlingError { #[error("Servers not listed in configuration: {0:?}")] InvalidServers(Vec), - /// The parameter should be the name of the argument that could not be - /// properly converted to a string. - #[error("{0} is not valid UTF-8")] - BadString(String), - /// Failed to rsync omicron to build host #[error("Failed to sync {src} with {dst}")] FailedSync { src: String, dst: String }, @@ -238,20 +233,14 @@ fn do_sync(config: &Config) -> Result<()> { // trailing slash. let src = format!( "{}/", - config - .omicron_path - .canonicalize() - .with_context(|| format!( - "could not canonicalize {}", - config.omicron_path.display() - ))? - .to_string_lossy() + config.omicron_path.canonicalize_utf8().with_context(|| format!( + "could not canonicalize {}", + config.omicron_path + ))? ); let dst = format!( "{}@{}:{}", - builder.username, - builder.addr, - config.builder.omicron_path.to_str().unwrap() + builder.username, builder.addr, config.builder.omicron_path ); println!("Synchronizing source files to: {}", dst); @@ -301,9 +290,7 @@ fn copy_to_deployment_staging_dir( || { let dst = format!( "{}@{}:{}", - server.username, - server.addr, - config.deployment.staging_dir.to_str().unwrap() + server.username, server.addr, config.deployment.staging_dir ); let mut cmd = partial_cmd(); cmd.arg(&dst); @@ -330,14 +317,10 @@ fn rsync_config_needed_for_tools(config: &Config) -> Result<()> { // the `./` here is load-bearing; it interacts with `--relative` to tell // rsync to create `smf/sled-agent` but none of its parents "{}/./smf/sled-agent/", - config - .omicron_path - .canonicalize() - .with_context(|| format!( - "could not canonicalize {}", - config.omicron_path.display() - ))? - .to_string_lossy() + config.omicron_path.canonicalize_utf8().with_context(|| format!( + "could not canonicalize {}", + config.omicron_path + ))? ); copy_to_deployment_staging_dir(config, src, "Copy smf/sled-agent dir") @@ -351,14 +334,10 @@ fn rsync_tools_dir_to_deployment_servers(config: &Config) -> Result<()> { // the `./` here is load-bearing; it interacts with `--relative` to tell // rsync to create `tools` but none of its parents "{}/./tools/", - config - .omicron_path - .canonicalize() - .with_context(|| format!( - "could not canonicalize {}", - config.omicron_path.display() - ))? - .to_string_lossy() + config.omicron_path.canonicalize_utf8().with_context(|| format!( + "could not canonicalize {}", + config.omicron_path + ))? ); copy_to_deployment_staging_dir(config, src, "Copy tools dir") } @@ -405,7 +384,7 @@ fn do_install_prereqs(config: &Config) -> Result<()> { let cmd = format!( "cd {} && mkdir -p out && pfexec ./tools/{}", - root_path.display(), + root_path.clone(), script ); println!( @@ -426,7 +405,7 @@ fn create_external_tls_cert_on_builder(config: &Config) -> Result<()> { let builder = &config.servers[&config.builder.server]; let cmd = format!( "cd {} && ./tools/create_self_signed_cert.sh", - config.builder.omicron_path.to_string_lossy() + config.builder.omicron_path, ); ssh_exec(&builder, &cmd, SshStrategy::NoForward) } @@ -434,7 +413,7 @@ fn create_external_tls_cert_on_builder(config: &Config) -> Result<()> { fn create_virtual_hardware_on_deployment_servers(config: &Config) { let cmd = format!( "cd {} && pfexec ./tools/create_virtual_hardware.sh", - config.deployment.staging_dir.display() + config.deployment.staging_dir ); let fns = config.deployment_servers().map(|server| { || { @@ -464,7 +443,7 @@ fn do_build_minimal(config: &Config) -> Result<()> { let server = &config.servers[&config.builder.server]; let cmd = format!( "cd {} && cargo build {} -p {} -p {}", - config.builder.omicron_path.to_string_lossy(), + config.builder.omicron_path, config.release_arg(), "omicron-package", "omicron-deploy" @@ -472,11 +451,8 @@ fn do_build_minimal(config: &Config) -> Result<()> { ssh_exec(&server, &cmd, SshStrategy::NoForward) } -fn do_package(config: &Config, artifact_dir: PathBuf) -> Result<()> { +fn do_package(config: &Config, artifact_dir: Utf8PathBuf) -> Result<()> { let builder = &config.servers[&config.builder.server]; - let artifact_dir = artifact_dir - .to_str() - .ok_or_else(|| FlingError::BadString("artifact_dir".to_string()))?; // We use a bash login shell to get a proper environment, so we have a path to // postgres, and $DEP_PQ_LIBDIRS is filled in. This is required for building @@ -487,9 +463,9 @@ fn do_package(config: &Config, artifact_dir: PathBuf) -> Result<()> { "bash -lc \ 'cd {} && \ cargo run {} --bin omicron-package -- package --out {}'", - config.builder.omicron_path.to_string_lossy(), + config.builder.omicron_path, config.release_arg(), - &artifact_dir, + artifact_dir, ); ssh_exec(&builder, &cmd, SshStrategy::NoForward) @@ -506,7 +482,7 @@ fn do_check(config: &Config) -> Result<()> { "bash -lc \ 'cd {} && \ cargo run {} --bin omicron-package -- check'", - config.builder.omicron_path.to_string_lossy(), + config.builder.omicron_path, config.release_arg(), ); @@ -521,7 +497,7 @@ fn do_uninstall(config: &Config) -> Result<()> { // Run `omicron-package uninstall` on the deployment server let cmd = format!( "cd {} && pfexec ./omicron-package uninstall", - config.deployment.staging_dir.to_string_lossy(), + config.deployment.staging_dir, ); println!("$ {}", cmd); ssh_exec(&server, &cmd, SshStrategy::Forward)?; @@ -531,10 +507,10 @@ fn do_uninstall(config: &Config) -> Result<()> { fn do_clean( config: &Config, - artifact_dir: PathBuf, - install_dir: PathBuf, + artifact_dir: Utf8PathBuf, + install_dir: Utf8PathBuf, ) -> Result<()> { - let mut deployment_src = PathBuf::from(&config.deployment.staging_dir); + let mut deployment_src = Utf8PathBuf::from(&config.deployment.staging_dir); deployment_src.push(&artifact_dir); let builder = &config.servers[&config.builder.server]; for server in config.deployment_servers() { @@ -543,9 +519,7 @@ fn do_clean( // Run `omicron-package uninstall` on the deployment server let cmd = format!( "cd {} && pfexec ./omicron-package clean --in {} --out {}", - config.deployment.staging_dir.to_string_lossy(), - deployment_src.to_string_lossy(), - install_dir.to_string_lossy() + config.deployment.staging_dir, deployment_src, install_dir, ); println!("$ {}", cmd); ssh_exec(&server, &cmd, SshStrategy::Forward)?; @@ -586,12 +560,14 @@ where .unwrap(); } -fn do_install(config: &Config, artifact_dir: &Path, install_dir: &Path) { +fn do_install( + config: &Config, + artifact_dir: &Utf8Path, + install_dir: &Utf8Path, +) { let builder = &config.servers[&config.builder.server]; - let mut pkg_dir = PathBuf::from(&config.builder.omicron_path); + let mut pkg_dir = Utf8PathBuf::from(&config.builder.omicron_path); pkg_dir.push(artifact_dir); - let pkg_dir = pkg_dir.to_string_lossy(); - let pkg_dir = &pkg_dir; let fns = config.deployment.servers.iter().map(|server_name| { (server_name, || { @@ -599,7 +575,7 @@ fn do_install(config: &Config, artifact_dir: &Path, install_dir: &Path) { config, &artifact_dir, &install_dir, - &pkg_dir, + pkg_dir.as_str(), builder, server_name, ) @@ -611,7 +587,7 @@ fn do_install(config: &Config, artifact_dir: &Path, install_dir: &Path) { fn do_overlay(config: &Config) -> Result<()> { let builder = &config.servers[&config.builder.server]; - let mut root_path = PathBuf::from(&config.builder.omicron_path); + let mut root_path = Utf8PathBuf::from(&config.builder.omicron_path); // TODO: This needs to match the artifact_dir in `package` root_path.push("out/overlay"); @@ -649,7 +625,7 @@ fn do_overlay(config: &Config) -> Result<()> { fn overlay_rss_config( builder: &Server, config: &Config, - rss_server_dir: &Path, + rss_server_dir: &Utf8Path, ) -> Result<()> { // Sync `config-rss.toml` to the directory for the RSS server on the // builder. @@ -660,9 +636,7 @@ fn overlay_rss_config( }; let dst = format!( "{}@{}:{}/config-rss.toml", - builder.username, - builder.addr, - rss_server_dir.display() + builder.username, builder.addr, rss_server_dir ); let mut cmd = rsync_common(); @@ -671,11 +645,7 @@ fn overlay_rss_config( let status = cmd.status().context(format!("Failed to run command: ({:?})", cmd))?; if !status.success() { - return Err(FlingError::FailedSync { - src: src.to_string_lossy().to_string(), - dst, - } - .into()); + return Err(FlingError::FailedSync { src: src.to_string(), dst }.into()); } Ok(()) @@ -683,8 +653,8 @@ fn overlay_rss_config( fn single_server_install( config: &Config, - artifact_dir: &Path, - install_dir: &Path, + artifact_dir: &Utf8Path, + install_dir: &Utf8Path, pkg_dir: &str, builder: &Server, server_name: &str, @@ -757,7 +727,7 @@ fn copy_package_artifacts_to_staging( pkg_dir, destination.username, destination.addr, - config.deployment.staging_dir.to_string_lossy() + config.deployment.staging_dir ); println!("$ {}", cmd); ssh_exec(builder, &cmd, SshStrategy::Forward) @@ -768,17 +738,17 @@ fn copy_omicron_package_binary_to_staging( builder: &Server, destination: &Server, ) -> Result<()> { - let mut bin_path = PathBuf::from(&config.builder.omicron_path); + let mut bin_path = Utf8PathBuf::from(&config.builder.omicron_path); bin_path.push(format!( "target/{}/omicron-package", if config.debug { "debug" } else { "release" } )); let cmd = format!( "rsync -avz {} {}@{}:{}", - bin_path.to_string_lossy(), + bin_path, destination.username, destination.addr, - config.deployment.staging_dir.to_string_lossy() + config.deployment.staging_dir ); println!("$ {}", cmd); ssh_exec(builder, &cmd, SshStrategy::Forward) @@ -789,14 +759,14 @@ fn copy_package_manifest_to_staging( builder: &Server, destination: &Server, ) -> Result<()> { - let mut path = PathBuf::from(&config.builder.omicron_path); + let mut path = Utf8PathBuf::from(&config.builder.omicron_path); path.push("package-manifest.toml"); let cmd = format!( "rsync {} {}@{}:{}", - path.to_string_lossy(), + path, destination.username, destination.addr, - config.deployment.staging_dir.to_string_lossy() + config.deployment.staging_dir ); println!("$ {}", cmd); ssh_exec(builder, &cmd, SshStrategy::Forward) @@ -805,13 +775,12 @@ fn copy_package_manifest_to_staging( fn run_omicron_package_activate_from_staging( config: &Config, destination: &Server, - install_dir: &Path, + install_dir: &Utf8Path, ) -> Result<()> { // Run `omicron-package activate` on the deployment server let cmd = format!( "cd {} && pfexec ./omicron-package activate --out {}", - config.deployment.staging_dir.to_string_lossy(), - install_dir.to_string_lossy(), + config.deployment.staging_dir, install_dir, ); println!("$ {}", cmd); @@ -821,18 +790,16 @@ fn run_omicron_package_activate_from_staging( fn run_omicron_package_unpack_from_staging( config: &Config, destination: &Server, - artifact_dir: &Path, - install_dir: &Path, + artifact_dir: &Utf8Path, + install_dir: &Utf8Path, ) -> Result<()> { - let mut deployment_src = PathBuf::from(&config.deployment.staging_dir); + let mut deployment_src = Utf8PathBuf::from(&config.deployment.staging_dir); deployment_src.push(&artifact_dir); // Run `omicron-package unpack` on the deployment server let cmd = format!( "cd {} && pfexec ./omicron-package unpack --in {} --out {}", - config.deployment.staging_dir.to_string_lossy(), - deployment_src.to_string_lossy(), - install_dir.to_string_lossy(), + config.deployment.staging_dir, deployment_src, install_dir, ); println!("$ {}", cmd); @@ -852,7 +819,7 @@ fn copy_overlay_files_to_staging( destination_name, destination.username, destination.addr, - config.deployment.staging_dir.to_string_lossy() + config.deployment.staging_dir ); println!("$ {}", cmd); ssh_exec(builder, &cmd, SshStrategy::Forward) @@ -861,12 +828,11 @@ fn copy_overlay_files_to_staging( fn install_overlay_files_from_staging( config: &Config, destination: &Server, - install_dir: &Path, + install_dir: &Utf8Path, ) -> Result<()> { let cmd = format!( "pfexec cp -r {}/overlay/* {}", - config.deployment.staging_dir.to_string_lossy(), - install_dir.to_string_lossy() + config.deployment.staging_dir, install_dir ); println!("$ {}", cmd); ssh_exec(&destination, &cmd, SshStrategy::NoForward) @@ -925,7 +891,7 @@ fn validate_servers( } fn validate_absolute_path( - path: &Path, + path: &Utf8Path, field: &'static str, ) -> Result<(), FlingError> { if path.is_absolute() || path.starts_with("$HOME") { @@ -970,7 +936,7 @@ fn main() -> Result<()> { SubCommand::Builder(BuildCommand::Target { .. }) => { todo!("Setting target not supported through thing-flinger") } - SubCommand::Builder(BuildCommand::Package) => { + SubCommand::Builder(BuildCommand::Package { .. }) => { do_package(&config, args.artifact_dir)?; } SubCommand::Builder(BuildCommand::Stamp { .. }) => { diff --git a/package/Cargo.toml b/package/Cargo.toml index 0dc86ceb8c..8067473aa0 100644 --- a/package/Cargo.toml +++ b/package/Cargo.toml @@ -7,6 +7,7 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true +camino.workspace = true clap.workspace = true futures.workspace = true hex.workspace = true @@ -22,6 +23,7 @@ serde.workspace = true sled-hardware.workspace = true slog.workspace = true slog-async.workspace = true +slog-bunyan.workspace = true slog-term.workspace = true smf.workspace = true strum.workspace = true diff --git a/package/src/bin/omicron-package.rs b/package/src/bin/omicron-package.rs index 59c5c6ffe6..3b8bd24918 100644 --- a/package/src/bin/omicron-package.rs +++ b/package/src/bin/omicron-package.rs @@ -5,6 +5,7 @@ //! Utility for bundling target binaries as tarfiles. use anyhow::{anyhow, bail, Context, Result}; +use camino::{Utf8Path, Utf8PathBuf}; use clap::{Parser, Subcommand}; use futures::stream::{self, StreamExt, TryStreamExt}; use illumos_utils::{zfs, zone}; @@ -26,7 +27,6 @@ use slog::{info, warn}; use std::env; use std::fs::create_dir_all; use std::io::Write; -use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; use swrite::{swrite, SWrite}; @@ -60,7 +60,7 @@ struct Args { help = "Path to package manifest toml file", action )] - manifest: PathBuf, + manifest: Utf8PathBuf, #[clap( short, @@ -72,7 +72,7 @@ struct Args { /// The output directory, where artifacts should be built and staged #[clap(long = "artifacts", default_value = "out/")] - artifact_dir: PathBuf, + artifact_dir: Utf8PathBuf, #[clap( short, @@ -203,13 +203,13 @@ async fn do_dot(config: &Config) -> Result<()> { const ACTIVE: &str = "active"; async fn do_target( - artifact_dir: &Path, + artifact_dir: &Utf8Path, name: &str, subcommand: &TargetCommand, ) -> Result<()> { let target_dir = artifact_dir.join("target"); tokio::fs::create_dir_all(&target_dir).await.with_context(|| { - format!("failed to create directory {}", target_dir.display()) + format!("failed to create directory {}", target_dir) })?; match subcommand { TargetCommand::Create { image, machine, switch, rack_topology } => { @@ -224,7 +224,7 @@ async fn do_target( tokio::fs::write(&path, Target::from(target).to_string()) .await .with_context(|| { - format!("failed to write target to {}", path.display()) + format!("failed to write target to {}", path) })?; replace_active_link(&name, &target_dir).await?; @@ -233,7 +233,7 @@ async fn do_target( } TargetCommand::List => { let active = tokio::fs::read_link(target_dir.join(ACTIVE)).await?; - let active = active.to_string_lossy(); + let active = Utf8PathBuf::try_from(active)?; for entry in walkdir::WalkDir::new(&target_dir) .max_depth(1) .sort_by_file_name() @@ -268,9 +268,9 @@ async fn do_target( } async fn get_single_target( - target_dir: impl AsRef, + target_dir: impl AsRef, name: &str, -) -> Result { +) -> Result { if name == ACTIVE { bail!( "The name '{name}' is reserved, please try another (e.g. 'default')\n\ @@ -282,29 +282,25 @@ async fn get_single_target( } async fn replace_active_link( - src: impl AsRef, - target_dir: impl AsRef, + src: impl AsRef, + target_dir: impl AsRef, ) -> Result<()> { let src = src.as_ref(); let target_dir = target_dir.as_ref(); let dst = target_dir.join(ACTIVE); if !target_dir.join(src).exists() { - bail!("Target file {} does not exist", src.display()); + bail!("Target file {} does not exist", src); } let _ = tokio::fs::remove_file(&dst).await; tokio::fs::symlink(src, &dst).await.with_context(|| { - format!( - "failed creating symlink to {} at {}", - src.display(), - dst.display() - ) + format!("failed creating symlink to {} at {}", src, dst) })?; Ok(()) } // Calculates the SHA256 digest for a file. -async fn get_sha256_digest(path: &PathBuf) -> Result { +async fn get_sha256_digest(path: &Utf8PathBuf) -> Result { let mut reader = BufReader::new( tokio::fs::File::open(&path) .await @@ -333,21 +329,21 @@ async fn download_prebuilt( repo: &str, commit: &str, expected_digest: &Vec, - path: &Path, + path: &Utf8Path, ) -> Result<()> { progress.set_message("downloading prebuilt".into()); let url = format!( "https://buildomat.eng.oxide.computer/public/file/oxidecomputer/{}/image/{}/{}", repo, commit, - path.file_name().unwrap().to_string_lossy(), + path.file_name().unwrap(), ); let response = reqwest::Client::new() .get(&url) .send() .await .with_context(|| format!("failed to get {url}"))?; - progress.set_length( + progress.increment_total( response .content_length() .ok_or_else(|| anyhow!("Missing Content Length"))?, @@ -367,7 +363,7 @@ async fn download_prebuilt( .await .with_context(|| format!("failed writing {path:?}"))?; // Record progress in the UI - progress.increment(chunk.len().try_into().unwrap()); + progress.increment_completed(chunk.len().try_into().unwrap()); } let digest = context.finish(); @@ -382,16 +378,16 @@ async fn download_prebuilt( } // Ensures a package exists, either by creating it or downloading it. -async fn get_package( +async fn ensure_package( config: &Config, - target: &Target, ui: &Arc, package_name: &String, package: &Package, - output_directory: &Path, + output_directory: &Utf8Path, + disable_cache: bool, ) -> Result<()> { - let total_work = package.get_total_work_for_target(&target)?; - let progress = ui.add_package(package_name.to_string(), total_work); + let target = &config.target; + let progress = ui.add_package(package_name.to_string()); match &package.source { PackageSource::Prebuilt { repo, commit, sha256 } => { let expected_digest = hex::decode(&sha256)?; @@ -435,19 +431,26 @@ async fn get_package( } } PackageSource::Manual => { + progress.set_message("confirming manual package".into()); let path = package.get_output_path(package_name, &output_directory); if !path.exists() { bail!( "The package for {} (expected at {}) does not exist.", package_name, - path.to_string_lossy(), + path, ); } } PackageSource::Local { .. } | PackageSource::Composite { .. } => { - progress.set_message("bundle package".into()); + progress.set_message("building package".into()); + + let build_config = omicron_zone_package::package::BuildConfig { + target, + progress: &progress, + cache_disabled: disable_cache, + }; package - .create_with_progress_for_target(&progress, &target, package_name, &output_directory) + .create(package_name, &output_directory, &build_config) .await .with_context(|| { let msg = format!("failed to create {package_name} in {output_directory:?}"); @@ -463,11 +466,15 @@ async fn get_package( Ok(()) } -async fn do_package(config: &Config, output_directory: &Path) -> Result<()> { +async fn do_package( + config: &Config, + output_directory: &Utf8Path, + disable_cache: bool, +) -> Result<()> { create_dir_all(&output_directory) .map_err(|err| anyhow!("Cannot create output directory: {}", err))?; - let ui = ProgressUI::new(); + let ui = ProgressUI::new(&config.log); do_build(&config).await?; @@ -482,13 +489,13 @@ async fn do_package(config: &Config, output_directory: &Path) -> Result<()> { .try_for_each_concurrent( None, |((package_name, package), ui)| async move { - get_package( + ensure_package( &config, - &config.target, &ui, package_name, package, output_directory, + disable_cache, ) .await }, @@ -502,7 +509,7 @@ async fn do_package(config: &Config, output_directory: &Path) -> Result<()> { async fn do_stamp( config: &Config, - output_directory: &Path, + output_directory: &Utf8Path, package_name: &str, version: &semver::Version, ) -> Result<()> { @@ -518,14 +525,14 @@ async fn do_stamp( // Stamp it let stamped_path = package.stamp(package_name, output_directory, version).await?; - println!("Created: {}", stamped_path.display()); + println!("Created: {}", stamped_path); Ok(()) } async fn do_unpack( config: &Config, - artifact_dir: &Path, - install_dir: &Path, + artifact_dir: &Utf8Path, + install_dir: &Utf8Path, ) -> Result<()> { create_dir_all(&install_dir).map_err(|err| { anyhow!("Cannot create installation directory: {}", err) @@ -543,14 +550,14 @@ async fn do_unpack( info!( &config.log, "Installing service"; - "src" => %src.to_string_lossy(), - "dst" => %dst.to_string_lossy(), + "src" => %src, + "dst" => %dst, ); std::fs::copy(&src, &dst).map_err(|err| { anyhow!( "Failed to copy {src} to {dst}: {err}", - src = src.display(), - dst = dst.display() + src = src, + dst = dst ) })?; Ok(()) @@ -575,8 +582,8 @@ async fn do_unpack( info!( &config.log, "Unpacking service tarball"; - "tar_path" => %tar_path.to_string_lossy(), - "service_path" => %service_path.to_string_lossy(), + "tar_path" => %tar_path, + "service_path" => %service_path, ); let tar_file = std::fs::File::open(&tar_path)?; @@ -589,7 +596,7 @@ async fn do_unpack( Ok(()) } -fn do_activate(config: &Config, install_dir: &Path) -> Result<()> { +fn do_activate(config: &Config, install_dir: &Utf8Path) -> Result<()> { // Install the bootstrap service, which itself extracts and // installs other services. if let Some(package) = @@ -601,8 +608,7 @@ fn do_activate(config: &Config, install_dir: &Path) -> Result<()> { .join("manifest.xml"); info!( config.log, - "Installing bootstrap service from {}", - manifest_path.to_string_lossy() + "Installing bootstrap service from {}", manifest_path ); smf::Config::import().run(&manifest_path)?; @@ -613,8 +619,8 @@ fn do_activate(config: &Config, install_dir: &Path) -> Result<()> { async fn do_install( config: &Config, - artifact_dir: &Path, - install_dir: &Path, + artifact_dir: &Utf8Path, + install_dir: &Utf8Path, ) -> Result<()> { do_unpack(config, artifact_dir, install_dir).await?; do_activate(config, install_dir) @@ -674,7 +680,9 @@ fn uninstall_all_packages(config: &Config) { } } -fn remove_file_unless_already_removed>(path: P) -> Result<()> { +fn remove_file_unless_already_removed>( + path: P, +) -> Result<()> { if let Err(e) = std::fs::remove_file(path.as_ref()) { match e.kind() { std::io::ErrorKind::NotFound => {} @@ -684,7 +692,9 @@ fn remove_file_unless_already_removed>(path: P) -> Result<()> { Ok(()) } -fn remove_all_unless_already_removed>(path: P) -> Result<()> { +fn remove_all_unless_already_removed>( + path: P, +) -> Result<()> { if let Err(e) = std::fs::remove_dir_all(path.as_ref()) { match e.kind() { std::io::ErrorKind::NotFound => {} @@ -694,22 +704,22 @@ fn remove_all_unless_already_removed>(path: P) -> Result<()> { Ok(()) } -fn remove_all_except>( +fn remove_all_except>( path: P, to_keep: &[&str], log: &Logger, ) -> Result<()> { - let dir = match path.as_ref().read_dir() { + let dir = match path.as_ref().read_dir_utf8() { Ok(dir) => dir, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), Err(e) => bail!(e), }; for entry in dir { let entry = entry?; - if to_keep.contains(&&*(entry.file_name().to_string_lossy())) { - info!(log, "Keeping: '{}'", entry.path().to_string_lossy()); + if to_keep.contains(&entry.file_name()) { + info!(log, "Keeping: '{}'", entry.path()); } else { - info!(log, "Removing: '{}'", entry.path().to_string_lossy()); + info!(log, "Removing: '{}'", entry.path()); if entry.metadata()?.is_dir() { remove_all_unless_already_removed(entry.path())?; } else { @@ -739,15 +749,11 @@ async fn do_uninstall(config: &Config) -> Result<()> { async fn do_clean( config: &Config, - artifact_dir: &Path, - install_dir: &Path, + artifact_dir: &Utf8Path, + install_dir: &Utf8Path, ) -> Result<()> { do_uninstall(&config).await?; - info!( - config.log, - "Removing artifacts from {}", - artifact_dir.to_string_lossy() - ); + info!(config.log, "Removing artifacts from {}", artifact_dir); const ARTIFACTS_TO_KEEP: &[&str] = &[ "clickhouse", "cockroachdb", @@ -757,11 +763,7 @@ async fn do_clean( "softnpu", ]; remove_all_except(artifact_dir, ARTIFACTS_TO_KEEP, &config.log)?; - info!( - config.log, - "Removing installed objects in: {}", - install_dir.to_string_lossy() - ); + info!(config.log, "Removing installed objects in: {}", install_dir); const INSTALLED_OBJECTS_TO_KEEP: &[&str] = &["opte"]; remove_all_except(install_dir, INSTALLED_OBJECTS_TO_KEEP, &config.log)?; @@ -770,52 +772,80 @@ async fn do_clean( fn in_progress_style() -> ProgressStyle { ProgressStyle::default_bar() - .template( - "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", - ) + .template("[{elapsed:>3}] {bar:30.cyan/blue} {pos:>7}/{len:7} {msg}") .expect("Invalid template") .progress_chars("#>.") } fn completed_progress_style() -> ProgressStyle { ProgressStyle::default_bar() - .template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg:.green}") + .template( + "[{elapsed:>3}] {bar:30.cyan/blue} {pos:>7}/{len:7} {msg:.green}", + ) .expect("Invalid template") .progress_chars("#>.") } fn error_progress_style() -> ProgressStyle { ProgressStyle::default_bar() - .template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg:.red}") + .template( + "[{elapsed:>3}] {bar:30.cyan/blue} {pos:>7}/{len:7} {msg:.red}", + ) .expect("Invalid template") .progress_chars("#>.") } // Struct managing display of progress to UI. struct ProgressUI { + log: Logger, multi: MultiProgress, style: ProgressStyle, } +impl ProgressUI { + fn new(log: &Logger) -> Arc { + Arc::new(Self { + log: log.clone(), + multi: MultiProgress::new(), + style: in_progress_style(), + }) + } + + fn add_package(&self, service_name: String) -> PackageProgress { + let pb = self.multi.add(ProgressBar::new(1)); + pb.set_style(self.style.clone()); + pb.set_message(service_name.clone()); + pb.tick(); + PackageProgress::new(&self.log, pb, service_name) + } +} + struct PackageProgress { + log: Logger, pb: ProgressBar, service_name: String, } impl PackageProgress { + fn new(log: &Logger, pb: ProgressBar, service_name: String) -> Self { + Self { + log: log.new(o!("package" => service_name.clone())), + pb, + service_name, + } + } + fn finish(&self) { self.pb.set_style(completed_progress_style()); self.pb.finish_with_message(format!("{}: done", self.service_name)); self.pb.tick(); } - fn set_length(&self, total: u64) { - self.pb.set_length(total); - } - fn set_error_message(&self, message: std::borrow::Cow<'static, str>) { self.pb.set_style(error_progress_style()); - self.pb.set_message(format!("{}: {}", self.service_name, message)); + let message = format!("{}: {}", self.service_name, message); + warn!(self.log, "{}", &message); + self.pb.set_message(message); self.pb.tick(); } @@ -827,29 +857,22 @@ impl PackageProgress { impl Progress for PackageProgress { fn set_message(&self, message: std::borrow::Cow<'static, str>) { self.pb.set_style(in_progress_style()); - self.pb.set_message(format!("{}: {}", self.service_name, message)); + let message = format!("{}: {}", self.service_name, message); + info!(self.log, "{}", &message); + self.pb.set_message(message); self.pb.tick(); } - fn increment(&self, delta: u64) { - self.pb.inc(delta); + fn get_log(&self) -> &Logger { + &self.log } -} -impl ProgressUI { - fn new() -> Arc { - Arc::new(Self { - multi: MultiProgress::new(), - style: in_progress_style(), - }) + fn increment_total(&self, delta: u64) { + self.pb.inc_length(delta); } - fn add_package(&self, service_name: String, total: u64) -> PackageProgress { - let pb = self.multi.add(ProgressBar::new(total)); - pb.set_style(self.style.clone()); - pb.set_message(service_name.clone()); - pb.tick(); - PackageProgress { pb, service_name } + fn increment_completed(&self, delta: u64) { + self.pb.inc(delta); } } @@ -891,10 +914,16 @@ async fn main() -> Result<()> { let args = Args::try_parse()?; let package_config = parse::<_, PackageConfig>(&args.manifest)?; - let decorator = slog_term::TermDecorator::new().build(); - let drain = slog_term::CompactFormat::new(decorator).build().fuse(); + let mut open_options = std::fs::OpenOptions::new(); + open_options.write(true).create(true).truncate(true); + tokio::fs::create_dir_all(&args.artifact_dir).await?; + let logpath = args.artifact_dir.join("LOG"); + let logfile = std::io::LineWriter::new(open_options.open(&logpath)?); + println!("Logging to: {}", std::fs::canonicalize(logpath)?.display()); + + let drain = slog_bunyan::new(logfile).build().fuse(); let drain = slog_async::Async::new(drain).build().fuse(); - let log = slog::Logger::root(drain, o!()); + let log = Logger::root(drain, o!()); let target_help_str = || -> String { format!( @@ -909,7 +938,7 @@ async fn main() -> Result<()> { std::fs::read_to_string(&target_path).map_err(|e| { eprintln!( "Failed to read build target: {}\n{}", - target_path.display(), + target_path, target_help_str() ); e @@ -918,7 +947,7 @@ async fn main() -> Result<()> { .map_err(|e| { eprintln!( "Failed to parse {} as target\n{}", - target_path.display(), + target_path, target_help_str() ); e @@ -938,10 +967,10 @@ async fn main() -> Result<()> { // Use a CWD that is the root of the Omicron repository. if let Ok(manifest) = env::var("CARGO_MANIFEST_DIR") { - let manifest_dir = PathBuf::from(manifest); + let manifest_dir = Utf8PathBuf::from(manifest); let root = manifest_dir.parent().unwrap(); env::set_current_dir(root).with_context(|| { - format!("failed to set current directory to {}", root.display()) + format!("failed to set current directory to {}", root) })?; } @@ -952,8 +981,9 @@ async fn main() -> Result<()> { SubCommand::Build(BuildCommand::Dot) => { do_dot(&get_config()?).await?; } - SubCommand::Build(BuildCommand::Package) => { - do_package(&get_config()?, &args.artifact_dir).await?; + SubCommand::Build(BuildCommand::Package { disable_cache }) => { + do_package(&get_config()?, &args.artifact_dir, *disable_cache) + .await?; } SubCommand::Build(BuildCommand::Stamp { package_name, version }) => { do_stamp(&get_config()?, &args.artifact_dir, package_name, version) diff --git a/package/src/dot.rs b/package/src/dot.rs index 133d5c0f00..3307d100ba 100644 --- a/package/src/dot.rs +++ b/package/src/dot.rs @@ -10,7 +10,6 @@ use petgraph::graph::EdgeReference; use petgraph::graph::NodeIndex; use petgraph::Graph; use std::collections::BTreeMap; -use std::path::Path; /// A node in our visual representation of the package manifest /// @@ -132,13 +131,11 @@ pub fn do_dot( let pkg_node = pkg_nodes .get(pkgname) .expect("expected node for package already"); - let output_directory = Path::new("/nonexistent"); + let output_directory = camino::Utf8Path::new("/nonexistent"); let output_basename = pkg .get_output_path(pkgname, output_directory) .file_name() - .unwrap() - .to_str() - .expect("expected package output filename to be UTF-8") + .expect("Missing file name") .to_string(); (output_basename, pkg_node) }) @@ -190,9 +187,8 @@ pub fn do_dot( // on which it depends. if let Some(blobs) = blobs { for b in blobs { - let s3_node = graph.add_node(GraphNode::Blob { - path: b.display().to_string(), - }); + let s3_node = graph + .add_node(GraphNode::Blob { path: b.to_string() }); graph.add_edge(*pkg_node, s3_node, "download"); } } diff --git a/package/src/lib.rs b/package/src/lib.rs index 395f3ed472..bba1a3a0cd 100644 --- a/package/src/lib.rs +++ b/package/src/lib.rs @@ -1,9 +1,8 @@ //! Common code shared between `omicron-package` and `thing-flinger` binaries. +use camino::{Utf8Path, Utf8PathBuf}; use clap::Subcommand; use serde::de::DeserializeOwned; -use std::path::Path; -use std::path::PathBuf; use thiserror::Error; pub mod dot; @@ -13,12 +12,12 @@ pub mod target; #[derive(Error, Debug)] pub enum ParseError { #[error("Error deserializing toml from {path}: {err}")] - Toml { path: PathBuf, err: toml::de::Error }, + Toml { path: Utf8PathBuf, err: toml::de::Error }, #[error("IO error: {message}: {err}")] Io { message: String, err: std::io::Error }, } -pub fn parse, C: DeserializeOwned>( +pub fn parse, C: DeserializeOwned>( path: P, ) -> Result { let path = path.as_ref(); @@ -93,7 +92,13 @@ pub enum BuildCommand { Dot, /// Builds the packages specified in a manifest, and places them into an /// 'out' directory. - Package, + Package { + /// If true, disables the cache. + /// + /// By default, the cache is used. + #[clap(short, long)] + disable_cache: bool, + }, /// Stamps semver versions onto packages within a manifest Stamp { /// The name of the artifact to be stamped. @@ -116,7 +121,7 @@ pub enum DeployCommand { /// /// Defaults to "/opt/oxide". #[clap(long = "out", default_value = "/opt/oxide", action)] - install_dir: PathBuf, + install_dir: Utf8PathBuf, }, /// Unpacks the files created by `package` to an install directory. /// Issues the `uninstall` command. @@ -134,7 +139,7 @@ pub enum DeployCommand { /// /// Defaults to "/opt/oxide". #[clap(long = "out", default_value = "/opt/oxide", action)] - install_dir: PathBuf, + install_dir: Utf8PathBuf, }, /// Imports and starts the sled-agent illumos service /// @@ -145,7 +150,7 @@ pub enum DeployCommand { /// /// Defaults to "/opt/oxide". #[clap(long = "out", default_value = "/opt/oxide", action)] - install_dir: PathBuf, + install_dir: Utf8PathBuf, }, /// Deletes all Omicron zones and stops all services. /// @@ -166,6 +171,6 @@ pub enum DeployCommand { /// /// Defaults to "/opt/oxide". #[clap(long = "out", default_value = "/opt/oxide", action)] - install_dir: PathBuf, + install_dir: Utf8PathBuf, }, } diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index be82bc5fda..c591669cc1 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -26,7 +26,6 @@ bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2.17" } bstr-dff4ba8e3ae991db = { package = "bstr", version = "1.6.0" } byteorder = { version = "1.5.0" } bytes = { version = "1.5.0", features = ["serde"] } -camino = { version = "1.1.6", default-features = false, features = ["serde1"] } chrono = { version = "0.4.31", features = ["alloc", "serde"] } cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] } clap = { version = "4.4.3", features = ["cargo", "derive", "env", "wrap_help"] } @@ -134,7 +133,6 @@ bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2.17" } bstr-dff4ba8e3ae991db = { package = "bstr", version = "1.6.0" } byteorder = { version = "1.5.0" } bytes = { version = "1.5.0", features = ["serde"] } -camino = { version = "1.1.6", default-features = false, features = ["serde1"] } chrono = { version = "0.4.31", features = ["alloc", "serde"] } cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] } clap = { version = "4.4.3", features = ["cargo", "derive", "env", "wrap_help"] }