From ee79d02d7a35df2d59ef041f81b314d8465d992e Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Sat, 11 May 2024 19:45:06 +0000 Subject: [PATCH 01/35] wip --- Cargo.lock | 24 ++ Cargo.toml | 2 + dev-tools/releng/Cargo.toml | 26 ++ dev-tools/releng/src/cmd.rs | 115 +++++++++ dev-tools/releng/src/job.rs | 248 +++++++++++++++++++ dev-tools/releng/src/main.rs | 450 +++++++++++++++++++++++++++++++++++ package-manifest.toml | 2 +- workspace-hack/Cargo.toml | 2 + 8 files changed, 868 insertions(+), 1 deletion(-) create mode 100644 dev-tools/releng/Cargo.toml create mode 100644 dev-tools/releng/src/cmd.rs create mode 100644 dev-tools/releng/src/job.rs create mode 100644 dev-tools/releng/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 11dd1839a5..5e32f7e33a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2547,6 +2547,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88a41f105fe1d5b6b34b2055e3dc59bb79b46b48b2040b9e6c7b4b5de097aa41" dependencies = [ "autocfg", + "tokio", ] [[package]] @@ -5591,6 +5592,28 @@ dependencies = [ "thiserror", ] +[[package]] +name = "omicron-releng" +version = "0.1.0" +dependencies = [ + "anyhow", + "camino", + "camino-tempfile", + "chrono", + "clap", + "fs-err", + "futures", + "omicron-workspace-hack", + "omicron-zone-package", + "semver 1.0.22", + "shell-words", + "slog", + "slog-async", + "slog-term", + "tar", + "tokio", +] + [[package]] name = "omicron-rpaths" version = "0.1.0" @@ -5777,6 +5800,7 @@ dependencies = [ "elliptic-curve", "ff", "flate2", + "fs-err", "futures", "futures-channel", "futures-core", diff --git a/Cargo.toml b/Cargo.toml index 7f778c0197..da03a6e792 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "dev-tools/omicron-dev", "dev-tools/oxlog", "dev-tools/reconfigurator-cli", + "dev-tools/releng", "dev-tools/xtask", "dns-server", "end-to-end-tests", @@ -103,6 +104,7 @@ default-members = [ "dev-tools/omicron-dev", "dev-tools/oxlog", "dev-tools/reconfigurator-cli", + "dev-tools/releng", # Do not include xtask in the list of default members, because this causes # hakari to not work as well and build times to be longer. # See omicron#4392. diff --git a/dev-tools/releng/Cargo.toml b/dev-tools/releng/Cargo.toml new file mode 100644 index 0000000000..95aa3b4bb4 --- /dev/null +++ b/dev-tools/releng/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "omicron-releng" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[dependencies] +anyhow.workspace = true +camino-tempfile.workspace = true +camino.workspace = true +fs-err = { workspace = true, features = ["tokio"] } +omicron-workspace-hack.workspace = true +omicron-zone-package.workspace = true +semver.workspace = true +shell-words.workspace = true +slog-async.workspace = true +slog-term.workspace = true +slog.workspace = true +tar.workspace = true +clap.workspace = true +tokio = { workspace = true, features = ["full"] } +chrono.workspace = true +futures.workspace = true + +[lints] +workspace = true diff --git a/dev-tools/releng/src/cmd.rs b/dev-tools/releng/src/cmd.rs new file mode 100644 index 0000000000..7fa9d2a713 --- /dev/null +++ b/dev-tools/releng/src/cmd.rs @@ -0,0 +1,115 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::ffi::OsStr; +use std::fmt::Write; +use std::process::ExitStatus; +use std::process::Output; +use std::process::Stdio; +use std::time::Instant; + +use anyhow::ensure; +use anyhow::Context; +use anyhow::Result; +use slog::debug; +use slog::Logger; +use tokio::process::Command; + +pub(crate) trait CommandExt { + fn check_status(&self, status: ExitStatus) -> Result<()>; + fn to_string(&self) -> String; + + async fn is_success(&mut self, logger: &Logger) -> Result; + async fn ensure_success(&mut self, logger: &Logger) -> Result<()>; + async fn ensure_stdout(&mut self, logger: &Logger) -> Result; +} + +impl CommandExt for Command { + fn check_status(&self, status: ExitStatus) -> Result<()> { + ensure!( + status.success(), + "command `{}` exited with {}", + self.to_string(), + status + ); + Ok(()) + } + + fn to_string(&self) -> String { + let command = self.as_std(); + let mut command_str = String::new(); + for (name, value) in command.get_envs() { + if let Some(value) = value { + write!( + command_str, + "{}={} ", + shell_words::quote(&name.to_string_lossy()), + shell_words::quote(&value.to_string_lossy()) + ) + .unwrap(); + } + } + write!( + command_str, + "{}", + shell_words::join( + std::iter::once(command.get_program()) + .chain(command.get_args()) + .map(OsStr::to_string_lossy) + ) + ) + .unwrap(); + command_str + } + + async fn is_success(&mut self, logger: &Logger) -> Result { + let output = run( + self.stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()), + logger, + ) + .await?; + Ok(output.status.success()) + } + + async fn ensure_success(&mut self, logger: &Logger) -> Result<()> { + let output = run( + self.stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()), + logger, + ) + .await?; + self.check_status(output.status) + } + + async fn ensure_stdout(&mut self, logger: &Logger) -> Result { + let output = run( + self.stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()), + logger, + ) + .await?; + self.check_status(output.status)?; + String::from_utf8(output.stdout).context("command stdout was not UTF-8") + } +} + +async fn run(command: &mut Command, logger: &Logger) -> Result { + debug!(logger, "running: {}", command.to_string()); + let start = Instant::now(); + let output = + command.kill_on_drop(true).output().await.with_context(|| { + format!("failed to exec `{}`", command.to_string()) + })?; + debug!( + logger, + "process exited with {} ({:?})", + output.status, + Instant::now().saturating_duration_since(start) + ); + Ok(output) +} diff --git a/dev-tools/releng/src/job.rs b/dev-tools/releng/src/job.rs new file mode 100644 index 0000000000..bfe42b5973 --- /dev/null +++ b/dev-tools/releng/src/job.rs @@ -0,0 +1,248 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::process::Stdio; +use std::time::Instant; + +use anyhow::anyhow; +use anyhow::Context; +use anyhow::Result; +use camino::Utf8Path; +use camino::Utf8PathBuf; +use fs_err::tokio::File; +use futures::stream::FuturesUnordered; +use futures::stream::TryStreamExt; +use slog::debug; +use slog::error; +use slog::Logger; +use tokio::io::AsyncBufReadExt; +use tokio::io::AsyncRead; +use tokio::io::AsyncWrite; +use tokio::io::AsyncWriteExt; +use tokio::io::BufReader; +use tokio::process::Command; +use tokio::sync::oneshot; +use tokio::sync::oneshot::error::RecvError; + +use crate::cmd::CommandExt; + +pub(crate) struct Jobs { + logger: Logger, + log_dir: Utf8PathBuf, + map: HashMap, +} + +struct Job { + future: Pin>>>, + wait_for: Vec>, + notify: Vec>, +} + +pub(crate) struct Selector<'a> { + jobs: &'a mut Jobs, + name: String, +} + +impl Jobs { + pub(crate) fn new(logger: &Logger, log_dir: &Utf8Path) -> Jobs { + Jobs { + logger: logger.clone(), + log_dir: log_dir.to_owned(), + map: HashMap::new(), + } + } + + pub(crate) fn push( + &mut self, + name: impl AsRef, + future: F, + ) -> Selector<'_> + where + F: Future> + 'static, + { + let name = name.as_ref().to_owned(); + assert!(!self.map.contains_key(&name), "duplicate job name {}", name); + self.map.insert( + name.clone(), + Job { + future: Box::pin(run_job( + self.logger.clone(), + name.clone(), + future, + )), + wait_for: Vec::new(), + notify: Vec::new(), + }, + ); + Selector { jobs: self, name } + } + + pub(crate) fn push_command( + &mut self, + name: impl AsRef, + command: &mut Command, + ) -> Selector<'_> { + let name = name.as_ref().to_owned(); + assert!(!self.map.contains_key(&name), "duplicate job name {}", name); + self.map.insert( + name.clone(), + Job { + future: Box::pin(spawn_with_output( + // terrible hack to deal with the `Command` builder + // returning &mut + std::mem::replace(command, Command::new("false")), + self.logger.clone(), + name.clone(), + self.log_dir.join(&name).with_extension("log"), + )), + wait_for: Vec::new(), + notify: Vec::new(), + }, + ); + Selector { jobs: self, name } + } + + pub(crate) fn select(&mut self, name: impl AsRef) -> Selector<'_> { + Selector { jobs: self, name: name.as_ref().to_owned() } + } + + pub(crate) async fn run_all(self) -> Result<()> { + self.map + .into_values() + .map(Job::run) + .collect::>() + .try_collect::<()>() + .await + } +} + +impl Job { + async fn run(self) -> Result<()> { + let result: Result<(), RecvError> = self + .wait_for + .into_iter() + .collect::>() + .try_collect::<()>() + .await; + result.map_err(|_| anyhow!("dependency failed"))?; + + self.future.await?; + for sender in self.notify { + sender.send(()).ok(); + } + Ok(()) + } +} + +impl<'a> Selector<'a> { + #[track_caller] + pub(crate) fn after(self, other: impl AsRef) -> Self { + let (sender, receiver) = oneshot::channel(); + self.jobs + .map + .get_mut(&self.name) + .expect("invalid job name") + .wait_for + .push(receiver); + self.jobs + .map + .get_mut(other.as_ref()) + .expect("invalid job name") + .notify + .push(sender); + self + } +} + +async fn run_job(logger: Logger, name: String, future: F) -> Result<()> +where + F: Future> + 'static, +{ + debug!(logger, "[{}] running task", name); + let start = Instant::now(); + let result = future.await; + let duration = Instant::now().saturating_duration_since(start); + match result { + Ok(()) => { + debug!(logger, "[{}] task succeeded ({:?})", name, duration); + Ok(()) + } + Err(err) => { + error!(logger, "[{}] task failed ({:?})", name, duration); + Err(err) + } + } +} + +async fn spawn_with_output( + mut command: Command, + logger: Logger, + name: String, + log_path: Utf8PathBuf, +) -> Result<()> { + let log_file_1 = File::create(log_path).await?; + let log_file_2 = log_file_1.try_clone().await?; + + debug!(logger, "[{}] running: {}", name, command.to_string()); + let start = Instant::now(); + let mut child = command + .kill_on_drop(true) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to exec `{}`", command.to_string()))?; + + let stdout = reader( + &name, + child.stdout.take().unwrap(), + tokio::io::stdout(), + log_file_1, + ); + let stderr = reader( + &name, + child.stderr.take().unwrap(), + tokio::io::stderr(), + log_file_2, + ); + match tokio::try_join!(child.wait(), stdout, stderr) { + Ok((status, (), ())) => { + debug!( + logger, + "[{}] process exited with {} ({:?})", + name, + status, + Instant::now().saturating_duration_since(start) + ); + command.check_status(status) + } + Err(err) => Err(err).with_context(|| { + format!("I/O error while waiting for job {:?} to complete", name) + }), + } +} + +async fn reader( + name: &str, + reader: impl AsyncRead + Unpin, + mut terminal_writer: impl AsyncWrite + Unpin, + logfile_writer: File, +) -> std::io::Result<()> { + let mut reader = BufReader::new(reader); + let mut logfile_writer = tokio::fs::File::from(logfile_writer); + let mut buf = format!("[{:>16}] ", name).into_bytes(); + let prefix_len = buf.len(); + loop { + buf.truncate(prefix_len); + let size = reader.read_until(b'\n', &mut buf).await?; + if size == 0 { + return Ok(()); + } + terminal_writer.write_all(&buf).await?; + logfile_writer.write_all(&buf[prefix_len..]).await?; + } +} diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs new file mode 100644 index 0000000000..a981155014 --- /dev/null +++ b/dev-tools/releng/src/main.rs @@ -0,0 +1,450 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +mod cmd; +mod job; + +use std::sync::Arc; + +use anyhow::ensure; +use anyhow::Context; +use anyhow::Result; +use camino::Utf8PathBuf; +use chrono::Utc; +use clap::Parser; +use fs_err::tokio as fs; +use omicron_zone_package::config::Config; +use semver::Version; +use slog::debug; +use slog::info; +use slog::Drain; +use slog::Logger; +use slog_term::FullFormat; +use slog_term::TermDecorator; +use tokio::process::Command; + +use crate::cmd::CommandExt; +use crate::job::Jobs; + +/// The base version we're currently building. Build information is appended to +/// this later on. +/// +/// Under current policy, each new release is a major version bump, and +/// generally referred to only by the major version (e.g. 8.0.0 is referred +/// to as "v8", "version 8", or "release 8" to customers). The use of semantic +/// versioning is mostly to hedge for perhaps wanting something more granular in +/// the future. +const BASE_VERSION: Version = Version::new(8, 0, 0); + +#[derive(Debug, Clone, Copy)] +enum InstallMethod { + /// Unpack the tarball to `/opt/oxide/`, and install + /// `pkg/manifest.xml` (if it exists) to + /// `/lib/svc/manifest/site/.xml`. + Install, + /// Copy the tarball to `/opt/oxide/.tar.gz`. + Bundle, +} + +/// Packages to install or bundle in the host OS image. +const HOST_IMAGE_PACKAGES: [(&str, InstallMethod); 7] = [ + ("mg-ddm-gz", InstallMethod::Install), + ("omicron-sled-agent", InstallMethod::Install), + ("overlay", InstallMethod::Bundle), + ("oxlog", InstallMethod::Install), + ("propolis-server", InstallMethod::Bundle), + ("pumpkind-gz", InstallMethod::Install), + ("switch-asic", InstallMethod::Bundle), +]; +/// Packages to install or bundle in the recovery (trampoline) OS image. +const RECOVERY_IMAGE_PACKAGES: [(&str, InstallMethod); 2] = [ + ("installinator", InstallMethod::Install), + ("mg-ddm-gz", InstallMethod::Install), +]; + +const HELIOS_REPO: &str = "https://pkg.oxide.computer/helios/2/dev/"; +const OPTE_VERSION: &str = include_str!("../../../tools/opte_version"); + +#[derive(Parser)] +struct Args { + /// Path to a Helios repository checkout (default: "helios" in the same + /// directory as "omicron") + #[clap(long)] + helios_path: Option, + + /// ZFS dataset to use for `helios-build` (default: "rpool/images/$LOGNAME") + #[clap(long)] + helios_image_dataset: Option, + + /// Ignore the current HEAD of the Helios repository checkout + #[clap(long)] + ignore_helios_origin: bool, + + /// Output dir for TUF repo and log files (default: "out/releng" in the + /// "omicron" directory) + #[clap(long)] + output_dir: Option, +} + +fn main() -> Result<()> { + let decorator = TermDecorator::new().build(); + let drain = FullFormat::new(decorator).build().fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + let logger = Logger::root(drain, slog::o!()); + + // Change the working directory to the workspace root. + let workspace_dir = + Utf8PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").context( + "$CARGO_MANIFEST_DIR is not set; run this via `cargo xtask releng`", + )?) + // $CARGO_MANIFEST_DIR is `.../omicron/dev-tools/releng` + .join("../..") + .canonicalize_utf8() + .context("failed to canonicalize workspace dir")?; + info!(logger, "changing working directory to {}", workspace_dir); + std::env::set_current_dir(&workspace_dir) + .context("failed to change working directory to workspace root")?; + + // Unset `CARGO*` and `RUSTUP_TOOLCHAIN`, which will interfere with various + // tools we're about to run. + for (name, _) in std::env::vars_os() { + if name + .to_str() + .map(|s| s.starts_with("CARGO") || s == "RUSTUP_TOOLCHAIN") + .unwrap_or(false) + { + debug!(logger, "unsetting {:?}", name); + std::env::remove_var(name); + } + } + + // Now that we're done mucking about with our environment (something that's + // not necessarily safe in multi-threaded programs), create a Tokio runtime + // and call `do_run`. + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap() + .block_on(do_run(logger, workspace_dir)) +} + +async fn do_run(logger: Logger, workspace_dir: Utf8PathBuf) -> Result<()> { + let args = Args::parse(); + + let helios_dir = args.helios_path.unwrap_or_else(|| { + workspace_dir + .parent() + .expect("omicron repo is not the root directory") + .join("helios") + }); + let output_dir = args + .output_dir + .unwrap_or_else(|| workspace_dir.join("out").join("releng")); + let tempdir = camino_tempfile::tempdir() + .context("failed to create temporary directory")?; + + let commit = Command::new("git") + .args(["rev-parse", "HEAD"]) + .ensure_stdout(&logger) + .await? + .trim() + .to_owned(); + + let mut version = BASE_VERSION.clone(); + // Differentiate between CI and local builds. + version.pre = + if std::env::var_os("CI").is_some() { "0.ci" } else { "0.local" } + .parse()?; + // Set the build metadata to the current commit hash. + let mut build = String::with_capacity(14); + build.push_str("git"); + build.extend(commit.chars().take(11)); + version.build = build.parse()?; + info!(logger, "version: {}", version); + + let manifest = Arc::new(omicron_zone_package::config::parse_manifest( + &fs::read_to_string(workspace_dir.join("package-manifest.toml")) + .await?, + )?); + + // PREFLIGHT ============================================================== + for (package, _) in HOST_IMAGE_PACKAGES + .into_iter() + .chain(RECOVERY_IMAGE_PACKAGES.into_iter()) + { + ensure!( + manifest.packages.contains_key(package), + "package {} to be installed in the OS image \ + is not listed in the package manifest", + package + ); + } + + // Ensure the Helios checkout exists + if helios_dir.exists() { + if !args.ignore_helios_origin { + // check that our helios clone is up to date + Command::new("git") + .arg("-C") + .arg(&helios_dir) + .args(["fetch", "--no-write-fetch-head", "origin", "master"]) + .ensure_success(&logger) + .await?; + let stdout = Command::new("git") + .arg("-C") + .arg(&helios_dir) + .args(["rev-parse", "HEAD", "origin/master"]) + .ensure_stdout(&logger) + .await?; + let mut lines = stdout.lines(); + let first = + lines.next().context("git-rev-parse output was empty")?; + ensure!( + lines.all(|line| line == first), + "helios checkout at {0} is out-of-date; run \ + `git pull -C {0}`, or run omicron-releng with \ + --ignore-helios-origin or --helios-path", + shell_words::quote(helios_dir.as_str()) + ); + } + } else { + info!(logger, "cloning helios to {}", helios_dir); + Command::new("git") + .args(["clone", "https://github.com/oxidecomputer/helios.git"]) + .arg(&helios_dir) + .ensure_success(&logger) + .await?; + } + + // Check that the omicron1 brand is installed + ensure!( + Command::new("pkg") + .args(["verify", "-q", "/system/zones/brand/omicron1/tools"]) + .is_success(&logger) + .await?, + "the omicron1 brand is not installed; install it with \ + `pfexec pkg install /system/zones/brand/omicron1/tools`" + ); + + // Check that the dataset for helios-image to use exists + let helios_image_dataset = match args.helios_image_dataset { + Some(s) => s, + None => format!( + "rpool/images/{}", + std::env::var("LOGNAME") + .context("$LOGNAME is not present in environment")? + ), + }; + ensure!( + Command::new("zfs") + .arg("list") + .arg(&helios_image_dataset) + .is_success(&logger) + .await?, + "the dataset {0} does not exist, which is required for helios-build; \ + run `pfexec zfs create -p {0}`, or run omicron-releng with \ + --helios-image-dataset to specify a different one", + shell_words::quote(&helios_image_dataset) + ); + + fs::create_dir_all(&output_dir).await?; + + // DEFINE JOBS ============================================================ + let mut jobs = Jobs::new(&logger, &output_dir); + + jobs.push_command( + "helios-setup", + Command::new("ptime") + .args(["-m", "gmake", "setup"]) + .current_dir(&helios_dir) + // ?! + .env("PWD", &helios_dir) + // Setting `BUILD_OS` to no makes setup skip repositories we don't need + // for building the OS itself (we are just building an image from an + // already-built OS). + .env("BUILD_OS", "no"), + ); + + jobs.push_command( + "omicron-package", + Command::new("ptime").args([ + "-m", + "cargo", + "build", + "--locked", + "--release", + "--bin", + "omicron-package", + ]), + ); + let omicron_package = workspace_dir.join("target/release/omicron-package"); + + macro_rules! os_image_jobs { + ( + target_name: $target_name:literal, + target_args: $target_args:expr, + proto_packages: $proto_packages:expr, + image_prefix: $image_prefix:literal, + image_build_args: $image_build_args:expr, + ) => { + jobs.push_command( + concat!($target_name, "-target"), + Command::new(&omicron_package) + .args(["--target", $target_name, "target", "create"]) + .args($target_args), + ) + .after("omicron-package"); + + jobs.push_command( + concat!($target_name, "-package"), + Command::new(&omicron_package).args([ + "--target", + $target_name, + "package", + ]), + ) + .after(concat!($target_name, "-target")); + + let proto_dir = tempdir.path().join("proto").join($target_name); + jobs.push( + concat!($target_name, "-proto"), + build_proto_area( + workspace_dir.join("out"), + proto_dir.clone(), + &$proto_packages, + manifest.clone(), + ), + ) + .after(concat!($target_name, "-package")); + + // The ${os_short_commit} token will be expanded by `helios-build` + let image_name = format!( + "{} {}/${{os_short_commit}} {}", + $image_prefix, + commit.chars().take(7).collect::(), + Utc::now().format("%Y-%m-%d %H:%M") + ); + + jobs.push_command( + concat!($target_name, "-image"), + Command::new("ptime") + .arg("-m") + .arg(helios_dir.join("helios-build")) + .arg("experiment-image") + .arg("-o") + .arg(output_dir.join($target_name)) + .arg("-p") + .arg(format!("helios-dev={}", HELIOS_REPO)) + .arg("-F") + .arg(format!("optever={}", OPTE_VERSION.trim())) + .arg("-P") + .arg(proto_dir.join("root")) + .arg("-N") + .arg(image_name) + .args($image_build_args) + .current_dir(&helios_dir), + ) + .after("helios-setup") + .after(concat!($target_name, "-proto")); + }; + } + + os_image_jobs! { + target_name: "recovery", + target_args: ["--image", "trampoline"], + proto_packages: RECOVERY_IMAGE_PACKAGES, + image_prefix: "recovery", + image_build_args: ["-R"], + } + os_image_jobs! { + target_name: "host", + target_args: [ + "--image", + "standard", + "--machine", + "gimlet", + "--switch", + "asic", + "--rack-topology", + "multi-sled" + ], + proto_packages: HOST_IMAGE_PACKAGES, + image_prefix: "ci", + image_build_args: ["-B"], + } + // avoid fighting for the target dir lock + jobs.select("host-package").after("recovery-package"); + // only one helios-build job can run at once + jobs.select("host-image").after("recovery-image"); + + // RUN JOBS =============================================================== + jobs.run_all().await?; + + // fs::create_dir_all(host_proto.path().join("root/root"))?; + // fs::write( + // host_proto.path().join("root/root/.profile"), + // "# Add opteadm, ddadm, oxlog to PATH\n\ + // export PATH=$PATH:/opt/oxide/opte/bin:/opt/oxide/mg-ddm:/opt/oxide/oxlog\n" + // )?; + + Ok(()) +} + +async fn build_proto_area( + package_dir: Utf8PathBuf, + proto_dir: Utf8PathBuf, + packages: &'static [(&'static str, InstallMethod)], + manifest: Arc, +) -> Result<()> { + let opt_oxide = proto_dir.join("root/opt/oxide"); + let manifest_site = proto_dir.join("root/lib/svc/manifest/site"); + fs::create_dir_all(&opt_oxide).await?; + + for &(package_name, method) in packages { + let package = + manifest.packages.get(package_name).expect("checked in preflight"); + match method { + InstallMethod::Install => { + let path = opt_oxide.join(&package.service_name); + fs::create_dir(&path).await?; + + let cloned_path = path.clone(); + let cloned_package_dir = package_dir.to_owned(); + tokio::task::spawn_blocking(move || -> Result<()> { + let mut archive = tar::Archive::new(std::fs::File::open( + cloned_package_dir + .join(package_name) + .with_extension("tar"), + )?); + archive.unpack(cloned_path).with_context(|| { + format!("failed to extract {}.tar.gz", package_name) + })?; + Ok(()) + }) + .await??; + + let smf_manifest = path.join("pkg").join("manifest.xml"); + if smf_manifest.exists() { + fs::create_dir_all(&manifest_site).await?; + fs::rename( + smf_manifest, + manifest_site + .join(&package.service_name) + .with_extension("xml"), + ) + .await?; + } + } + InstallMethod::Bundle => { + fs::copy( + package_dir.join(format!("{}.tar.gz", package_name)), + opt_oxide.join(format!("{}.tar.gz", package.service_name)), + ) + .await?; + } + } + } + + Ok(()) +} diff --git a/package-manifest.toml b/package-manifest.toml index 55e6f78221..98cd5035d0 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -592,7 +592,7 @@ only_for_targets.image = "standard" only_for_targets.switch = "asic" [package.pumpkind-gz] -service_name = "pumpkind-gz" +service_name = "pumpkind" source.type = "prebuilt" source.repo = "pumpkind" source.commit = "3fe9c306590fb2f28f54ace7fd18b3c126323683" diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 2fecfa3a75..32423cb009 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -46,6 +46,7 @@ either = { version = "1.11.0" } elliptic-curve = { version = "0.13.8", features = ["ecdh", "hazmat", "pem", "std"] } ff = { version = "0.13.0", default-features = false, features = ["alloc"] } flate2 = { version = "1.0.28" } +fs-err = { version = "2.11.0", default-features = false, features = ["tokio"] } futures = { version = "0.3.30" } futures-channel = { version = "0.3.30", features = ["sink"] } futures-core = { version = "0.3.30" } @@ -153,6 +154,7 @@ either = { version = "1.11.0" } elliptic-curve = { version = "0.13.8", features = ["ecdh", "hazmat", "pem", "std"] } ff = { version = "0.13.0", default-features = false, features = ["alloc"] } flate2 = { version = "1.0.28" } +fs-err = { version = "2.11.0", default-features = false, features = ["tokio"] } futures = { version = "0.3.30" } futures-channel = { version = "0.3.30", features = ["sink"] } futures-core = { version = "0.3.30" } From d58426263086c90e18a9a9a265cb3a4851d33634 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Sat, 11 May 2024 22:35:40 +0000 Subject: [PATCH 02/35] bug fixes and performance improvements --- Cargo.lock | 1 + dev-tools/releng/Cargo.toml | 11 +- dev-tools/releng/src/main.rs | 291 +++++++++++++++++++++-------------- 3 files changed, 185 insertions(+), 118 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5e32f7e33a..e39fb836c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5605,6 +5605,7 @@ dependencies = [ "futures", "omicron-workspace-hack", "omicron-zone-package", + "once_cell", "semver 1.0.22", "shell-words", "slog", diff --git a/dev-tools/releng/Cargo.toml b/dev-tools/releng/Cargo.toml index 95aa3b4bb4..25a7b09cdf 100644 --- a/dev-tools/releng/Cargo.toml +++ b/dev-tools/releng/Cargo.toml @@ -6,21 +6,22 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true -camino-tempfile.workspace = true camino.workspace = true +camino-tempfile.workspace = true +chrono.workspace = true +clap.workspace = true fs-err = { workspace = true, features = ["tokio"] } +futures.workspace = true omicron-workspace-hack.workspace = true omicron-zone-package.workspace = true +once_cell.workspace = true semver.workspace = true shell-words.workspace = true +slog.workspace = true slog-async.workspace = true slog-term.workspace = true -slog.workspace = true tar.workspace = true -clap.workspace = true tokio = { workspace = true, features = ["full"] } -chrono.workspace = true -futures.workspace = true [lints] workspace = true diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index a981155014..c45376cac9 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -6,8 +6,9 @@ mod cmd; mod job; use std::sync::Arc; +use std::time::Instant; -use anyhow::ensure; +use anyhow::bail; use anyhow::Context; use anyhow::Result; use camino::Utf8PathBuf; @@ -15,8 +16,10 @@ use chrono::Utc; use clap::Parser; use fs_err::tokio as fs; use omicron_zone_package::config::Config; +use once_cell::sync::Lazy; use semver::Version; use slog::debug; +use slog::error; use slog::info; use slog::Drain; use slog::Logger; @@ -66,48 +69,86 @@ const RECOVERY_IMAGE_PACKAGES: [(&str, InstallMethod); 2] = [ const HELIOS_REPO: &str = "https://pkg.oxide.computer/helios/2/dev/"; const OPTE_VERSION: &str = include_str!("../../../tools/opte_version"); +static WORKSPACE_DIR: Lazy = Lazy::new(|| { + // $CARGO_MANIFEST_DIR is at `.../omicron/dev-tools/releng` + let mut dir = + Utf8PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect( + "$CARGO_MANIFEST_DIR is not set; run this via `cargo xtask releng`", + )); + dir.pop(); + dir.pop(); + dir +}); + #[derive(Parser)] +/// Run the Oxide release engineering process and produce a TUF repo that can be +/// used to update a rack. +/// +/// For more information, see `docs/releng.adoc` in the Omicron repository. +/// +/// Note that `--host-dataset` and `--recovery-dataset` must be set to different +/// values to build the two OS images in parallel. This is strongly recommended. struct Args { + /// ZFS dataset to use for `helios-build` when building the host image + #[clap(long, default_value_t = Self::default_dataset("host"))] + host_dataset: String, + + /// ZFS dataset to use for `helios-build` when building the recovery + /// (trampoline) image + #[clap(long, default_value_t = Self::default_dataset("recovery"))] + recovery_dataset: String, + /// Path to a Helios repository checkout (default: "helios" in the same /// directory as "omicron") - #[clap(long)] - helios_path: Option, - - /// ZFS dataset to use for `helios-build` (default: "rpool/images/$LOGNAME") - #[clap(long)] - helios_image_dataset: Option, + #[clap(long, default_value_t = Self::default_helios_dir())] + helios_dir: Utf8PathBuf, /// Ignore the current HEAD of the Helios repository checkout #[clap(long)] ignore_helios_origin: bool, - /// Output dir for TUF repo and log files (default: "out/releng" in the - /// "omicron" directory) - #[clap(long)] - output_dir: Option, + /// Output dir for TUF repo and log files + #[clap(long, default_value_t = Self::default_output_dir())] + output_dir: Utf8PathBuf, +} + +impl Args { + fn default_dataset(name: &str) -> String { + format!( + "rpool/images/{}/{}", + std::env::var("LOGNAME").expect("$LOGNAME is not set"), + name + ) + } + + fn default_helios_dir() -> Utf8PathBuf { + WORKSPACE_DIR + .parent() + .expect("omicron is presumably not cloned at /") + .join("helios") + } + + fn default_output_dir() -> Utf8PathBuf { + WORKSPACE_DIR.join("out/releng") + } } fn main() -> Result<()> { + let args = Args::parse(); + let decorator = TermDecorator::new().build(); let drain = FullFormat::new(decorator).build().fuse(); let drain = slog_async::Async::new(drain).build().fuse(); let logger = Logger::root(drain, slog::o!()); // Change the working directory to the workspace root. - let workspace_dir = - Utf8PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").context( - "$CARGO_MANIFEST_DIR is not set; run this via `cargo xtask releng`", - )?) - // $CARGO_MANIFEST_DIR is `.../omicron/dev-tools/releng` - .join("../..") - .canonicalize_utf8() - .context("failed to canonicalize workspace dir")?; - info!(logger, "changing working directory to {}", workspace_dir); - std::env::set_current_dir(&workspace_dir) + info!(logger, "changing working directory to {}", *WORKSPACE_DIR); + std::env::set_current_dir(&*WORKSPACE_DIR) .context("failed to change working directory to workspace root")?; - // Unset `CARGO*` and `RUSTUP_TOOLCHAIN`, which will interfere with various - // tools we're about to run. + // Unset `$CARGO*` and `$RUSTUP_TOOLCHAIN`, which will interfere with + // various tools we're about to run. (This needs to come _after_ we read + // from `WORKSPACE_DIR` as it relies on `$CARGO_MANIFEST_DIR`.) for (name, _) in std::env::vars_os() { if name .to_str() @@ -126,24 +167,10 @@ fn main() -> Result<()> { .enable_all() .build() .unwrap() - .block_on(do_run(logger, workspace_dir)) + .block_on(do_run(logger, args)) } -async fn do_run(logger: Logger, workspace_dir: Utf8PathBuf) -> Result<()> { - let args = Args::parse(); - - let helios_dir = args.helios_path.unwrap_or_else(|| { - workspace_dir - .parent() - .expect("omicron repo is not the root directory") - .join("helios") - }); - let output_dir = args - .output_dir - .unwrap_or_else(|| workspace_dir.join("out").join("releng")); - let tempdir = camino_tempfile::tempdir() - .context("failed to create temporary directory")?; - +async fn do_run(logger: Logger, args: Args) -> Result<()> { let commit = Command::new("git") .args(["rev-parse", "HEAD"]) .ensure_stdout(&logger) @@ -164,105 +191,125 @@ async fn do_run(logger: Logger, workspace_dir: Utf8PathBuf) -> Result<()> { info!(logger, "version: {}", version); let manifest = Arc::new(omicron_zone_package::config::parse_manifest( - &fs::read_to_string(workspace_dir.join("package-manifest.toml")) + &fs::read_to_string(WORKSPACE_DIR.join("package-manifest.toml")) .await?, )?); // PREFLIGHT ============================================================== + let mut preflight_ok = true; + for (package, _) in HOST_IMAGE_PACKAGES .into_iter() .chain(RECOVERY_IMAGE_PACKAGES.into_iter()) { - ensure!( - manifest.packages.contains_key(package), - "package {} to be installed in the OS image \ - is not listed in the package manifest", - package - ); + if !manifest.packages.contains_key(package) { + error!( + logger, + "package {} to be installed in the OS image \ + is not listed in the package manifest", + package + ); + preflight_ok = false; + } } // Ensure the Helios checkout exists - if helios_dir.exists() { + if args.helios_dir.exists() { if !args.ignore_helios_origin { // check that our helios clone is up to date Command::new("git") .arg("-C") - .arg(&helios_dir) + .arg(&args.helios_dir) .args(["fetch", "--no-write-fetch-head", "origin", "master"]) .ensure_success(&logger) .await?; let stdout = Command::new("git") .arg("-C") - .arg(&helios_dir) + .arg(&args.helios_dir) .args(["rev-parse", "HEAD", "origin/master"]) .ensure_stdout(&logger) .await?; let mut lines = stdout.lines(); let first = lines.next().context("git-rev-parse output was empty")?; - ensure!( - lines.all(|line| line == first), - "helios checkout at {0} is out-of-date; run \ - `git pull -C {0}`, or run omicron-releng with \ - --ignore-helios-origin or --helios-path", - shell_words::quote(helios_dir.as_str()) - ); + if !lines.all(|line| line == first) { + error!( + logger, + "helios checkout at {0} is out-of-date; run \ + `git pull -C {0}`, or run omicron-releng with \ + --ignore-helios-origin or --helios-path", + shell_words::quote(args.helios_dir.as_str()) + ); + preflight_ok = false; + } } } else { - info!(logger, "cloning helios to {}", helios_dir); + info!(logger, "cloning helios to {}", args.helios_dir); Command::new("git") .args(["clone", "https://github.com/oxidecomputer/helios.git"]) - .arg(&helios_dir) + .arg(&args.helios_dir) .ensure_success(&logger) .await?; } // Check that the omicron1 brand is installed - ensure!( - Command::new("pkg") - .args(["verify", "-q", "/system/zones/brand/omicron1/tools"]) - .is_success(&logger) - .await?, - "the omicron1 brand is not installed; install it with \ - `pfexec pkg install /system/zones/brand/omicron1/tools`" - ); + if !Command::new("pkg") + .args(["verify", "-q", "/system/zones/brand/omicron1/tools"]) + .is_success(&logger) + .await? + { + error!( + logger, + "the omicron1 brand is not installed; install it with \ + `pfexec pkg install /system/zones/brand/omicron1/tools`" + ); + preflight_ok = false; + } - // Check that the dataset for helios-image to use exists - let helios_image_dataset = match args.helios_image_dataset { - Some(s) => s, - None => format!( - "rpool/images/{}", - std::env::var("LOGNAME") - .context("$LOGNAME is not present in environment")? - ), - }; - ensure!( - Command::new("zfs") + // Check that the datasets for helios-image to use exist + for (dataset, option) in [ + (&args.host_dataset, "--host-dataset"), + (&args.recovery_dataset, "--recovery-dataset"), + ] { + if !Command::new("zfs") .arg("list") - .arg(&helios_image_dataset) + .arg(dataset) .is_success(&logger) - .await?, - "the dataset {0} does not exist, which is required for helios-build; \ - run `pfexec zfs create -p {0}`, or run omicron-releng with \ - --helios-image-dataset to specify a different one", - shell_words::quote(&helios_image_dataset) - ); + .await? + { + error!( + logger, + "the dataset {0} does not exist; run `pfexec zfs create \ + -p {0}`, or specify a different one with {1}", + shell_words::quote(dataset), + option + ); + preflight_ok = false; + } + } - fs::create_dir_all(&output_dir).await?; + if !preflight_ok { + bail!("some preflight checks failed"); + } + + fs::create_dir_all(&args.output_dir).await?; // DEFINE JOBS ============================================================ - let mut jobs = Jobs::new(&logger, &output_dir); + let tempdir = camino_tempfile::tempdir() + .context("failed to create temporary directory")?; + let mut jobs = Jobs::new(&logger, &args.output_dir); jobs.push_command( "helios-setup", Command::new("ptime") .args(["-m", "gmake", "setup"]) - .current_dir(&helios_dir) - // ?! - .env("PWD", &helios_dir) - // Setting `BUILD_OS` to no makes setup skip repositories we don't need - // for building the OS itself (we are just building an image from an - // already-built OS). + .current_dir(&args.helios_dir) + // ?!?! + // somehow, the Makefile does not see a new `$(PWD)` without this. + .env("PWD", &args.helios_dir) + // Setting `BUILD_OS` to no makes setup skip repositories we don't + // need for building the OS itself (we are just building an image + // from an already-built OS). .env("BUILD_OS", "no"), ); @@ -278,7 +325,7 @@ async fn do_run(logger: Logger, workspace_dir: Utf8PathBuf) -> Result<()> { "omicron-package", ]), ); - let omicron_package = workspace_dir.join("target/release/omicron-package"); + let omicron_package = WORKSPACE_DIR.join("target/release/omicron-package"); macro_rules! os_image_jobs { ( @@ -287,6 +334,7 @@ async fn do_run(logger: Logger, workspace_dir: Utf8PathBuf) -> Result<()> { proto_packages: $proto_packages:expr, image_prefix: $image_prefix:literal, image_build_args: $image_build_args:expr, + image_dataset: $image_dataset:expr, ) => { jobs.push_command( concat!($target_name, "-target"), @@ -310,7 +358,7 @@ async fn do_run(logger: Logger, workspace_dir: Utf8PathBuf) -> Result<()> { jobs.push( concat!($target_name, "-proto"), build_proto_area( - workspace_dir.join("out"), + WORKSPACE_DIR.join("out"), proto_dir.clone(), &$proto_packages, manifest.clone(), @@ -330,33 +378,29 @@ async fn do_run(logger: Logger, workspace_dir: Utf8PathBuf) -> Result<()> { concat!($target_name, "-image"), Command::new("ptime") .arg("-m") - .arg(helios_dir.join("helios-build")) + .arg(args.helios_dir.join("helios-build")) .arg("experiment-image") - .arg("-o") - .arg(output_dir.join($target_name)) - .arg("-p") + .arg("-o") // output directory for image + .arg(args.output_dir.join($target_name)) + .arg("-p") // use an external package repository .arg(format!("helios-dev={}", HELIOS_REPO)) - .arg("-F") + .arg("-F") // pass extra image builder features .arg(format!("optever={}", OPTE_VERSION.trim())) - .arg("-P") + .arg("-P") // include all files from extra proto area .arg(proto_dir.join("root")) - .arg("-N") + .arg("-N") // image name .arg(image_name) + .arg("-s") // tempdir name suffix + .arg($target_name) .args($image_build_args) - .current_dir(&helios_dir), + .current_dir(&args.helios_dir) + .env("IMAGE_DATASET", &$image_dataset), ) .after("helios-setup") .after(concat!($target_name, "-proto")); }; } - os_image_jobs! { - target_name: "recovery", - target_args: ["--image", "trampoline"], - proto_packages: RECOVERY_IMAGE_PACKAGES, - image_prefix: "recovery", - image_build_args: ["-R"], - } os_image_jobs! { target_name: "host", target_args: [ @@ -372,14 +416,35 @@ async fn do_run(logger: Logger, workspace_dir: Utf8PathBuf) -> Result<()> { proto_packages: HOST_IMAGE_PACKAGES, image_prefix: "ci", image_build_args: ["-B"], + image_dataset: args.host_dataset, + } + os_image_jobs! { + target_name: "recovery", + target_args: ["--image", "trampoline"], + proto_packages: RECOVERY_IMAGE_PACKAGES, + image_prefix: "recovery", + image_build_args: ["-R"], + image_dataset: args.recovery_dataset, + } + + // Build the recovery target after we build the host target. Only one + // of these will build at a time since Cargo locks its target directory; + // since host-package and host-image both take longer than their recovery + // counterparts, this should be the fastest option to go first. + jobs.select("recovery-package").after("host-package"); + if args.host_dataset == args.recovery_dataset { + // If the datasets are the same, we can't parallelize these. + jobs.select("recovery-image").after("host-image"); } - // avoid fighting for the target dir lock - jobs.select("host-package").after("recovery-package"); - // only one helios-build job can run at once - jobs.select("host-image").after("recovery-image"); // RUN JOBS =============================================================== + let start = Instant::now(); jobs.run_all().await?; + debug!( + logger, + "all jobs completed in {:?}", + Instant::now().saturating_duration_since(start) + ); // fs::create_dir_all(host_proto.path().join("root/root"))?; // fs::write( From e948b3cb86ab450bdb714d67a07f372a74dbffea Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Sat, 11 May 2024 22:39:35 +0000 Subject: [PATCH 03/35] i did not know you could do this! --- dev-tools/releng/src/main.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index c45376cac9..4d77423bd5 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -163,13 +163,10 @@ fn main() -> Result<()> { // Now that we're done mucking about with our environment (something that's // not necessarily safe in multi-threaded programs), create a Tokio runtime // and call `do_run`. - tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .unwrap() - .block_on(do_run(logger, args)) + do_run(logger, args) } +#[tokio::main] async fn do_run(logger: Logger, args: Args) -> Result<()> { let commit = Command::new("git") .args(["rev-parse", "HEAD"]) From eab1717809baf4caafa8b95c02fbb2df8d807687 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Sat, 11 May 2024 22:54:49 +0000 Subject: [PATCH 04/35] various logging tweaks --- dev-tools/releng/src/job.rs | 42 ++++++++++++++++++++++-------------- dev-tools/releng/src/main.rs | 23 ++++++++------------ 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/dev-tools/releng/src/job.rs b/dev-tools/releng/src/job.rs index bfe42b5973..acbac3dd8d 100644 --- a/dev-tools/releng/src/job.rs +++ b/dev-tools/releng/src/job.rs @@ -16,8 +16,7 @@ use camino::Utf8PathBuf; use fs_err::tokio::File; use futures::stream::FuturesUnordered; use futures::stream::TryStreamExt; -use slog::debug; -use slog::error; +use slog::info; use slog::Logger; use tokio::io::AsyncBufReadExt; use tokio::io::AsyncRead; @@ -158,24 +157,33 @@ impl<'a> Selector<'a> { } } +macro_rules! info_or_error { + ($logger:expr, $result:expr, $($tt:tt)*) => { + if $result.is_ok() { + ::slog::info!($logger, $($tt)*); + } else { + ::slog::error!($logger, $($tt)*); + } + }; +} + async fn run_job(logger: Logger, name: String, future: F) -> Result<()> where F: Future> + 'static, { - debug!(logger, "[{}] running task", name); + info!(logger, "[{}] running task", name); let start = Instant::now(); let result = future.await; let duration = Instant::now().saturating_duration_since(start); - match result { - Ok(()) => { - debug!(logger, "[{}] task succeeded ({:?})", name, duration); - Ok(()) - } - Err(err) => { - error!(logger, "[{}] task failed ({:?})", name, duration); - Err(err) - } - } + info_or_error!( + logger, + result, + "[{}] task {} ({:?})", + name, + if result.is_ok() { "succeeded" } else { "failed" }, + duration + ); + result } async fn spawn_with_output( @@ -187,7 +195,7 @@ async fn spawn_with_output( let log_file_1 = File::create(log_path).await?; let log_file_2 = log_file_1.try_clone().await?; - debug!(logger, "[{}] running: {}", name, command.to_string()); + info!(logger, "[{}] running: {}", name, command.to_string()); let start = Instant::now(); let mut child = command .kill_on_drop(true) @@ -211,14 +219,16 @@ async fn spawn_with_output( ); match tokio::try_join!(child.wait(), stdout, stderr) { Ok((status, (), ())) => { - debug!( + let result = command.check_status(status); + info_or_error!( logger, + result, "[{}] process exited with {} ({:?})", name, status, Instant::now().saturating_duration_since(start) ); - command.check_status(status) + result } Err(err) => Err(err).with_context(|| { format!("I/O error while waiting for job {:?} to complete", name) diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 4d77423bd5..b33498ef36 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -142,22 +142,17 @@ fn main() -> Result<()> { let logger = Logger::root(drain, slog::o!()); // Change the working directory to the workspace root. - info!(logger, "changing working directory to {}", *WORKSPACE_DIR); + debug!(logger, "changing working directory to {}", *WORKSPACE_DIR); std::env::set_current_dir(&*WORKSPACE_DIR) .context("failed to change working directory to workspace root")?; - // Unset `$CARGO*` and `$RUSTUP_TOOLCHAIN`, which will interfere with - // various tools we're about to run. (This needs to come _after_ we read - // from `WORKSPACE_DIR` as it relies on `$CARGO_MANIFEST_DIR`.) - for (name, _) in std::env::vars_os() { - if name - .to_str() - .map(|s| s.starts_with("CARGO") || s == "RUSTUP_TOOLCHAIN") - .unwrap_or(false) - { - debug!(logger, "unsetting {:?}", name); - std::env::remove_var(name); - } + // Unset `$CARGO`, `$CARGO_MANIFEST_DIR`, and `$RUSTUP_TOOLCHAIN` (all + // set by cargo or its rustup proxy), which will interfere with various + // tools we're about to run. (This needs to come _after_ we read from + // `WORKSPACE_DIR` as it relies on `$CARGO_MANIFEST_DIR`.) + for var in ["CARGO", "CARGO_MANIFEST_DIR", "RUSTUP_TOOLCHAIN"] { + debug!(logger, "unsetting ${}", var); + std::env::remove_var(var); } // Now that we're done mucking about with our environment (something that's @@ -437,7 +432,7 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { // RUN JOBS =============================================================== let start = Instant::now(); jobs.run_all().await?; - debug!( + info!( logger, "all jobs completed in {:?}", Instant::now().saturating_duration_since(start) From 3b009024de8903ea6743a1207de12047189934b8 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Sun, 12 May 2024 06:35:01 +0000 Subject: [PATCH 05/35] hubris bits, stamping, job limits --- Cargo.lock | 6 ++ Cargo.toml | 1 + dev-tools/releng/Cargo.toml | 6 ++ dev-tools/releng/src/hubris.rs | 180 +++++++++++++++++++++++++++++++++ dev-tools/releng/src/job.rs | 24 ++++- dev-tools/releng/src/main.rs | 109 ++++++++++++++++++-- 6 files changed, 318 insertions(+), 8 deletions(-) create mode 100644 dev-tools/releng/src/hubris.rs diff --git a/Cargo.lock b/Cargo.lock index e39fb836c3..30f713e2b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5603,16 +5603,22 @@ dependencies = [ "clap", "fs-err", "futures", + "num_cpus", + "omicron-common", "omicron-workspace-hack", "omicron-zone-package", "once_cell", + "reqwest", "semver 1.0.22", + "serde", "shell-words", "slog", "slog-async", "slog-term", "tar", "tokio", + "toml 0.8.12", + "tufaceous-lib", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index da03a6e792..4cfa231365 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -338,6 +338,7 @@ nexus-test-utils = { path = "nexus/test-utils" } nexus-types = { path = "nexus/types" } num-integer = "0.1.46" num = { version = "0.4.2", default-features = false, features = [ "libm" ] } +num_cpus = "1.16.0" omicron-common = { path = "common" } omicron-gateway = { path = "gateway" } omicron-nexus = { path = "nexus" } diff --git a/dev-tools/releng/Cargo.toml b/dev-tools/releng/Cargo.toml index 25a7b09cdf..9e57db3e92 100644 --- a/dev-tools/releng/Cargo.toml +++ b/dev-tools/releng/Cargo.toml @@ -12,16 +12,22 @@ chrono.workspace = true clap.workspace = true fs-err = { workspace = true, features = ["tokio"] } futures.workspace = true +num_cpus.workspace = true +omicron-common.workspace = true omicron-workspace-hack.workspace = true omicron-zone-package.workspace = true once_cell.workspace = true +reqwest.workspace = true semver.workspace = true +serde.workspace = true shell-words.workspace = true slog.workspace = true slog-async.workspace = true slog-term.workspace = true tar.workspace = true tokio = { workspace = true, features = ["full"] } +toml.workspace = true +tufaceous-lib.workspace = true [lints] workspace = true diff --git a/dev-tools/releng/src/hubris.rs b/dev-tools/releng/src/hubris.rs new file mode 100644 index 0000000000..89a680214b --- /dev/null +++ b/dev-tools/releng/src/hubris.rs @@ -0,0 +1,180 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::collections::HashMap; + +use anyhow::Context; +use anyhow::Result; +use camino::Utf8PathBuf; +use fs_err::tokio as fs; +use fs_err::tokio::File; +use futures::future::TryFutureExt; +use futures::stream::FuturesUnordered; +use futures::stream::TryStreamExt; +use omicron_common::api::external::SemverVersion; +use omicron_common::api::internal::nexus::KnownArtifactKind; +use semver::Version; +use serde::Deserialize; +use serde::Serialize; +use tokio::io::AsyncWriteExt; +use tufaceous_lib::assemble::DeserializedArtifactData; +use tufaceous_lib::assemble::DeserializedArtifactSource; +use tufaceous_lib::assemble::DeserializedFileArtifactSource; + +async fn fetch_one( + base_url: &'static str, + client: reqwest::Client, + hash: &str, +) -> Result> { + client + .get(format!("{}/artifact/{}", base_url, hash)) + .send() + .and_then(|response| response.json()) + .await + .with_context(|| { + format!( + "failed to fetch hubris artifact {} from {}", + hash, base_url + ) + }) +} + +pub(crate) async fn fetch_hubris_artifacts( + base_url: &'static str, + client: reqwest::Client, + manifest_list: Utf8PathBuf, + output_dir: Utf8PathBuf, +) -> Result<()> { + fs::create_dir_all(&output_dir).await?; + + let (manifests, hashes) = fs::read_to_string(manifest_list) + .await? + .lines() + .filter_map(|line| line.split_whitespace().next()) + .map(|hash| { + let hash = hash.to_owned(); + let client = client.clone(); + async move { + let data = fetch_one(base_url, client, &hash).await?; + let str = String::from_utf8(data) + .context("hubris artifact manifest was not UTF-8")?; + let hash_manifest: Manifest = toml::from_str(&str) + .context( + "failed to deserialize hubris artifact manifest", + )?; + + let mut hashes = Vec::new(); + for artifact in hash_manifest.artifacts.values().flatten() { + match &artifact.source { + Source::File(file) => hashes.push(file.hash.clone()), + Source::CompositeRot { archive_a, archive_b } => hashes + .extend([ + archive_a.hash.clone(), + archive_b.hash.clone(), + ]), + } + } + + let path_manifest: Manifest = + hash_manifest.into(); + anyhow::Ok((path_manifest, hashes)) + } + }) + .collect::>() + .try_collect::<(Vec<_>, Vec<_>)>() + .await?; + + let mut output_manifest = + File::create(output_dir.join("manifest.toml")).await?; + for manifest in manifests { + output_manifest + .write_all(toml::to_string_pretty(&manifest)?.as_bytes()) + .await?; + } + + hashes + .into_iter() + .flatten() + .map(|hash| { + let client = client.clone(); + let output_dir = output_dir.clone(); + async move { + let data = fetch_one(base_url, client, &hash).await?; + fs::write(output_dir.join(hash).with_extension("zip"), data) + .await?; + anyhow::Ok(()) + } + }) + .collect::>() + .try_collect::<()>() + .await?; + + output_manifest.sync_data().await?; + Ok(()) +} + +#[derive(Serialize, Deserialize)] +struct Manifest { + #[serde(rename = "artifact")] + artifacts: HashMap>, +} + +#[derive(Deserialize)] +struct Artifact { + name: String, + version: Version, + source: Source, +} + +#[derive(Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case")] +enum Source { + File(FileSource), + CompositeRot { archive_a: FileSource, archive_b: FileSource }, +} + +#[derive(Deserialize)] +struct FileSource { + hash: String, +} + +impl From> for Manifest { + fn from( + manifest: Manifest, + ) -> Manifest { + fn zip(hash: String) -> Utf8PathBuf { + Utf8PathBuf::from(hash).with_extension("zip") + } + + let mut artifacts = HashMap::new(); + for (kind, old_data) in manifest.artifacts { + let mut new_data = Vec::new(); + for artifact in old_data { + let source = match artifact.source { + Source::File(file) => DeserializedArtifactSource::File { + path: zip(file.hash), + }, + Source::CompositeRot { archive_a, archive_b } => { + DeserializedArtifactSource::CompositeRot { + archive_a: DeserializedFileArtifactSource::File { + path: zip(archive_a.hash), + }, + archive_b: DeserializedFileArtifactSource::File { + path: zip(archive_b.hash), + }, + } + } + }; + new_data.push(DeserializedArtifactData { + name: artifact.name, + version: SemverVersion(artifact.version), + source, + }); + } + artifacts.insert(kind, new_data); + } + + Manifest { artifacts } + } +} diff --git a/dev-tools/releng/src/job.rs b/dev-tools/releng/src/job.rs index acbac3dd8d..aa00c316f4 100644 --- a/dev-tools/releng/src/job.rs +++ b/dev-tools/releng/src/job.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::future::Future; use std::pin::Pin; use std::process::Stdio; +use std::sync::Arc; use std::time::Instant; use anyhow::anyhow; @@ -26,11 +27,13 @@ use tokio::io::BufReader; use tokio::process::Command; use tokio::sync::oneshot; use tokio::sync::oneshot::error::RecvError; +use tokio::sync::Semaphore; use crate::cmd::CommandExt; pub(crate) struct Jobs { logger: Logger, + permits: Arc, log_dir: Utf8PathBuf, map: HashMap, } @@ -47,9 +50,14 @@ pub(crate) struct Selector<'a> { } impl Jobs { - pub(crate) fn new(logger: &Logger, log_dir: &Utf8Path) -> Jobs { + pub(crate) fn new( + logger: &Logger, + permits: Arc, + log_dir: &Utf8Path, + ) -> Jobs { Jobs { logger: logger.clone(), + permits, log_dir: log_dir.to_owned(), map: HashMap::new(), } @@ -70,6 +78,7 @@ impl Jobs { Job { future: Box::pin(run_job( self.logger.clone(), + self.permits.clone(), name.clone(), future, )), @@ -95,6 +104,7 @@ impl Jobs { // returning &mut std::mem::replace(command, Command::new("false")), self.logger.clone(), + self.permits.clone(), name.clone(), self.log_dir.join(&name).with_extension("log"), )), @@ -167,10 +177,17 @@ macro_rules! info_or_error { }; } -async fn run_job(logger: Logger, name: String, future: F) -> Result<()> +async fn run_job( + logger: Logger, + permits: Arc, + name: String, + future: F, +) -> Result<()> where F: Future> + 'static, { + let _ = permits.acquire_owned().await?; + info!(logger, "[{}] running task", name); let start = Instant::now(); let result = future.await; @@ -189,9 +206,12 @@ where async fn spawn_with_output( mut command: Command, logger: Logger, + permits: Arc, name: String, log_path: Utf8PathBuf, ) -> Result<()> { + let _ = permits.acquire_owned().await?; + let log_file_1 = File::create(log_path).await?; let log_file_2 = log_file_1.try_clone().await?; diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index b33498ef36..7e66e581dd 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -3,9 +3,11 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. mod cmd; +mod hubris; mod job; use std::sync::Arc; +use std::time::Duration; use std::time::Instant; use anyhow::bail; @@ -26,6 +28,7 @@ use slog::Logger; use slog_term::FullFormat; use slog_term::TermDecorator; use tokio::process::Command; +use tokio::sync::Semaphore; use crate::cmd::CommandExt; use crate::job::Jobs; @@ -65,9 +68,22 @@ const RECOVERY_IMAGE_PACKAGES: [(&str, InstallMethod); 2] = [ ("installinator", InstallMethod::Install), ("mg-ddm-gz", InstallMethod::Install), ]; +/// Packages to ship with the TUF repo. +const TUF_PACKAGES: [&str; 11] = [ + "clickhouse_keeper", + "clickhouse", + "cockroachdb", + "crucible-pantry-zone", + "crucible-zone", + "external-dns", + "internal-dns", + "nexus", + "ntp", + "oximeter", + "probe", +]; const HELIOS_REPO: &str = "https://pkg.oxide.computer/helios/2/dev/"; -const OPTE_VERSION: &str = include_str!("../../../tools/opte_version"); static WORKSPACE_DIR: Lazy = Lazy::new(|| { // $CARGO_MANIFEST_DIR is at `.../omicron/dev-tools/releng` @@ -163,6 +179,8 @@ fn main() -> Result<()> { #[tokio::main] async fn do_run(logger: Logger, args: Args) -> Result<()> { + let permits = Arc::new(Semaphore::new(num_cpus::get())); + let commit = Command::new("git") .args(["rev-parse", "HEAD"]) .ensure_stdout(&logger) @@ -186,6 +204,14 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { &fs::read_to_string(WORKSPACE_DIR.join("package-manifest.toml")) .await?, )?); + let opte_version = + fs::read_to_string(WORKSPACE_DIR.join("tools/opte_version")).await?; + + let client = reqwest::ClientBuilder::new() + .connect_timeout(Duration::from_secs(15)) + .timeout(Duration::from_secs(15)) + .build() + .context("failed to build reqwest client")?; // PREFLIGHT ============================================================== let mut preflight_ok = true; @@ -289,7 +315,7 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { // DEFINE JOBS ============================================================ let tempdir = camino_tempfile::tempdir() .context("failed to create temporary directory")?; - let mut jobs = Jobs::new(&logger, &args.output_dir); + let mut jobs = Jobs::new(&logger, permits.clone(), &args.output_dir); jobs.push_command( "helios-setup", @@ -346,6 +372,20 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { ) .after(concat!($target_name, "-target")); + jobs.push( + concat!($target_name, "-stamp"), + stamp_packages( + logger.clone(), + permits.clone(), + args.output_dir.clone(), + omicron_package.clone(), + $target_name, + version.clone(), + $proto_packages.iter().map(|(name, _)| *name), + ), + ) + .after(concat!($target_name, "-package")); + let proto_dir = tempdir.path().join("proto").join($target_name); jobs.push( concat!($target_name, "-proto"), @@ -356,7 +396,7 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { manifest.clone(), ), ) - .after(concat!($target_name, "-package")); + .after(concat!($target_name, "-stamp")); // The ${os_short_commit} token will be expanded by `helios-build` let image_name = format!( @@ -373,11 +413,11 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { .arg(args.helios_dir.join("helios-build")) .arg("experiment-image") .arg("-o") // output directory for image - .arg(args.output_dir.join($target_name)) + .arg(args.output_dir.join(concat!("os-", $target_name))) .arg("-p") // use an external package repository .arg(format!("helios-dev={}", HELIOS_REPO)) .arg("-F") // pass extra image builder features - .arg(format!("optever={}", OPTE_VERSION.trim())) + .arg(format!("optever={}", opte_version.trim())) .arg("-P") // include all files from extra proto area .arg(proto_dir.join("root")) .arg("-N") // image name @@ -418,7 +458,6 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { image_build_args: ["-R"], image_dataset: args.recovery_dataset, } - // Build the recovery target after we build the host target. Only one // of these will build at a time since Cargo locks its target directory; // since host-package and host-image both take longer than their recovery @@ -429,6 +468,39 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { jobs.select("recovery-image").after("host-image"); } + jobs.push( + "tuf-stamp", + stamp_packages( + logger.clone(), + permits.clone(), + args.output_dir.clone(), + omicron_package.clone(), + "host", + version.clone(), + TUF_PACKAGES.into_iter(), + ), + ) + .after("host-proto"); + + jobs.push( + "hubris-staging", + hubris::fetch_hubris_artifacts( + "https://permslip-staging.corp.oxide.computer", + client.clone(), + WORKSPACE_DIR.join("tools/permslip_staging"), + args.output_dir.join("hubris-staging"), + ), + ); + jobs.push( + "hubris-production", + hubris::fetch_hubris_artifacts( + "https://signer-us-west.corp.oxide.computer", + client.clone(), + WORKSPACE_DIR.join("tools/permslip_production"), + args.output_dir.join("hubris-production"), + ), + ); + // RUN JOBS =============================================================== let start = Instant::now(); jobs.run_all().await?; @@ -448,6 +520,31 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { Ok(()) } +async fn stamp_packages( + logger: Logger, + permits: Arc, + output_dir: Utf8PathBuf, + omicron_package: Utf8PathBuf, + target_name: &'static str, + version: Version, + packages: impl Iterator, +) -> Result<()> { + let version = version.to_string(); + let mut jobs = Jobs::new(&logger, permits, &output_dir); + for package in packages { + jobs.push_command( + format!("stamp-{}", package), + Command::new(&omicron_package) + .arg("--target") + .arg(target_name) + .arg("stamp") + .arg(package) + .arg(&version), + ); + } + jobs.run_all().await +} + async fn build_proto_area( package_dir: Utf8PathBuf, proto_dir: Utf8PathBuf, From 5dcbb21db92f373f2af0311241780c1d155cb290 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Sun, 12 May 2024 20:06:12 +0000 Subject: [PATCH 06/35] tuf repo! --- Cargo.lock | 2 + dev-tools/releng/Cargo.toml | 2 + dev-tools/releng/src/hubris.rs | 213 +++++++++++-------------- dev-tools/releng/src/main.rs | 58 ++++--- dev-tools/releng/src/tuf.rs | 149 +++++++++++++++++ tufaceous-lib/src/assemble/manifest.rs | 13 +- 6 files changed, 283 insertions(+), 154 deletions(-) create mode 100644 dev-tools/releng/src/tuf.rs diff --git a/Cargo.lock b/Cargo.lock index 30f713e2b5..01a66c631f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5603,6 +5603,7 @@ dependencies = [ "clap", "fs-err", "futures", + "hex", "num_cpus", "omicron-common", "omicron-workspace-hack", @@ -5611,6 +5612,7 @@ dependencies = [ "reqwest", "semver 1.0.22", "serde", + "sha2", "shell-words", "slog", "slog-async", diff --git a/dev-tools/releng/Cargo.toml b/dev-tools/releng/Cargo.toml index 9e57db3e92..1e6d2df2da 100644 --- a/dev-tools/releng/Cargo.toml +++ b/dev-tools/releng/Cargo.toml @@ -12,6 +12,7 @@ chrono.workspace = true clap.workspace = true fs-err = { workspace = true, features = ["tokio"] } futures.workspace = true +hex.workspace = true num_cpus.workspace = true omicron-common.workspace = true omicron-workspace-hack.workspace = true @@ -20,6 +21,7 @@ once_cell.workspace = true reqwest.workspace = true semver.workspace = true serde.workspace = true +sha2.workspace = true shell-words.workspace = true slog.workspace = true slog-async.workspace = true diff --git a/dev-tools/releng/src/hubris.rs b/dev-tools/releng/src/hubris.rs index 89a680214b..4452249917 100644 --- a/dev-tools/releng/src/hubris.rs +++ b/dev-tools/releng/src/hubris.rs @@ -2,29 +2,108 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::collections::BTreeMap; use std::collections::HashMap; use anyhow::Context; use anyhow::Result; use camino::Utf8PathBuf; use fs_err::tokio as fs; -use fs_err::tokio::File; use futures::future::TryFutureExt; -use futures::stream::FuturesUnordered; -use futures::stream::TryStreamExt; use omicron_common::api::external::SemverVersion; use omicron_common::api::internal::nexus::KnownArtifactKind; use semver::Version; use serde::Deserialize; -use serde::Serialize; -use tokio::io::AsyncWriteExt; use tufaceous_lib::assemble::DeserializedArtifactData; use tufaceous_lib::assemble::DeserializedArtifactSource; use tufaceous_lib::assemble::DeserializedFileArtifactSource; +use tufaceous_lib::assemble::DeserializedManifest; -async fn fetch_one( +pub(crate) async fn fetch_hubris_artifacts( base_url: &'static str, client: reqwest::Client, + manifest_list: Utf8PathBuf, + output_dir: Utf8PathBuf, +) -> Result<()> { + macro_rules! zip { + ($expr:expr) => { + output_dir.join(format!("{}.zip", $expr)) + }; + } + + fs::create_dir_all(&output_dir).await?; + + // This could be parallelized with FuturesUnordered but in practice this + // takes less time than OS builds. + + let mut manifest = DeserializedManifest { + system_version: SemverVersion(Version::new(0, 0, 0)), + artifacts: BTreeMap::new(), + }; + + for line in fs::read_to_string(manifest_list).await?.lines() { + if let Some(hash) = line.split_whitespace().next() { + let data = fetch_hash(base_url, &client, hash).await?; + let str = String::from_utf8(data).with_context(|| { + format!("hubris artifact manifest {} was not UTF-8", hash) + })?; + let hash_manifest: Manifest = + toml::from_str(&str).with_context(|| { + format!( + "failed to deserialize hubris artifact manifest {}", + hash + ) + })?; + for (kind, artifacts) in hash_manifest.artifacts { + for artifact in artifacts { + let (source, hashes) = match artifact.source { + Source::File(file) => ( + DeserializedArtifactSource::File { + path: zip!(file.hash), + }, + vec![file.hash], + ), + Source::CompositeRot { archive_a, archive_b } => ( + DeserializedArtifactSource::CompositeRot { + archive_a: + DeserializedFileArtifactSource::File { + path: zip!(archive_a.hash), + }, + archive_b: + DeserializedFileArtifactSource::File { + path: zip!(archive_b.hash), + }, + }, + vec![archive_a.hash, archive_b.hash], + ), + }; + manifest.artifacts.entry(kind).or_default().push( + DeserializedArtifactData { + name: artifact.name, + version: artifact.version, + source, + }, + ); + for hash in hashes { + let data = fetch_hash(base_url, &client, &hash).await?; + fs::write(output_dir.join(zip!(hash)), data).await?; + } + } + } + } + } + + fs::write( + output_dir.join("manifest.toml"), + toml::to_string_pretty(&manifest)?.into_bytes(), + ) + .await?; + Ok(()) +} + +async fn fetch_hash( + base_url: &'static str, + client: &reqwest::Client, hash: &str, ) -> Result> { client @@ -40,90 +119,16 @@ async fn fetch_one( }) } -pub(crate) async fn fetch_hubris_artifacts( - base_url: &'static str, - client: reqwest::Client, - manifest_list: Utf8PathBuf, - output_dir: Utf8PathBuf, -) -> Result<()> { - fs::create_dir_all(&output_dir).await?; - - let (manifests, hashes) = fs::read_to_string(manifest_list) - .await? - .lines() - .filter_map(|line| line.split_whitespace().next()) - .map(|hash| { - let hash = hash.to_owned(); - let client = client.clone(); - async move { - let data = fetch_one(base_url, client, &hash).await?; - let str = String::from_utf8(data) - .context("hubris artifact manifest was not UTF-8")?; - let hash_manifest: Manifest = toml::from_str(&str) - .context( - "failed to deserialize hubris artifact manifest", - )?; - - let mut hashes = Vec::new(); - for artifact in hash_manifest.artifacts.values().flatten() { - match &artifact.source { - Source::File(file) => hashes.push(file.hash.clone()), - Source::CompositeRot { archive_a, archive_b } => hashes - .extend([ - archive_a.hash.clone(), - archive_b.hash.clone(), - ]), - } - } - - let path_manifest: Manifest = - hash_manifest.into(); - anyhow::Ok((path_manifest, hashes)) - } - }) - .collect::>() - .try_collect::<(Vec<_>, Vec<_>)>() - .await?; - - let mut output_manifest = - File::create(output_dir.join("manifest.toml")).await?; - for manifest in manifests { - output_manifest - .write_all(toml::to_string_pretty(&manifest)?.as_bytes()) - .await?; - } - - hashes - .into_iter() - .flatten() - .map(|hash| { - let client = client.clone(); - let output_dir = output_dir.clone(); - async move { - let data = fetch_one(base_url, client, &hash).await?; - fs::write(output_dir.join(hash).with_extension("zip"), data) - .await?; - anyhow::Ok(()) - } - }) - .collect::>() - .try_collect::<()>() - .await?; - - output_manifest.sync_data().await?; - Ok(()) -} - -#[derive(Serialize, Deserialize)] -struct Manifest { +#[derive(Deserialize)] +struct Manifest { #[serde(rename = "artifact")] - artifacts: HashMap>, + artifacts: HashMap>, } #[derive(Deserialize)] struct Artifact { name: String, - version: Version, + version: SemverVersion, source: Source, } @@ -138,43 +143,3 @@ enum Source { struct FileSource { hash: String, } - -impl From> for Manifest { - fn from( - manifest: Manifest, - ) -> Manifest { - fn zip(hash: String) -> Utf8PathBuf { - Utf8PathBuf::from(hash).with_extension("zip") - } - - let mut artifacts = HashMap::new(); - for (kind, old_data) in manifest.artifacts { - let mut new_data = Vec::new(); - for artifact in old_data { - let source = match artifact.source { - Source::File(file) => DeserializedArtifactSource::File { - path: zip(file.hash), - }, - Source::CompositeRot { archive_a, archive_b } => { - DeserializedArtifactSource::CompositeRot { - archive_a: DeserializedFileArtifactSource::File { - path: zip(archive_a.hash), - }, - archive_b: DeserializedFileArtifactSource::File { - path: zip(archive_b.hash), - }, - } - } - }; - new_data.push(DeserializedArtifactData { - name: artifact.name, - version: SemverVersion(artifact.version), - source, - }); - } - artifacts.insert(kind, new_data); - } - - Manifest { artifacts } - } -} diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 7e66e581dd..9b19ec213e 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -5,6 +5,7 @@ mod cmd; mod hubris; mod job; +mod tuf; use std::sync::Arc; use std::time::Duration; @@ -216,9 +217,11 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { // PREFLIGHT ============================================================== let mut preflight_ok = true; - for (package, _) in HOST_IMAGE_PACKAGES + for package in HOST_IMAGE_PACKAGES .into_iter() - .chain(RECOVERY_IMAGE_PACKAGES.into_iter()) + .chain(RECOVERY_IMAGE_PACKAGES) + .map(|(package, _)| package) + .chain(TUF_PACKAGES) { if !manifest.packages.contains_key(package) { error!( @@ -482,24 +485,35 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { ) .after("host-proto"); + for (name, base_url) in [ + ("staging", "https://permslip-staging.corp.oxide.computer"), + ("production", "https://signer-us-west.corp.oxide.computer"), + ] { + jobs.push( + format!("hubris-{}", name), + hubris::fetch_hubris_artifacts( + base_url, + client.clone(), + WORKSPACE_DIR.join(format!("tools/permslip_{}", name)), + args.output_dir.join(format!("hubris-{}", name)), + ), + ); + } + jobs.push( - "hubris-staging", - hubris::fetch_hubris_artifacts( - "https://permslip-staging.corp.oxide.computer", - client.clone(), - WORKSPACE_DIR.join("tools/permslip_staging"), - args.output_dir.join("hubris-staging"), - ), - ); - jobs.push( - "hubris-production", - hubris::fetch_hubris_artifacts( - "https://signer-us-west.corp.oxide.computer", - client.clone(), - WORKSPACE_DIR.join("tools/permslip_production"), - args.output_dir.join("hubris-production"), + "tuf-repo", + tuf::build_tuf_repo( + logger.clone(), + args.output_dir.clone(), + version.clone(), + manifest.clone(), ), - ); + ) + .after("tuf-stamp") + .after("host-image") + .after("recovery-image") + .after("hubris-staging") + .after("hubris-production"); // RUN JOBS =============================================================== let start = Instant::now(); @@ -509,14 +523,6 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { "all jobs completed in {:?}", Instant::now().saturating_duration_since(start) ); - - // fs::create_dir_all(host_proto.path().join("root/root"))?; - // fs::write( - // host_proto.path().join("root/root/.profile"), - // "# Add opteadm, ddadm, oxlog to PATH\n\ - // export PATH=$PATH:/opt/oxide/opte/bin:/opt/oxide/mg-ddm:/opt/oxide/oxlog\n" - // )?; - Ok(()) } diff --git a/dev-tools/releng/src/tuf.rs b/dev-tools/releng/src/tuf.rs new file mode 100644 index 0000000000..dc7d7cd46f --- /dev/null +++ b/dev-tools/releng/src/tuf.rs @@ -0,0 +1,149 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::sync::Arc; + +use anyhow::Context; +use anyhow::Result; +use camino::Utf8PathBuf; +use chrono::Duration; +use chrono::Timelike; +use chrono::Utc; +use fs_err::tokio as fs; +use fs_err::tokio::File; +use omicron_common::api::external::SemverVersion; +use omicron_common::api::internal::nexus::KnownArtifactKind; +use omicron_zone_package::config::Config; +use semver::Version; +use sha2::Digest; +use sha2::Sha256; +use slog::Logger; +use tokio::io::AsyncReadExt; +use tufaceous_lib::assemble::ArtifactManifest; +use tufaceous_lib::assemble::DeserializedArtifactData; +use tufaceous_lib::assemble::DeserializedArtifactSource; +use tufaceous_lib::assemble::DeserializedControlPlaneZoneSource; +use tufaceous_lib::assemble::DeserializedManifest; +use tufaceous_lib::assemble::OmicronRepoAssembler; +use tufaceous_lib::Key; + +pub(crate) async fn build_tuf_repo( + logger: Logger, + output_dir: Utf8PathBuf, + version: Version, + package_manifest: Arc, +) -> Result<()> { + // We currently go about this somewhat strangely; the old release + // engineering process produced a Tufaceous manifest, and (the now very many + // copies of) the TUF repo download-and-unpack script we use expects to be + // able to download a manifest. So we build up a `DeserializedManifest`, + // write it to disk, and then turn it into an `ArtifactManifest` to actually + // build the repo. + + // Start a new manifest by loading the Hubris staging manifest. + let mut manifest = DeserializedManifest::from_path( + &output_dir.join("hubris-staging/manifest.toml"), + ) + .context("failed to open intermediate hubris staging manifest")?; + // Set the version. + manifest.system_version = SemverVersion(version); + + // Load the Hubris production manifest and merge it in. + let hubris_production = DeserializedManifest::from_path( + &output_dir.join("hubris-production/manifest.toml"), + ) + .context("failed to open intermediate hubris production manifest")?; + for (kind, artifacts) in hubris_production.artifacts { + manifest.artifacts.entry(kind).or_default().extend(artifacts); + } + + // Add the OS images. + manifest.artifacts.insert( + KnownArtifactKind::Host, + vec![DeserializedArtifactData { + name: "host".to_string(), + version: manifest.system_version.clone(), + source: DeserializedArtifactSource::File { + path: output_dir.join("os-host/os.tar.gz"), + }, + }], + ); + manifest.artifacts.insert( + KnownArtifactKind::Trampoline, + vec![DeserializedArtifactData { + name: "trampoline".to_string(), + version: manifest.system_version.clone(), + source: DeserializedArtifactSource::File { + path: output_dir.join("os-recovery/os.tar.gz"), + }, + }], + ); + + // Add the control plane zones. + let mut zones = Vec::new(); + for package in crate::TUF_PACKAGES { + zones.push(DeserializedControlPlaneZoneSource::File { + file_name: Some(format!( + "{}.tar.gz", + package_manifest + .packages + .get(package) + .expect("checked in preflight") + .service_name + )), + path: crate::WORKSPACE_DIR + .join("out") + .join(format!("{}.tar.gz", package)), + }); + } + manifest.artifacts.insert( + KnownArtifactKind::ControlPlane, + vec![DeserializedArtifactData { + name: "control-plane".to_string(), + version: manifest.system_version.clone(), + source: DeserializedArtifactSource::CompositeControlPlane { zones }, + }], + ); + + // Serialize the manifest out. + fs::write( + output_dir.join("manifest.toml"), + toml::to_string_pretty(&manifest)?.into_bytes(), + ) + .await?; + + // Convert the manifest. + let manifest = ArtifactManifest::from_deserialized(&output_dir, manifest)?; + manifest.verify_all_present()?; + // Assemble the repo. + let keys = vec![Key::generate_ed25519()]; + let expiry = Utc::now().with_nanosecond(0).unwrap() + Duration::weeks(1); + OmicronRepoAssembler::new( + &logger, + manifest, + keys, + expiry, + output_dir.join("repo.zip"), + ) + .build() + .await?; + // Generate the checksum file. + let mut hasher = Sha256::new(); + let mut buf = [0; 8192]; + let mut file = File::open(output_dir.join("repo.zip")).await?; + loop { + let n = file.read(&mut buf).await?; + if n == 0 { + break; + } + hasher.update(&buf); + } + fs::write( + output_dir.join("repo.zip.sha256.txt"), + format!("{}\n", hex::encode(&hasher.finalize())), + ) + .await?; + + Ok(()) +} diff --git a/tufaceous-lib/src/assemble/manifest.rs b/tufaceous-lib/src/assemble/manifest.rs index 8825327c1d..1c4a676f4c 100644 --- a/tufaceous-lib/src/assemble/manifest.rs +++ b/tufaceous-lib/src/assemble/manifest.rs @@ -524,6 +524,8 @@ impl DeserializedFileArtifactSource { pub enum DeserializedControlPlaneZoneSource { File { path: Utf8PathBuf, + #[serde(skip_serializing_if = "Option::is_none")] + file_name: Option, }, Fake { name: String, @@ -542,12 +544,15 @@ impl DeserializedControlPlaneZoneSource { F: FnOnce(&str, CompositeEntry<'_>) -> Result, { let (name, data, mtime_source) = match self { - DeserializedControlPlaneZoneSource::File { path } => { + DeserializedControlPlaneZoneSource::File { path, file_name } => { let data = std::fs::read(path) .with_context(|| format!("failed to read {path}"))?; - let name = path.file_name().with_context(|| { - format!("zone path missing file name: {path}") - })?; + let name = file_name + .as_deref() + .or_else(|| path.file_name()) + .with_context(|| { + format!("zone path missing file name: {path}") + })?; // For now, always use the current time as the source. (Maybe // change this to use the mtime on disk in the future?) (name, data, MtimeSource::Now) From e6a0129bfbbeaf0089d6cb4c73c9af78949ce3a3 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Sun, 12 May 2024 21:48:17 +0000 Subject: [PATCH 07/35] will it blend? --- .github/buildomat/jobs/ci-tools.sh | 77 --------------- .github/buildomat/jobs/deploy.sh | 12 +-- .github/buildomat/jobs/host-image.sh | 93 ----------------- .github/buildomat/jobs/package.sh | 115 ++++----------------- .github/buildomat/jobs/tuf-repo.sh | 143 +++++++++------------------ dev-tools/releng/src/main.rs | 27 ++++- package/src/bin/omicron-package.rs | 25 ++++- package/src/lib.rs | 5 + tools/build-host-image.sh | 111 --------------------- tools/permslip_commit | 1 - 10 files changed, 119 insertions(+), 490 deletions(-) delete mode 100755 .github/buildomat/jobs/ci-tools.sh delete mode 100755 .github/buildomat/jobs/host-image.sh delete mode 100755 tools/build-host-image.sh delete mode 100644 tools/permslip_commit diff --git a/.github/buildomat/jobs/ci-tools.sh b/.github/buildomat/jobs/ci-tools.sh deleted file mode 100755 index 4c58731e24..0000000000 --- a/.github/buildomat/jobs/ci-tools.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -#: -#: name = "helios / CI tools" -#: variety = "basic" -#: target = "helios-2.0" -#: rust_toolchain = "1.72.1" -#: output_rules = [ -#: "=/work/end-to-end-tests/*.gz", -#: "=/work/caboose-util.gz", -#: "=/work/tufaceous.gz", -#: "=/work/commtest", -#: "=/work/permslip.gz", -#: ] -#: access_repos = [ -#: "oxidecomputer/permission-slip", -#: "oxidecomputer/sshauth" -#: ] - -set -o errexit -set -o pipefail -set -o xtrace - -cargo --version -rustc --version - -ptime -m ./tools/install_builder_prerequisites.sh -yp - -########## end-to-end-tests ########## - -banner end-to-end-tests - -# -# Reduce debuginfo just to line tables. -# -export CARGO_PROFILE_DEV_DEBUG=1 -export CARGO_PROFILE_TEST_DEBUG=1 -export CARGO_INCREMENTAL=0 - -ptime -m cargo build --locked -p end-to-end-tests --tests --bin bootstrap \ - --message-format json-render-diagnostics >/tmp/output.end-to-end.json - -mkdir -p /work -ptime -m cargo build --locked -p end-to-end-tests --tests --bin commtest -cp target/debug/commtest /work/commtest - -mkdir -p /work/end-to-end-tests -for p in target/debug/bootstrap $(/opt/ooce/bin/jq -r 'select(.profile.test) | .executable' /tmp/output.end-to-end.json); do - # shellcheck disable=SC2094 - ptime -m gzip < "$p" > /work/end-to-end-tests/"$(basename "$p").gz" -done - -########## caboose-util ########## - -banner caboose-util - -ptime -m cargo build --locked -p caboose-util --release -ptime -m gzip < target/release/caboose-util > /work/caboose-util.gz - -########## tufaceous ########## - -banner tufaceous - -ptime -m cargo build --locked -p tufaceous --release -ptime -m gzip < target/release/tufaceous > /work/tufaceous.gz - -########## permission-slip ########## - -banner permission-slip - -source "./tools/permslip_commit" -git init /work/permission-slip-build -pushd /work/permission-slip-build -git remote add origin https://github.com/oxidecomputer/permission-slip.git -ptime -m git fetch --depth 1 origin "$COMMIT" -git checkout FETCH_HEAD -ptime -m cargo build --locked -p permission-slip-client --release -ptime -m gzip < target/release/permslip > /work/permslip.gz diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index 8d3e94cd5e..6f80b72f2d 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -20,8 +20,6 @@ #: [dependencies.package] #: job = "helios / package" #: -#: [dependencies.ci-tools] -#: job = "helios / CI tools" set -o errexit set -o pipefail @@ -144,13 +142,8 @@ pfexec chown build:build /opt/oxide/work cd /opt/oxide/work ptime -m tar xvzf /input/package/work/package.tar.gz -cp /input/package/work/zones/* out/ -mv out/nexus-single-sled.tar.gz out/nexus.tar.gz mkdir tests -for p in /input/ci-tools/work/end-to-end-tests/*.gz; do - ptime -m gunzip < "$p" > "tests/$(basename "${p%.gz}")" - chmod a+x "tests/$(basename "${p%.gz}")" -done +cp /input/package/work/target/debug/deps/* tests/ # Ask buildomat for the range of extra addresses that we're allowed to use, and # break them up into the ranges we need. @@ -354,7 +347,7 @@ echo "Waited for nexus: ${retry}s" export RUST_BACKTRACE=1 export E2E_TLS_CERT IPPOOL_START IPPOOL_END -eval "$(./tests/bootstrap)" +eval "$(./target/debug/bootstrap)" export OXIDE_HOST OXIDE_TOKEN # @@ -387,7 +380,6 @@ done /usr/oxide/oxide --resolve "$OXIDE_RESOLVE" --cacert "$E2E_TLS_CERT" \ image promote --project images --image debian11 -rm ./tests/bootstrap for test_bin in tests/*; do ./"$test_bin" done diff --git a/.github/buildomat/jobs/host-image.sh b/.github/buildomat/jobs/host-image.sh deleted file mode 100755 index 2f4d146a48..0000000000 --- a/.github/buildomat/jobs/host-image.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash -#: -#: name = "helios / build OS images" -#: variety = "basic" -#: target = "helios-2.0" -#: rust_toolchain = "1.72.1" -#: output_rules = [ -#: "=/work/helios/upload/os-host.tar.gz", -#: "=/work/helios/upload/os-trampoline.tar.gz", -#: ] -#: access_repos = [ -#: "oxidecomputer/amd-apcb", -#: "oxidecomputer/amd-efs", -#: "oxidecomputer/amd-firmware", -#: "oxidecomputer/amd-flash", -#: "oxidecomputer/amd-host-image-builder", -#: "oxidecomputer/boot-image-tools", -#: "oxidecomputer/chelsio-t6-roms", -#: "oxidecomputer/compliance-pilot", -#: "oxidecomputer/facade", -#: "oxidecomputer/helios", -#: "oxidecomputer/helios-omicron-brand", -#: "oxidecomputer/helios-omnios-build", -#: "oxidecomputer/helios-omnios-extra", -#: "oxidecomputer/nanobl-rs", -#: ] -#: -#: [dependencies.package] -#: job = "helios / package" -#: -#: [[publish]] -#: series = "image" -#: name = "os.tar.gz" -#: from_output = "/work/helios/image/output/os.tar.gz" -#: - -set -o errexit -set -o pipefail -set -o xtrace - -cargo --version -rustc --version - -TOP=$PWD - -source "$TOP/tools/include/force-git-over-https.sh" - -# Check out helios into /work/helios -HELIOSDIR=/work/helios -git clone https://github.com/oxidecomputer/helios.git "$HELIOSDIR" -cd "$HELIOSDIR" -# Record the branch and commit in the output -git status --branch --porcelain=2 -# Setting BUILD_OS to no makes setup skip repositories we don't need for -# building the OS itself (we are just building an image from already built OS). -BUILD_OS=no gmake setup - -# Commands that "helios-build" would ask us to run (either explicitly or -# implicitly, to avoid an error). -rc=0 -pfexec pkg install -q /system/zones/brand/omicron1/tools || rc=$? -case $rc in - # `man pkg` notes that exit code 4 means no changes were made because - # there is nothing to do; that's fine. Any other exit code is an error. - 0 | 4) ;; - *) exit $rc ;; -esac - -pfexec zfs create -p "rpool/images/$USER" - - -# TODO: Consider importing zones here too? - -cd "$TOP" -OUTPUTDIR="$HELIOSDIR/upload" -mkdir "$OUTPUTDIR" - -banner OS -./tools/build-host-image.sh -B \ - -S /input/package/work/zones/switch-asic.tar.gz \ - "$HELIOSDIR" \ - /input/package/work/global-zone-packages.tar.gz - -mv "$HELIOSDIR/image/output/os.tar.gz" "$OUTPUTDIR/os-host.tar.gz" - -banner Trampoline - -./tools/build-host-image.sh -R \ - "$HELIOSDIR" \ - /input/package/work/trampoline-global-zone-packages.tar.gz - -mv "$HELIOSDIR/image/output/os.tar.gz" "$OUTPUTDIR/os-trampoline.tar.gz" - diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index 11a5a1a0ee..eedfb0dda3 100755 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -3,24 +3,11 @@ #: name = "helios / package" #: variety = "basic" #: target = "helios-2.0" -#: rust_toolchain = "1.72.1" +#: rust_toolchain = "1.77.2" #: output_rules = [ -#: "=/work/version.txt", #: "=/work/package.tar.gz", -#: "=/work/global-zone-packages.tar.gz", -#: "=/work/trampoline-global-zone-packages.tar.gz", -#: "=/work/zones/*.tar.gz", #: ] #: -#: [[publish]] -#: series = "image" -#: name = "global-zone-packages" -#: from_output = "/work/global-zone-packages.tar.gz" -#: -#: [[publish]] -#: series = "image" -#: name = "trampoline-global-zone-packages" -#: from_output = "/work/trampoline-global-zone-packages.tar.gz" set -o errexit set -o pipefail @@ -32,17 +19,6 @@ rustc --version WORK=/work pfexec mkdir -p $WORK && pfexec chown $USER $WORK -# -# Generate the version for control plane artifacts here. We use `0.git` as the -# prerelease field because it comes before `alpha`. -# -# In this job, we stamp the version into packages installed in the host and -# trampoline global zone images. -# -COMMIT=$(git rev-parse HEAD) -VERSION="8.0.0-0.ci+git${COMMIT:0:11}" -echo "$VERSION" >/work/version.txt - ptime -m ./tools/install_builder_prerequisites.sh -yp ptime -m ./tools/ci_download_softnpu_machinery @@ -52,88 +28,33 @@ ptime -m cargo run --locked --release --bin omicron-package -- \ -t test target create -i standard -m non-gimlet -s softnpu -r single-sled ptime -m cargo run --locked --release --bin omicron-package -- \ -t test package +mapfile -t packages \ + < <(cargo run --locked --release --bin omicron-package -- -t test list-outputs) # Build the xtask binary used by the deploy job ptime -m cargo build --locked --release -p xtask -# Assemble some utilities into a tarball that can be used by deployment -# phases of buildomat. +# Build the end-to-end tests +# Reduce debuginfo just to line tables. +export CARGO_PROFILE_DEV_DEBUG=1 +export CARGO_PROFILE_TEST_DEBUG=1 +ptime -m cargo build --locked -p end-to-end-tests --tests --bin bootstrap \ + --message-format json-render-diagnostics >/tmp/output.end-to-end.json +mapfile -t test_bins \ + < <(/opt/ooce/bin/jq -r 'select(.profile.test) | .executable' /tmp/output.end-to-end.json \ + | cut -c$((${#PWD} + 2))-) + +# Assemble these outputs and some utilities into a tarball that can be used by +# deployment phases of buildomat. files=( - out/*.tar out/target/test out/npuzone/* package-manifest.toml smf/sled-agent/non-gimlet/config.toml target/release/omicron-package target/release/xtask + target/debug/bootstrap ) - -ptime -m tar cvzf $WORK/package.tar.gz "${files[@]}" - -tarball_src_dir="$(pwd)/out/versioned" -stamp_packages() { - for package in "$@"; do - cargo run --locked --release --bin omicron-package -- stamp "$package" "$VERSION" - done -} - -# Keep the single-sled Nexus zone around for the deploy job. (The global zone -# build below overwrites the file.) -mv out/nexus.tar.gz out/nexus-single-sled.tar.gz - -# Build necessary for the global zone -ptime -m cargo run --locked --release --bin omicron-package -- \ - -t host target create -i standard -m gimlet -s asic -r multi-sled -ptime -m cargo run --locked --release --bin omicron-package -- \ - -t host package -stamp_packages omicron-sled-agent mg-ddm-gz propolis-server overlay oxlog pumpkind-gz - -# Create global zone package @ $WORK/global-zone-packages.tar.gz -ptime -m ./tools/build-global-zone-packages.sh "$tarball_src_dir" $WORK - -# Non-Global Zones - -# Assemble Zone Images into their respective output locations. -# -# Zones that are included into another are intentionally omitted from this list -# (e.g., the switch zone tarballs contain several other zone tarballs: dendrite, -# mg-ddm, etc.). -# -# Note that when building for a real gimlet, `propolis-server` and `switch-*` -# should be included in the OS ramdisk. -mkdir -p $WORK/zones -zones=( - out/clickhouse.tar.gz - out/clickhouse_keeper.tar.gz - out/cockroachdb.tar.gz - out/crucible-pantry-zone.tar.gz - out/crucible-zone.tar.gz - out/external-dns.tar.gz - out/internal-dns.tar.gz - out/nexus.tar.gz - out/nexus-single-sled.tar.gz - out/oximeter.tar.gz - out/propolis-server.tar.gz - out/switch-*.tar.gz - out/ntp.tar.gz - out/omicron-gateway-softnpu.tar.gz - out/omicron-gateway-asic.tar.gz - out/overlay.tar.gz - out/probe.tar.gz -) -cp "${zones[@]}" $WORK/zones/ - -# -# Global Zone files for Trampoline image -# - -# Build necessary for the trampoline image -ptime -m cargo run --locked --release --bin omicron-package -- \ - -t recovery target create -i trampoline -ptime -m cargo run --locked --release --bin omicron-package -- \ - -t recovery package -stamp_packages installinator mg-ddm-gz - -# Create trampoline global zone package @ $WORK/trampoline-global-zone-packages.tar.gz -ptime -m ./tools/build-trampoline-global-zone-packages.sh "$tarball_src_dir" $WORK +ptime -m tar cvzf $WORK/package.tar.gz \ + "${files[@]}" "${packages[@]}" "${test_bins[@]}" diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh index 89928a0030..93e5668c75 100755 --- a/.github/buildomat/jobs/tuf-repo.sh +++ b/.github/buildomat/jobs/tuf-repo.sh @@ -3,20 +3,36 @@ #: name = "helios / build TUF repo" #: variety = "basic" #: target = "helios-2.0" +#: rust_toolchain = "1.77.2" #: output_rules = [ -#: "=/work/manifest*.toml", -#: "=/work/repo-*.zip", -#: "=/work/repo-*.zip.sha256.txt", +#: "%/work/*.log", +#: "=/work/manifest.toml", +#: "=/work/repo.zip", +#: "=/work/repo.zip.sha256.txt", +#: "=/work/os-host/os.tar.gz", +#: "=/work/os-recovery/os.tar.gz", +#: ] +#: access_repos = [ +#: "oxidecomputer/amd-apcb", +#: "oxidecomputer/amd-efs", +#: "oxidecomputer/amd-firmware", +#: "oxidecomputer/amd-flash", +#: "oxidecomputer/amd-host-image-builder", +#: "oxidecomputer/boot-image-tools", +#: "oxidecomputer/chelsio-t6-roms", +#: "oxidecomputer/compliance-pilot", +#: "oxidecomputer/facade", +#: "oxidecomputer/helios", +#: "oxidecomputer/helios-omicron-brand", +#: "oxidecomputer/helios-omnios-build", +#: "oxidecomputer/helios-omnios-extra", +#: "oxidecomputer/nanobl-rs", #: ] #: -#: [dependencies.ci-tools] -#: job = "helios / CI tools" -#: -#: [dependencies.package] -#: job = "helios / package" -#: -#: [dependencies.host] -#: job = "helios / build OS images" +#: [[publish]] +#: series = "image" +#: name = "os.tar.gz" +#: from_output = "/work/os-host/os.tar.gz" #: #: [[publish]] #: series = "rot-all" @@ -26,105 +42,34 @@ #: [[publish]] #: series = "rot-all" #: name = "repo.zip" -#: from_output = "/work/repo-rot-all.zip" +#: from_output = "/work/repo.zip" #: #: [[publish]] #: series = "rot-all" #: name = "repo.zip.sha256.txt" -#: from_output = "/work/repo-rot-all.zip.sha256.txt" +#: from_output = "/work/repo.zip.sha256.txt" #: set -o errexit set -o pipefail set -o xtrace -TOP=$PWD -VERSION=$(< /input/package/work/version.txt) - -for bin in caboose-util tufaceous permslip; do - ptime -m gunzip < /input/ci-tools/work/$bin.gz > /work/$bin - chmod a+x /work/$bin -done - -# -# We do two things here: -# 1. Run `omicron-package stamp` on all the zones. -# 2. Run `omicron-package unpack` to switch from "package-name.tar.gz" to "service_name.tar.gz". -# -mkdir /work/package -pushd /work/package -tar xf /input/package/work/package.tar.gz out package-manifest.toml target/release/omicron-package -target/release/omicron-package -t default target create -i standard -m gimlet -s asic -r multi-sled -ln -s /input/package/work/zones/* out/ -rm out/switch-softnpu.tar.gz # not used when target switch=asic -rm out/omicron-gateway-softnpu.tar.gz # not used when target switch=asic -rm out/nexus-single-sled.tar.gz # only used for deploy tests -for zone in out/*.tar.gz; do - target/release/omicron-package stamp "$(basename "${zone%.tar.gz}")" "$VERSION" -done -mv out/versioned/* out/ -OMICRON_NO_UNINSTALL=1 target/release/omicron-package unpack --out install -popd - -# Generate a throwaway repository key. -python3 -c 'import secrets; open("/work/key.txt", "w").write("ed25519:%s\n" % secrets.token_hex(32))' -read -r TUFACEOUS_KEY /work/manifest.toml <>/work/manifest.toml <>/work/manifest.toml <> /work/manifest.toml - done < $TOP/tools/permslip_$name - popd -} +rc=0 +pfexec pkg install -q /system/zones/brand/omicron1/tools || rc=$? +case $rc in + # `man pkg` notes that exit code 4 means no changes were made because + # there is nothing to do; that's fine. Any other exit code is an error. + 0 | 4) ;; + *) exit $rc ;; +esac -mkdir /work/hubris -pushd /work/hubris -download_region_manifests https://permslip-staging.corp.oxide.computer staging -download_region_manifests https://signer-us-west.corp.oxide.computer production -popd +pfexec zfs create -p "rpool/images/$USER/host" +pfexec zfs create -p "rpool/images/$USER/recovery" -/work/tufaceous assemble --no-generate-key /work/manifest.toml /work/repo-rot-all.zip -digest -a sha256 /work/repo-rot-all.zip > /work/repo-rot-all.zip.sha256.txt +cargo run --release --bin omicron-releng -- --output-dir /work diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 9b19ec213e..255ae08c3c 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -190,7 +190,8 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { .to_owned(); let mut version = BASE_VERSION.clone(); - // Differentiate between CI and local builds. + // Differentiate between CI and local builds. We use `0.word` as the + // prerelease field because it comes before `alpha`. version.pre = if std::env::var_os("CI").is_some() { "0.ci" } else { "0.local" } .parse()?; @@ -272,6 +273,13 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { .ensure_success(&logger) .await?; } + // Record the branch and commit in the output + Command::new("git") + .arg("-C") + .arg(&args.helios_dir) + .args(["status", "--branch", "--porcelain=2"]) + .ensure_success(&logger) + .await?; // Check that the omicron1 brand is installed if !Command::new("pkg") @@ -470,6 +478,13 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { // If the datasets are the same, we can't parallelize these. jobs.select("recovery-image").after("host-image"); } + // Set up /root/.profile in the host OS image. + jobs.push( + "host-profile", + host_add_root_profile(tempdir.path().join("proto/host/root/root")), + ) + .after("host-proto"); + jobs.select("host-image").after("host-profile"); jobs.push( "tuf-stamp", @@ -608,3 +623,13 @@ async fn build_proto_area( Ok(()) } + +async fn host_add_root_profile(host_proto_root: Utf8PathBuf) -> Result<()> { + fs::create_dir_all(&host_proto_root).await?; + fs::write( + host_proto_root.join(".profile"), + "# Add opteadm, ddadm, oxlog to PATH\n\ + export PATH=$PATH:/opt/oxide/opte/bin:/opt/oxide/mg-ddm:/opt/oxide/oxlog\n", + ).await?; + Ok(()) +} diff --git a/package/src/bin/omicron-package.rs b/package/src/bin/omicron-package.rs index 3b8bd24918..09fa7ab178 100644 --- a/package/src/bin/omicron-package.rs +++ b/package/src/bin/omicron-package.rs @@ -199,6 +199,25 @@ async fn do_dot(config: &Config) -> Result<()> { Ok(()) } +async fn do_list_outputs( + config: &Config, + output_directory: &Utf8Path, + intermediate: bool, +) -> Result<()> { + for (name, package) in + config.package_config.packages_to_build(&config.target).0 + { + if !intermediate + && package.output + == (PackageOutput::Zone { intermediate_only: true }) + { + continue; + } + println!("{}", package.get_output_path(name, output_directory)); + } + Ok(()) +} + // The name reserved for the currently-in-use build target. const ACTIVE: &str = "active"; @@ -919,7 +938,7 @@ async fn main() -> Result<()> { tokio::fs::create_dir_all(&args.artifact_dir).await?; let logpath = args.artifact_dir.join("LOG"); let logfile = std::io::LineWriter::new(open_options.open(&logpath)?); - println!("Logging to: {}", std::fs::canonicalize(logpath)?.display()); + eprintln!("Logging to: {}", std::fs::canonicalize(logpath)?.display()); let drain = slog_bunyan::new(logfile).build().fuse(); let drain = slog_async::Async::new(drain).build().fuse(); @@ -981,6 +1000,10 @@ async fn main() -> Result<()> { SubCommand::Build(BuildCommand::Dot) => { do_dot(&get_config()?).await?; } + SubCommand::Build(BuildCommand::ListOutputs { intermediate }) => { + do_list_outputs(&get_config()?, &args.artifact_dir, *intermediate) + .await?; + } SubCommand::Build(BuildCommand::Package { disable_cache }) => { do_package(&get_config()?, &args.artifact_dir, *disable_cache) .await?; diff --git a/package/src/lib.rs b/package/src/lib.rs index bba1a3a0cd..2b99cfbe07 100644 --- a/package/src/lib.rs +++ b/package/src/lib.rs @@ -90,6 +90,11 @@ pub enum BuildCommand { }, /// Make a `dot` graph to visualize the package tree Dot, + /// List the output packages for the current target + ListOutputs { + #[clap(long)] + intermediate: bool, + }, /// Builds the packages specified in a manifest, and places them into an /// 'out' directory. Package { diff --git a/tools/build-host-image.sh b/tools/build-host-image.sh deleted file mode 100755 index e90d800849..0000000000 --- a/tools/build-host-image.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o pipefail -set -o xtrace - -function usage -{ - echo "usage: $0 [-fRB] HELIOS_PATH PACKAGES_TARBALL" - echo - echo " -f Force helios build despite git hash mismatch" - echo " -R Build recovery (trampoline) image" - echo " -B Build standard image" - exit 1 -} - -function main -{ - while getopts ":hfRBS:" opt; do - case $opt in - f) - FORCE=1 - ;; - R) - BUILD_RECOVERY=1 - HELIOS_BUILD_EXTRA_ARGS=-R - IMAGE_PREFIX=recovery - ;; - B) - BUILD_STANDARD=1 - HELIOS_BUILD_EXTRA_ARGS=-B - IMAGE_PREFIX=ci - ;; - S) - SWITCH_ZONE=$OPTARG - ;; - h | \?) - usage - ;; - esac - done - shift $((OPTIND-1)) - - # Ensure we got either -R or -B but not both - case "x$BUILD_RECOVERY$BUILD_STANDARD" in - x11) - echo "specify at most one of -R, -B" - exit 1 - ;; - x) - echo "must specify either -R or -B" - exit 1 - ;; - *) ;; - esac - - if [ "$#" != "2" ]; then - usage - fi - HELIOS_PATH=$1 - GLOBAL_ZONE_TARBALL_PATH=$2 - - TOOLS_DIR="$(pwd)/$(dirname "$0")" - - # Grab the opte version - OPTE_VER=$(cat "$TOOLS_DIR/opte_version") - - # Assemble global zone files in a temporary directory. - if ! tmp_gz=$(mktemp -d); then - exit 1 - fi - trap 'cd /; rm -rf "$tmp_gz"' EXIT - - # Extract the global zone tarball into a tmp_gz directory - echo "Extracting gz packages into $tmp_gz" - ptime -m tar xvzf "$GLOBAL_ZONE_TARBALL_PATH" -C "$tmp_gz" - - # If the user specified a switch zone (which is probably named - # `switch-SOME_VARIANT.tar.gz`), stage it in the right place and rename it - # to just `switch.tar.gz`. - if [ "x$SWITCH_ZONE" != "x" ]; then - mkdir -p "$tmp_gz/root/opt/oxide" - cp "$SWITCH_ZONE" "$tmp_gz/root/opt/oxide/switch.tar.gz" - fi - - if [ "x$BUILD_STANDARD" != "x" ]; then - mkdir -p "$tmp_gz/root/root" - echo "# Add opteadm, ddmadm, oxlog to PATH" >> "$tmp_gz/root/root/.profile" - echo 'export PATH=$PATH:/opt/oxide/opte/bin:/opt/oxide/mg-ddm:/opt/oxide/oxlog' >> "$tmp_gz/root/root/.profile" - fi - - # Move to the helios checkout - cd "$HELIOS_PATH" - - HELIOS_REPO=https://pkg.oxide.computer/helios/2/dev/ - - # Build an image name that includes the omicron and host OS hashes - IMAGE_NAME="$IMAGE_PREFIX ${GITHUB_SHA:0:7}" - # The ${os_short_commit} token will be expanded by `helios-build` - IMAGE_NAME+='/${os_short_commit}' - IMAGE_NAME+=" $(date +'%Y-%m-%d %H:%M')" - - ./helios-build experiment-image \ - -p helios-dev="$HELIOS_REPO" \ - -F optever="$OPTE_VER" \ - -P "$tmp_gz/root" \ - -N "$IMAGE_NAME" \ - $HELIOS_BUILD_EXTRA_ARGS -} - -main "$@" diff --git a/tools/permslip_commit b/tools/permslip_commit deleted file mode 100644 index 58140df7da..0000000000 --- a/tools/permslip_commit +++ /dev/null @@ -1 +0,0 @@ -COMMIT=5d44e0065f90051a28881c75e3574142ada9b695 From 12d1cf052e94e337e63364bd4a1471f87c0920f4 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Sun, 12 May 2024 22:53:33 +0000 Subject: [PATCH 08/35] move logfiles to end of output_rules --- .github/buildomat/jobs/tuf-repo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh index 93e5668c75..396c410260 100755 --- a/.github/buildomat/jobs/tuf-repo.sh +++ b/.github/buildomat/jobs/tuf-repo.sh @@ -5,12 +5,12 @@ #: target = "helios-2.0" #: rust_toolchain = "1.77.2" #: output_rules = [ -#: "%/work/*.log", #: "=/work/manifest.toml", #: "=/work/repo.zip", #: "=/work/repo.zip.sha256.txt", #: "=/work/os-host/os.tar.gz", #: "=/work/os-recovery/os.tar.gz", +#: "%/work/*.log", #: ] #: access_repos = [ #: "oxidecomputer/amd-apcb", From 0219f87f3f31060b3b110480a15ee3bafc2a24ae Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 04:05:11 +0000 Subject: [PATCH 09/35] attempt to fix deploy job --- .github/buildomat/jobs/deploy.sh | 2 -- .github/buildomat/jobs/package.sh | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index 6f80b72f2d..c947a05e10 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -142,8 +142,6 @@ pfexec chown build:build /opt/oxide/work cd /opt/oxide/work ptime -m tar xvzf /input/package/work/package.tar.gz -mkdir tests -cp /input/package/work/target/debug/deps/* tests/ # Ask buildomat for the range of extra addresses that we're allowed to use, and # break them up into the ranges we need. diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index eedfb0dda3..b98abca5ae 100755 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -40,9 +40,9 @@ export CARGO_PROFILE_DEV_DEBUG=1 export CARGO_PROFILE_TEST_DEBUG=1 ptime -m cargo build --locked -p end-to-end-tests --tests --bin bootstrap \ --message-format json-render-diagnostics >/tmp/output.end-to-end.json -mapfile -t test_bins \ - < <(/opt/ooce/bin/jq -r 'select(.profile.test) | .executable' /tmp/output.end-to-end.json \ - | cut -c$((${#PWD} + 2))-) +mkdir tests +/opt/ooce/bin/jq -r 'select(.profile.test) | .executable' /tmp/output.end-to-end.json \ + | xargs -I {} -t cp {} tests/ # Assemble these outputs and some utilities into a tarball that can be used by # deployment phases of buildomat. @@ -55,6 +55,6 @@ files=( target/release/omicron-package target/release/xtask target/debug/bootstrap + tests/* ) -ptime -m tar cvzf $WORK/package.tar.gz \ - "${files[@]}" "${packages[@]}" "${test_bins[@]}" +ptime -m tar cvzf $WORK/package.tar.gz "${files[@]}" "${packages[@]}" From 2ae7fc553d3b9279d629ef04481797bc3348cc3c Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 04:06:08 +0000 Subject: [PATCH 10/35] don't separately upload OS images (it's slow) --- .github/buildomat/jobs/tuf-repo.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh index 396c410260..2ed1ae08c3 100755 --- a/.github/buildomat/jobs/tuf-repo.sh +++ b/.github/buildomat/jobs/tuf-repo.sh @@ -8,8 +8,6 @@ #: "=/work/manifest.toml", #: "=/work/repo.zip", #: "=/work/repo.zip.sha256.txt", -#: "=/work/os-host/os.tar.gz", -#: "=/work/os-recovery/os.tar.gz", #: "%/work/*.log", #: ] #: access_repos = [ @@ -30,11 +28,6 @@ #: ] #: #: [[publish]] -#: series = "image" -#: name = "os.tar.gz" -#: from_output = "/work/os-host/os.tar.gz" -#: -#: [[publish]] #: series = "rot-all" #: name = "manifest.toml" #: from_output = "/work/manifest.toml" From 712aef140a025188c2bd54caa8d42a5bd50e63ab Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 05:32:48 +0000 Subject: [PATCH 11/35] [shakes fist at tokio] --- dev-tools/releng/src/cmd.rs | 75 ++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 30 deletions(-) diff --git a/dev-tools/releng/src/cmd.rs b/dev-tools/releng/src/cmd.rs index 7fa9d2a713..bb2ce82477 100644 --- a/dev-tools/releng/src/cmd.rs +++ b/dev-tools/releng/src/cmd.rs @@ -14,6 +14,7 @@ use anyhow::Context; use anyhow::Result; use slog::debug; use slog::Logger; +use tokio::io::AsyncWriteExt; use tokio::process::Command; pub(crate) trait CommandExt { @@ -64,52 +65,66 @@ impl CommandExt for Command { } async fn is_success(&mut self, logger: &Logger) -> Result { - let output = run( - self.stdin(Stdio::null()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()), - logger, - ) - .await?; - Ok(output.status.success()) + Ok(xtrace(self, logger, Command::status).await?.success()) } async fn ensure_success(&mut self, logger: &Logger) -> Result<()> { - let output = run( - self.stdin(Stdio::null()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()), - logger, - ) - .await?; - self.check_status(output.status) + let status = xtrace(self, logger, Command::status).await?; + self.check_status(status) } async fn ensure_stdout(&mut self, logger: &Logger) -> Result { - let output = run( - self.stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::inherit()), - logger, - ) - .await?; + let output = xtrace(self, logger, Command::output).await?; + + // Obnoxiously, `tokio::process::Command::output` overrides + // your stdout and stderr settings (because it doesn't use + // std::process::Command::output). + // + // Compensate by dumping whatever is in `output.stderr` to stderr. + tokio::io::stderr().write_all(&output.stderr).await?; + self.check_status(output.status)?; String::from_utf8(output.stdout).context("command stdout was not UTF-8") } } -async fn run(command: &mut Command, logger: &Logger) -> Result { +trait AsStatus { + fn as_status(&self) -> &ExitStatus; +} + +impl AsStatus for ExitStatus { + fn as_status(&self) -> &ExitStatus { + &self + } +} + +impl AsStatus for Output { + fn as_status(&self) -> &ExitStatus { + &self.status + } +} + +async fn xtrace( + command: &mut Command, + logger: &Logger, + f: F, +) -> Result +where + F: FnOnce(&mut Command) -> Fut, + Fut: std::future::Future>, + T: AsStatus, +{ + command.stdin(Stdio::null()).kill_on_drop(true); debug!(logger, "running: {}", command.to_string()); let start = Instant::now(); - let output = - command.kill_on_drop(true).output().await.with_context(|| { - format!("failed to exec `{}`", command.to_string()) - })?; + let result = f(command) + .await + .with_context(|| format!("failed to exec `{}`", command.to_string()))?; debug!( logger, "process exited with {} ({:?})", - output.status, + result.as_status(), Instant::now().saturating_duration_since(start) ); - Ok(output) + Ok(result) } From aa7506d2e9121e9903fe2740a5e3e5a169b8b909 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 14:53:25 +0000 Subject: [PATCH 12/35] try to get to host-image faster --- dev-tools/releng/src/job.rs | 8 +++++++- dev-tools/releng/src/main.rs | 12 +++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/dev-tools/releng/src/job.rs b/dev-tools/releng/src/job.rs index aa00c316f4..063e12b1fa 100644 --- a/dev-tools/releng/src/job.rs +++ b/dev-tools/releng/src/job.rs @@ -31,6 +31,10 @@ use tokio::sync::Semaphore; use crate::cmd::CommandExt; +// We want these two jobs to run without delay because they take the longest +// amount of time, so we allow them to run without taking a permit first. +const PERMIT_NOT_REQUIRED: [&str; 2] = ["host-package", "host-image"]; + pub(crate) struct Jobs { logger: Logger, permits: Arc, @@ -186,7 +190,9 @@ async fn run_job( where F: Future> + 'static, { - let _ = permits.acquire_owned().await?; + if !PERMIT_NOT_REQUIRED.contains(&name.as_str()) { + let _ = permits.acquire_owned().await?; + } info!(logger, "[{}] running task", name); let start = Instant::now(); diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 255ae08c3c..8a0c8161ea 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -342,6 +342,16 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { .env("BUILD_OS", "no"), ); + // Download the toolchain for phbl before we get to the image build steps. + // (This is possibly a micro-optimization.) + jobs.push_command( + "phbl-toolchain", + Command::new("cargo") + .arg("--version") + .current_dir(args.helios_dir.join("projects/phbl")), + ) + .after("helios-setup"); + jobs.push_command( "omicron-package", Command::new("ptime").args([ @@ -498,7 +508,7 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { TUF_PACKAGES.into_iter(), ), ) - .after("host-proto"); + .after("host-stamp"); for (name, base_url) in [ ("staging", "https://permslip-staging.corp.oxide.computer"), From 9bb1c17b3f82180f619c5ca52615977c556f7d7e Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 14:54:02 +0000 Subject: [PATCH 13/35] correctness: put recovery artifacts in a new dir --- dev-tools/releng/src/main.rs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 8a0c8161ea..c4a11d2063 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -375,10 +375,23 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { image_build_args: $image_build_args:expr, image_dataset: $image_dataset:expr, ) => { + let artifacts_path = if concat!($target_name) == "host" { + Utf8PathBuf::from("out/") + } else { + args.output_dir.join("artifacts-recovery") + }; + jobs.push_command( concat!($target_name, "-target"), Command::new(&omicron_package) - .args(["--target", $target_name, "target", "create"]) + .args([ + "--target", + $target_name, + "--artifacts", + artifacts_path.as_str(), + "target", + "create", + ]) .args($target_args), ) .after("omicron-package"); @@ -388,6 +401,8 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { Command::new(&omicron_package).args([ "--target", $target_name, + "--artifacts", + artifacts_path.as_str(), "package", ]), ) @@ -401,6 +416,7 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { args.output_dir.clone(), omicron_package.clone(), $target_name, + artifacts_path.clone(), version.clone(), $proto_packages.iter().map(|(name, _)| *name), ), @@ -411,7 +427,7 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { jobs.push( concat!($target_name, "-proto"), build_proto_area( - WORKSPACE_DIR.join("out"), + artifacts_path, proto_dir.clone(), &$proto_packages, manifest.clone(), @@ -504,6 +520,7 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { args.output_dir.clone(), omicron_package.clone(), "host", + WORKSPACE_DIR.join("out"), version.clone(), TUF_PACKAGES.into_iter(), ), @@ -551,12 +568,14 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { Ok(()) } +#[allow(clippy::too_many_arguments)] async fn stamp_packages( logger: Logger, permits: Arc, output_dir: Utf8PathBuf, omicron_package: Utf8PathBuf, target_name: &'static str, + artifacts_path: Utf8PathBuf, version: Version, packages: impl Iterator, ) -> Result<()> { @@ -568,6 +587,8 @@ async fn stamp_packages( Command::new(&omicron_package) .arg("--target") .arg(target_name) + .arg("--artifacts") + .arg(&artifacts_path) .arg("stamp") .arg(package) .arg(&version), From 6dbca3c52b5355b542824a20baf05eee95fa0e71 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 14:54:25 +0000 Subject: [PATCH 14/35] actually use the stamped artifacts --- dev-tools/releng/src/main.rs | 5 ++++- dev-tools/releng/src/tuf.rs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index c4a11d2063..396b337367 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -598,7 +598,7 @@ async fn stamp_packages( } async fn build_proto_area( - package_dir: Utf8PathBuf, + mut package_dir: Utf8PathBuf, proto_dir: Utf8PathBuf, packages: &'static [(&'static str, InstallMethod)], manifest: Arc, @@ -607,6 +607,9 @@ async fn build_proto_area( let manifest_site = proto_dir.join("root/lib/svc/manifest/site"); fs::create_dir_all(&opt_oxide).await?; + // use the stamped packages + package_dir.push("versioned"); + for &(package_name, method) in packages { let package = manifest.packages.get(package_name).expect("checked in preflight"); diff --git a/dev-tools/releng/src/tuf.rs b/dev-tools/releng/src/tuf.rs index dc7d7cd46f..59c5d90089 100644 --- a/dev-tools/releng/src/tuf.rs +++ b/dev-tools/releng/src/tuf.rs @@ -93,7 +93,7 @@ pub(crate) async fn build_tuf_repo( .service_name )), path: crate::WORKSPACE_DIR - .join("out") + .join("out/versioned") .join(format!("{}.tar.gz", package)), }); } From f9186f9f8613ba045d0f3bcf99cbb0109a8bb9e8 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 16:19:00 +0000 Subject: [PATCH 15/35] delete caboose-util (we haven't used this since starting to use permslip for hubris artifacts, anyway) --- Cargo.lock | 9 --------- Cargo.toml | 2 -- caboose-util/Cargo.toml | 13 ------------- caboose-util/src/main.rs | 32 -------------------------------- 4 files changed, 56 deletions(-) delete mode 100644 caboose-util/Cargo.toml delete mode 100644 caboose-util/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 01a66c631f..2db239ca19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -785,15 +785,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "caboose-util" -version = "0.1.0" -dependencies = [ - "anyhow", - "hubtools", - "omicron-workspace-hack", -] - [[package]] name = "camino" version = "1.1.6" diff --git a/Cargo.toml b/Cargo.toml index 4cfa231365..6d0fb99554 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,6 @@ members = [ "api_identity", "bootstore", - "caboose-util", "certificates", "clients/bootstrap-agent-client", "clients/ddm-admin-client", @@ -85,7 +84,6 @@ members = [ default-members = [ "bootstore", - "caboose-util", "certificates", "clients/bootstrap-agent-client", "clients/ddm-admin-client", diff --git a/caboose-util/Cargo.toml b/caboose-util/Cargo.toml deleted file mode 100644 index ceff70b41d..0000000000 --- a/caboose-util/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "caboose-util" -version = "0.1.0" -edition = "2021" -license = "MPL-2.0" - -[lints] -workspace = true - -[dependencies] -anyhow.workspace = true -hubtools.workspace = true -omicron-workspace-hack.workspace = true diff --git a/caboose-util/src/main.rs b/caboose-util/src/main.rs deleted file mode 100644 index 36851cd36d..0000000000 --- a/caboose-util/src/main.rs +++ /dev/null @@ -1,32 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -// Copyright 2023 Oxide Computer Company - -use anyhow::{bail, Context, Result}; -use hubtools::{Caboose, RawHubrisArchive}; - -fn main() -> Result<()> { - let mut args = std::env::args().skip(1); - match args.next().context("subcommand required")?.as_str() { - "read-board" => { - let caboose = read_caboose(args.next())?; - println!("{}", std::str::from_utf8(caboose.board()?)?); - Ok(()) - } - "read-version" => { - let caboose = read_caboose(args.next())?; - println!("{}", std::str::from_utf8(caboose.version()?)?); - Ok(()) - } - unknown => bail!("unknown command {}", unknown), - } -} - -fn read_caboose(path: Option) -> Result { - let archive = RawHubrisArchive::load( - &path.context("path to hubris archive required")?, - )?; - Ok(archive.read_caboose()?) -} From 35ec9e676142512af485167a203d35f7db1bb365 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 17:44:45 +0000 Subject: [PATCH 16/35] we no longer use tools/hubris_{checksums,version} --- tools/hubris_checksums | 8 -------- tools/hubris_version | 1 - 2 files changed, 9 deletions(-) delete mode 100644 tools/hubris_checksums delete mode 100644 tools/hubris_version diff --git a/tools/hubris_checksums b/tools/hubris_checksums deleted file mode 100644 index 913cc460c4..0000000000 --- a/tools/hubris_checksums +++ /dev/null @@ -1,8 +0,0 @@ -4d38415a186fb1058c991d0e5ed44711457526e32687ff48ab6d6feadd8b4aa4 build-gimlet-c-image-default-v1.0.13.zip -ead1988cfebb4f79c364a2207f0bda741b8dd0e4f02fb34b4d341c648ecaa733 build-gimlet-d-image-default-v1.0.13.zip -85f5fc9c206c5fc61b4c2380b94a337220e944d67c0cb6bb2cb2486f8d5bc193 build-gimlet-e-image-default-v1.0.13.zip -ac7d898369e94e33b3556a405352b24a1ee107ce877d416811d9e9fae1f1a1ec build-gimlet-f-image-default-v1.0.13.zip -8cf812dc4aacc013335eb932d2bfaf8a542dec7bc29ea671d9a4235c12d61564 build-psc-b-image-default-v1.0.13.zip -85622677eef52c6d210f44e82b2b6cdc5a8357e509744abe1693883b7635b38c build-psc-c-image-default-v1.0.13.zip -87d6cd4add1aabe53756ba8f66a461cd3aa08f1a0093f94ea81a35a6a175ed21 build-sidecar-b-image-default-v1.0.13.zip -d50d6f77da6fc736843b5418359532f18b7ffa090c2a3d68b5dc1d35281385f5 build-sidecar-c-image-default-v1.0.13.zip diff --git a/tools/hubris_version b/tools/hubris_version deleted file mode 100644 index 717d36cec2..0000000000 --- a/tools/hubris_version +++ /dev/null @@ -1 +0,0 @@ -TAGS=(gimlet-v1.0.13 psc-v1.0.13 sidecar-v1.0.13) From 2e0916ef300bb5b90e3b160a64b75b9f2cfaa43d Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 17:47:00 +0000 Subject: [PATCH 17/35] write releng.adoc --- docs/releng.adoc | 79 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 docs/releng.adoc diff --git a/docs/releng.adoc b/docs/releng.adoc new file mode 100644 index 0000000000..f184cd9131 --- /dev/null +++ b/docs/releng.adoc @@ -0,0 +1,79 @@ +:showtitle: +:numbered: +:toc: left + += Oxide Release Engineering + +Omicron is the Oxide control plane, and thus brings together all of the +various components outside of this repo that make up the software on the +product. This includes (but definitely isn't limited to): + +- https://github.com/oxidecomputer/propolis[Propolis], our hypervisor +- https://github.com/oxidecomputer/helios[Helios], our host operating + system +- https://github.com/oxidecomputer/crucible[Crucible], our block storage + service +- https://github.com/oxidecomputer/maghemite[Maghemite], our switch + control software and routing protocol +- https://github.com/oxidecomputer/hubris[Hubris], our embedded + microcontroller operating system used on the root of trust and service + processors +- https://github.com/oxidecomputer/console[The web console] + +Each of these has their own build processes that produce some sort of +usable artifact, whether that is an illumos zone or a tarball of static +assets. + +The release engineering process builds the control plane and combines +it with the many external artifacts into a final artifact -- a Zip +archive of a TUF repository -- that contains everything necessary for +the product to operate. This process is run on each commit to ensure it +is always functional. You can also run the process locally with `cargo +xtask releng`. + +== Process overview + +. `tools/install_builder_prerequisites.sh` downloads several artifacts + (via the `tools/ci_*` scripts) that are necessary to build Omicron; + many of these are ultimately packaged by `omicron-package`. These + scripts are generally controlled by the `tools/*_version` and + `tools/*_checksums` files. +. `cargo xtask releng` downloads the current root of trust and + service processor images built by the Hubris release engineering + process, which are signed in https://github.com/oxidecomputer/permission-slip[Permission Slip]. + This is controlled by the `tools/permslip_production` and + `tools/permslip_staging` files. +. `omicron-package` is the heart of the release engineering process; it + reads the manifest from `package-manifest.toml`, runs an appropriate + `cargo build` command, downloads any additional artifacts, and + packages them into a series of illumos zones and tarballs. (It can + also manage installation and uninstallation of these zones; see + how-to-run.adoc.) +. Some of the packaged artifacts are installed directly in the OS + images; `cargo xtask releng` unpacks these into a temporary directory + that is overlaid onto the OS image in the next step. +. `helios-build` from the https://github.com/oxidecomputer/helios[Helios] + repository then builds two images: the *host* image, which is used + during normal operation, and the *trampoline* (or *recovery*) image, + which is used to update the host image. +. Finally, `cargo xtask releng` generates a Zip archive of a + https://theupdateframework.io/[TUF] repository, which contains the + host and trampoline OS images, the ROT and SP images, and all the + illumos zones that are not installed into the OS images. This archive + can be uploaded to Wicket to perform an upgrade of the rack while the + control plane is not running. + +`cargo xtask releng` performs all of these steps in parallel (with +the temporary exception of artifact downloads handled by +`tools/install_builder_prerequisites.sh`.) + +== Beyond `cargo xtask releng` + +Currently we use TUF repos generated in CI (by `cargo xtask releng`) +directly. These repositories use a generated throwaway key to sign +the TUF metadata. In the limit, we will have a process to sign release +builds of these TUF repositories, which will be available as a Zip +archive for an operator to upload to Nexus or Wicket, as well as an +HTTPS repository for racks connected to the internet or with access to +a proxy to perform automatic updates. The exact nature of the PKI and +trust policies for each of these update flows is under discussion. From 84be01c59b74ee087fe55e11b51c2352189f8f7e Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 19:52:54 +0000 Subject: [PATCH 18/35] oops --- dev-tools/releng/src/job.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev-tools/releng/src/job.rs b/dev-tools/releng/src/job.rs index 063e12b1fa..abdc1020ee 100644 --- a/dev-tools/releng/src/job.rs +++ b/dev-tools/releng/src/job.rs @@ -216,7 +216,9 @@ async fn spawn_with_output( name: String, log_path: Utf8PathBuf, ) -> Result<()> { - let _ = permits.acquire_owned().await?; + if !PERMIT_NOT_REQUIRED.contains(&name.as_str()) { + let _ = permits.acquire_owned().await?; + } let log_file_1 = File::create(log_path).await?; let log_file_2 = log_file_1.try_clone().await?; From c3a05371a730598907064cf75e61869fa2213c99 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 20:02:23 +0000 Subject: [PATCH 19/35] num_cpus -> std::thread::available_parallelism --- Cargo.lock | 1 - Cargo.toml | 1 - dev-tools/releng/Cargo.toml | 1 - dev-tools/releng/src/main.rs | 6 +++++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2db239ca19..d887914dee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5595,7 +5595,6 @@ dependencies = [ "fs-err", "futures", "hex", - "num_cpus", "omicron-common", "omicron-workspace-hack", "omicron-zone-package", diff --git a/Cargo.toml b/Cargo.toml index 6d0fb99554..1bed43cefb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -336,7 +336,6 @@ nexus-test-utils = { path = "nexus/test-utils" } nexus-types = { path = "nexus/types" } num-integer = "0.1.46" num = { version = "0.4.2", default-features = false, features = [ "libm" ] } -num_cpus = "1.16.0" omicron-common = { path = "common" } omicron-gateway = { path = "gateway" } omicron-nexus = { path = "nexus" } diff --git a/dev-tools/releng/Cargo.toml b/dev-tools/releng/Cargo.toml index 1e6d2df2da..3d25b60cc4 100644 --- a/dev-tools/releng/Cargo.toml +++ b/dev-tools/releng/Cargo.toml @@ -13,7 +13,6 @@ clap.workspace = true fs-err = { workspace = true, features = ["tokio"] } futures.workspace = true hex.workspace = true -num_cpus.workspace = true omicron-common.workspace = true omicron-workspace-hack.workspace = true omicron-zone-package.workspace = true diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 396b337367..663785e75c 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -180,7 +180,11 @@ fn main() -> Result<()> { #[tokio::main] async fn do_run(logger: Logger, args: Args) -> Result<()> { - let permits = Arc::new(Semaphore::new(num_cpus::get())); + let permits = Arc::new(Semaphore::new( + std::thread::available_parallelism() + .context("couldn't get available parallelism")? + .into(), + )); let commit = Command::new("git") .args(["rev-parse", "HEAD"]) From b720b63e8fcf84de7e76a4d48a94b661cea9f454 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 20:29:41 +0000 Subject: [PATCH 20/35] spawn job work onto tasks --- dev-tools/releng/src/job.rs | 67 ++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/dev-tools/releng/src/job.rs b/dev-tools/releng/src/job.rs index abdc1020ee..d5cc64bbc4 100644 --- a/dev-tools/releng/src/job.rs +++ b/dev-tools/releng/src/job.rs @@ -4,7 +4,6 @@ use std::collections::HashMap; use std::future::Future; -use std::pin::Pin; use std::process::Stdio; use std::sync::Arc; use std::time::Instant; @@ -15,6 +14,8 @@ use anyhow::Result; use camino::Utf8Path; use camino::Utf8PathBuf; use fs_err::tokio::File; +use futures::future::BoxFuture; +use futures::future::FutureExt; use futures::stream::FuturesUnordered; use futures::stream::TryStreamExt; use slog::info; @@ -43,7 +44,7 @@ pub(crate) struct Jobs { } struct Job { - future: Pin>>>, + future: BoxFuture<'static, Result<()>>, wait_for: Vec>, notify: Vec>, } @@ -67,25 +68,23 @@ impl Jobs { } } - pub(crate) fn push( + pub(crate) fn push( &mut self, name: impl AsRef, - future: F, - ) -> Selector<'_> - where - F: Future> + 'static, - { + future: impl Future> + Send + 'static, + ) -> Selector<'_> { let name = name.as_ref().to_owned(); assert!(!self.map.contains_key(&name), "duplicate job name {}", name); self.map.insert( name.clone(), Job { - future: Box::pin(run_job( + future: run_job( self.logger.clone(), self.permits.clone(), name.clone(), future, - )), + ) + .boxed(), wait_for: Vec::new(), notify: Vec::new(), }, @@ -181,22 +180,19 @@ macro_rules! info_or_error { }; } -async fn run_job( +async fn run_job( logger: Logger, permits: Arc, name: String, - future: F, -) -> Result<()> -where - F: Future> + 'static, -{ + future: impl Future> + Send + 'static, +) -> Result<()> { if !PERMIT_NOT_REQUIRED.contains(&name.as_str()) { let _ = permits.acquire_owned().await?; } info!(logger, "[{}] running task", name); let start = Instant::now(); - let result = future.await; + let result = tokio::spawn(future).await?; let duration = Instant::now().saturating_duration_since(start); info_or_error!( logger, @@ -233,14 +229,14 @@ async fn spawn_with_output( .spawn() .with_context(|| format!("failed to exec `{}`", command.to_string()))?; - let stdout = reader( - &name, + let stdout = spawn_reader( + format!("[{:>16}] ", name), child.stdout.take().unwrap(), tokio::io::stdout(), log_file_1, ); - let stderr = reader( - &name, + let stderr = spawn_reader( + format!("[{:>16}] ", name), child.stderr.take().unwrap(), tokio::io::stderr(), log_file_2, @@ -264,23 +260,26 @@ async fn spawn_with_output( } } -async fn reader( - name: &str, - reader: impl AsyncRead + Unpin, - mut terminal_writer: impl AsyncWrite + Unpin, +async fn spawn_reader( + prefix: String, + reader: impl AsyncRead + Send + Unpin + 'static, + mut terminal_writer: impl AsyncWrite + Send + Unpin + 'static, logfile_writer: File, ) -> std::io::Result<()> { let mut reader = BufReader::new(reader); let mut logfile_writer = tokio::fs::File::from(logfile_writer); - let mut buf = format!("[{:>16}] ", name).into_bytes(); + let mut buf = prefix.into_bytes(); let prefix_len = buf.len(); - loop { - buf.truncate(prefix_len); - let size = reader.read_until(b'\n', &mut buf).await?; - if size == 0 { - return Ok(()); + tokio::spawn(async move { + loop { + buf.truncate(prefix_len); + let size = reader.read_until(b'\n', &mut buf).await?; + if size == 0 { + return Ok(()); + } + terminal_writer.write_all(&buf).await?; + logfile_writer.write_all(&buf[prefix_len..]).await?; } - terminal_writer.write_all(&buf).await?; - logfile_writer.write_all(&buf[prefix_len..]).await?; - } + }) + .await? } From 8b3c76df837a0bc8405446919284d6a7eb46b5ab Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 20:50:48 +0000 Subject: [PATCH 21/35] fix incorrect sha256 --- dev-tools/releng/src/tuf.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/releng/src/tuf.rs b/dev-tools/releng/src/tuf.rs index 59c5d90089..2a880210eb 100644 --- a/dev-tools/releng/src/tuf.rs +++ b/dev-tools/releng/src/tuf.rs @@ -137,7 +137,7 @@ pub(crate) async fn build_tuf_repo( if n == 0 { break; } - hasher.update(&buf); + hasher.update(&buf[..n]); } fs::write( output_dir.join("repo.zip.sha256.txt"), From 6b6fc8cd21463486c5b049ed510b79924ceed3b3 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 20:54:02 +0000 Subject: [PATCH 22/35] asciidoc :( --- docs/releng.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/releng.adoc b/docs/releng.adoc index f184cd9131..27e67af3d2 100644 --- a/docs/releng.adoc +++ b/docs/releng.adoc @@ -28,8 +28,8 @@ The release engineering process builds the control plane and combines it with the many external artifacts into a final artifact -- a Zip archive of a TUF repository -- that contains everything necessary for the product to operate. This process is run on each commit to ensure it -is always functional. You can also run the process locally with `cargo -xtask releng`. +is always functional. You can also run the process locally with +`cargo xtask releng`. == Process overview From 86003af335f1291d3d70acf06f6f9749bacd918b Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 20:57:06 +0000 Subject: [PATCH 23/35] not sure why these fell out but sure --- Cargo.lock | 4 ---- workspace-hack/Cargo.toml | 8 -------- 2 files changed, 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d887914dee..12484631ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5837,11 +5837,8 @@ dependencies = [ "pem-rfc7468", "petgraph", "postgres-types", - "ppv-lite86", "predicates", "proc-macro2", - "rand 0.8.5", - "rand_chacha 0.3.1", "regex", "regex-automata 0.4.5", "regex-syntax 0.8.2", @@ -5879,7 +5876,6 @@ dependencies = [ "yasna", "zerocopy 0.7.32", "zeroize", - "zip", ] [[package]] diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 32423cb009..2ae7fe03cf 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -82,11 +82,8 @@ peg-runtime = { version = "0.8.3", default-features = false, features = ["std"] pem-rfc7468 = { version = "0.7.0", default-features = false, features = ["std"] } petgraph = { version = "0.6.4", features = ["serde-1"] } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } -ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } predicates = { version = "3.1.0" } proc-macro2 = { version = "1.0.81" } -rand = { version = "0.8.5" } -rand_chacha = { version = "0.3.1", default-features = false, features = ["std"] } regex = { version = "1.10.4" } regex-automata = { version = "0.4.5", default-features = false, features = ["dfa", "hybrid", "meta", "nfa", "perf", "unicode"] } regex-syntax = { version = "0.8.2" } @@ -122,7 +119,6 @@ uuid = { version = "1.8.0", features = ["serde", "v4"] } yasna = { version = "0.5.2", features = ["bit-vec", "num-bigint", "std", "time"] } zerocopy = { version = "0.7.32", features = ["derive", "simd"] } zeroize = { version = "1.7.0", features = ["std", "zeroize_derive"] } -zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } [build-dependencies] ahash = { version = "0.8.8" } @@ -190,11 +186,8 @@ peg-runtime = { version = "0.8.3", default-features = false, features = ["std"] pem-rfc7468 = { version = "0.7.0", default-features = false, features = ["std"] } petgraph = { version = "0.6.4", features = ["serde-1"] } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } -ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } predicates = { version = "3.1.0" } proc-macro2 = { version = "1.0.81" } -rand = { version = "0.8.5" } -rand_chacha = { version = "0.3.1", default-features = false, features = ["std"] } regex = { version = "1.10.4" } regex-automata = { version = "0.4.5", default-features = false, features = ["dfa", "hybrid", "meta", "nfa", "perf", "unicode"] } regex-syntax = { version = "0.8.2" } @@ -231,7 +224,6 @@ uuid = { version = "1.8.0", features = ["serde", "v4"] } yasna = { version = "0.5.2", features = ["bit-vec", "num-bigint", "std", "time"] } zerocopy = { version = "0.7.32", features = ["derive", "simd"] } zeroize = { version = "1.7.0", features = ["std", "zeroize_derive"] } -zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } [target.x86_64-unknown-linux-gnu.dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.2", default-features = false, features = ["std"] } From 18ac79831e99d8820af03290132380a8be791cdb Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 21:56:37 +0000 Subject: [PATCH 24/35] explain the hubris structs --- dev-tools/releng/src/hubris.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev-tools/releng/src/hubris.rs b/dev-tools/releng/src/hubris.rs index 4452249917..685a729a9f 100644 --- a/dev-tools/releng/src/hubris.rs +++ b/dev-tools/releng/src/hubris.rs @@ -119,6 +119,9 @@ async fn fetch_hash( }) } +// These structs are similar to `DeserializeManifest` and friends from +// tufaceous-lib, except that the source is a hash instead of a file path. This +// hash is used to download the artifact from Permission Slip. #[derive(Deserialize)] struct Manifest { #[serde(rename = "artifact")] From fe6adf4d9f0d5a1976de5da11784a29920658a3a Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Mon, 13 May 2024 23:20:20 +0000 Subject: [PATCH 25/35] refactor out the `os_image_jobs!` macro --- dev-tools/releng/src/job.rs | 5 +- dev-tools/releng/src/main.rs | 355 +++++++++++++++++++---------------- 2 files changed, 194 insertions(+), 166 deletions(-) diff --git a/dev-tools/releng/src/job.rs b/dev-tools/releng/src/job.rs index d5cc64bbc4..396f856307 100644 --- a/dev-tools/releng/src/job.rs +++ b/dev-tools/releng/src/job.rs @@ -102,7 +102,7 @@ impl Jobs { self.map.insert( name.clone(), Job { - future: Box::pin(spawn_with_output( + future: spawn_with_output( // terrible hack to deal with the `Command` builder // returning &mut std::mem::replace(command, Command::new("false")), @@ -110,7 +110,8 @@ impl Jobs { self.permits.clone(), name.clone(), self.log_dir.join(&name).with_extension("log"), - )), + ) + .boxed(), wait_for: Vec::new(), notify: Vec::new(), }, diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 663785e75c..0d54629cd1 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -204,7 +204,8 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { build.push_str("git"); build.extend(commit.chars().take(11)); version.build = build.parse()?; - info!(logger, "version: {}", version); + let version_str = version.to_string(); + info!(logger, "version: {}", version_str); let manifest = Arc::new(omicron_zone_package::config::parse_manifest( &fs::read_to_string(WORKSPACE_DIR.join("package-manifest.toml")) @@ -370,134 +371,125 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { ); let omicron_package = WORKSPACE_DIR.join("target/release/omicron-package"); - macro_rules! os_image_jobs { - ( - target_name: $target_name:literal, - target_args: $target_args:expr, - proto_packages: $proto_packages:expr, - image_prefix: $image_prefix:literal, - image_build_args: $image_build_args:expr, - image_dataset: $image_dataset:expr, - ) => { - let artifacts_path = if concat!($target_name) == "host" { - Utf8PathBuf::from("out/") - } else { - args.output_dir.join("artifacts-recovery") - }; - - jobs.push_command( - concat!($target_name, "-target"), - Command::new(&omicron_package) - .args([ + // Generate `omicron-package stamp` jobs for a list of packages as a nested + // `Jobs`. Returns the selector for the outer job. + // + // (This could be a function but the resulting function would have too many + // confusable arguments.) + macro_rules! stamp_packages { + ($name:expr, $target:expr, $packages:expr) => {{ + let mut stamp_jobs = + Jobs::new(&logger, permits.clone(), &args.output_dir); + for package in $packages { + stamp_jobs.push_command( + format!("stamp-{}", package), + Command::new(&omicron_package).args([ "--target", - $target_name, + $target.as_str(), "--artifacts", - artifacts_path.as_str(), - "target", - "create", - ]) - .args($target_args), - ) - .after("omicron-package"); - - jobs.push_command( - concat!($target_name, "-package"), - Command::new(&omicron_package).args([ + $target.artifacts_path(&args).as_str(), + "stamp", + package, + &version_str, + ]), + ); + } + jobs.push($name, stamp_jobs.run_all()) + }}; + } + + for target in [Target::Host, Target::Recovery] { + let artifacts_path = target.artifacts_path(&args); + + // omicron-package target create + jobs.push_command( + format!("{}-target", target), + Command::new(&omicron_package) + .args([ "--target", - $target_name, + target.as_str(), "--artifacts", artifacts_path.as_str(), - "package", - ]), - ) - .after(concat!($target_name, "-target")); - - jobs.push( - concat!($target_name, "-stamp"), - stamp_packages( - logger.clone(), - permits.clone(), - args.output_dir.clone(), - omicron_package.clone(), - $target_name, - artifacts_path.clone(), - version.clone(), - $proto_packages.iter().map(|(name, _)| *name), - ), - ) - .after(concat!($target_name, "-package")); - - let proto_dir = tempdir.path().join("proto").join($target_name); - jobs.push( - concat!($target_name, "-proto"), - build_proto_area( - artifacts_path, - proto_dir.clone(), - &$proto_packages, - manifest.clone(), - ), - ) - .after(concat!($target_name, "-stamp")); - - // The ${os_short_commit} token will be expanded by `helios-build` - let image_name = format!( - "{} {}/${{os_short_commit}} {}", - $image_prefix, - commit.chars().take(7).collect::(), - Utc::now().format("%Y-%m-%d %H:%M") - ); + "target", + "create", + ]) + .args(target.target_args()), + ) + .after("omicron-package"); - jobs.push_command( - concat!($target_name, "-image"), - Command::new("ptime") - .arg("-m") - .arg(args.helios_dir.join("helios-build")) - .arg("experiment-image") - .arg("-o") // output directory for image - .arg(args.output_dir.join(concat!("os-", $target_name))) - .arg("-p") // use an external package repository - .arg(format!("helios-dev={}", HELIOS_REPO)) - .arg("-F") // pass extra image builder features - .arg(format!("optever={}", opte_version.trim())) - .arg("-P") // include all files from extra proto area - .arg(proto_dir.join("root")) - .arg("-N") // image name - .arg(image_name) - .arg("-s") // tempdir name suffix - .arg($target_name) - .args($image_build_args) - .current_dir(&args.helios_dir) - .env("IMAGE_DATASET", &$image_dataset), - ) - .after("helios-setup") - .after(concat!($target_name, "-proto")); - }; - } + // omicron-package package + jobs.push_command( + format!("{}-package", target), + Command::new(&omicron_package).args([ + "--target", + target.as_str(), + "--artifacts", + artifacts_path.as_str(), + "package", + ]), + ) + .after(format!("{}-target", target)); - os_image_jobs! { - target_name: "host", - target_args: [ - "--image", - "standard", - "--machine", - "gimlet", - "--switch", - "asic", - "--rack-topology", - "multi-sled" - ], - proto_packages: HOST_IMAGE_PACKAGES, - image_prefix: "ci", - image_build_args: ["-B"], - image_dataset: args.host_dataset, - } - os_image_jobs! { - target_name: "recovery", - target_args: ["--image", "trampoline"], - proto_packages: RECOVERY_IMAGE_PACKAGES, - image_prefix: "recovery", - image_build_args: ["-R"], - image_dataset: args.recovery_dataset, + // omicron-package stamp + stamp_packages!( + format!("{}-stamp", target), + target, + target.proto_package_names() + ) + .after(format!("{}-package", target)); + + // [build proto dir, to be overlaid into disk image] + let proto_dir = tempdir.path().join("proto").join(target.as_str()); + jobs.push( + format!("{}-proto", target), + build_proto_area( + artifacts_path, + proto_dir.clone(), + target.proto_packages(), + manifest.clone(), + ), + ) + .after(format!("{}-stamp", target)); + + // The ${os_short_commit} token will be expanded by `helios-build` + let image_name = format!( + "{} {}/${{os_short_commit}} {}", + target.image_prefix(), + commit.chars().take(7).collect::(), + Utc::now().format("%Y-%m-%d %H:%M") + ); + + // helios-build experiment-image + jobs.push_command( + format!("{}-image", target), + Command::new("ptime") + .arg("-m") + .arg(args.helios_dir.join("helios-build")) + .arg("experiment-image") + .arg("-o") // output directory for image + .arg(args.output_dir.join(format!("os-{}", target))) + .arg("-p") // use an external package repository + .arg(format!("helios-dev={}", HELIOS_REPO)) + .arg("-F") // pass extra image builder features + .arg(format!("optever={}", opte_version.trim())) + .arg("-P") // include all files from extra proto area + .arg(proto_dir.join("root")) + .arg("-N") // image name + .arg(image_name) + .arg("-s") // tempdir name suffix + .arg(target.as_str()) + .args(target.image_build_args()) + .current_dir(&args.helios_dir) + .env( + "IMAGE_DATASET", + match target { + Target::Host => &args.host_dataset, + Target::Recovery => &args.recovery_dataset, + }, + ), + ) + .after("helios-setup") + .after(format!("{}-proto", target)); } // Build the recovery target after we build the host target. Only one // of these will build at a time since Cargo locks its target directory; @@ -508,6 +500,7 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { // If the datasets are the same, we can't parallelize these. jobs.select("recovery-image").after("host-image"); } + // Set up /root/.profile in the host OS image. jobs.push( "host-profile", @@ -516,20 +509,8 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { .after("host-proto"); jobs.select("host-image").after("host-profile"); - jobs.push( - "tuf-stamp", - stamp_packages( - logger.clone(), - permits.clone(), - args.output_dir.clone(), - omicron_package.clone(), - "host", - WORKSPACE_DIR.join("out"), - version.clone(), - TUF_PACKAGES.into_iter(), - ), - ) - .after("host-stamp"); + stamp_packages!("tuf-stamp", Target::Host, TUF_PACKAGES) + .after("host-stamp"); for (name, base_url) in [ ("staging", "https://permslip-staging.corp.oxide.computer"), @@ -551,8 +532,8 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { tuf::build_tuf_repo( logger.clone(), args.output_dir.clone(), - version.clone(), - manifest.clone(), + version, + manifest, ), ) .after("tuf-stamp") @@ -572,33 +553,79 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { Ok(()) } -#[allow(clippy::too_many_arguments)] -async fn stamp_packages( - logger: Logger, - permits: Arc, - output_dir: Utf8PathBuf, - omicron_package: Utf8PathBuf, - target_name: &'static str, - artifacts_path: Utf8PathBuf, - version: Version, - packages: impl Iterator, -) -> Result<()> { - let version = version.to_string(); - let mut jobs = Jobs::new(&logger, permits, &output_dir); - for package in packages { - jobs.push_command( - format!("stamp-{}", package), - Command::new(&omicron_package) - .arg("--target") - .arg(target_name) - .arg("--artifacts") - .arg(&artifacts_path) - .arg("stamp") - .arg(package) - .arg(&version), - ); +#[derive(Clone, Copy)] +enum Target { + Host, + Recovery, +} + +impl Target { + fn as_str(self) -> &'static str { + match self { + Target::Host => "host", + Target::Recovery => "recovery", + } + } + + fn artifacts_path(self, args: &Args) -> Utf8PathBuf { + match self { + Target::Host => WORKSPACE_DIR.join("out"), + Target::Recovery => { + args.output_dir.join(format!("artifacts-{}", self)) + } + } + } + + fn target_args(self) -> &'static [&'static str] { + match self { + Target::Host => &[ + "--image", + "standard", + "--machine", + "gimlet", + "--switch", + "asic", + "--rack-topology", + "multi-sled", + ], + Target::Recovery => &["--image", "trampoline"], + } + } + + fn proto_packages(self) -> &'static [(&'static str, InstallMethod)] { + match self { + Target::Host => &HOST_IMAGE_PACKAGES, + Target::Recovery => &RECOVERY_IMAGE_PACKAGES, + } + } + + fn proto_package_names(self) -> impl Iterator { + self.proto_packages().iter().map(|(name, _)| *name) + } + + fn image_prefix(self) -> &'static str { + match self { + Target::Host => "ci", + Target::Recovery => "recovery", + } + } + + fn image_build_args(self) -> &'static [&'static str] { + match self { + Target::Host => &[ + "-B", // include omicron1 brand + ], + Target::Recovery => &[ + "-R", // recovery image + ], + } + } +} + +impl std::fmt::Display for Target { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) } - jobs.run_all().await } async fn build_proto_area( From e457190803b13c533939927bcb155e6c6f5a891c Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 14 May 2024 01:52:15 +0000 Subject: [PATCH 26/35] job runner comments --- dev-tools/releng/src/job.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dev-tools/releng/src/job.rs b/dev-tools/releng/src/job.rs index 396f856307..3b75a856f4 100644 --- a/dev-tools/releng/src/job.rs +++ b/dev-tools/releng/src/job.rs @@ -2,6 +2,19 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +//! A quick-and-dirty job runner. +//! +//! Jobs are async functions given a name. All jobs must be described before the +//! jobs can be run (`Jobs::run_all` consumes the job runner). Jobs can depend +//! on other jobs, which is implemented via `tokio::sync::oneshot` channels; a +//! completed job sends a message to all registered receivers, which are waiting +//! on the messages in order to run. This essentially creates a DAG, except +//! instead of us having to keep track of it, we make it Tokio's problem. +//! +//! A `tokio::sync::Semaphore` is used to restrict the number of jobs to +//! `std::thread::available_parallelism`, except for a hardcoded list of +//! prioritized job names that are allowed to ignore this. + use std::collections::HashMap; use std::future::Future; use std::process::Stdio; @@ -145,6 +158,9 @@ impl Job { self.future.await?; for sender in self.notify { + // Ignore the error here -- the only reason we should fail to send + // our message is if a task has failed or the user hit Ctrl-C, at + // which point a bunch of error logging is not particularly useful. sender.send(()).ok(); } Ok(()) @@ -274,6 +290,9 @@ async fn spawn_reader( tokio::spawn(async move { loop { buf.truncate(prefix_len); + // We have no particular control over the output from the child + // processes we run, so we read until a newline character without + // relying on valid UTF-8 output. let size = reader.read_until(b'\n', &mut buf).await?; if size == 0 { return Ok(()); From 61400a30e33030c8ff053b649b3dbe5cd4a74d3d Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 14 May 2024 01:54:33 +0000 Subject: [PATCH 27/35] use the new-ish actually line tables only setting --- .github/buildomat/jobs/package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index b98abca5ae..63e5e1ce71 100755 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -36,8 +36,8 @@ ptime -m cargo build --locked --release -p xtask # Build the end-to-end tests # Reduce debuginfo just to line tables. -export CARGO_PROFILE_DEV_DEBUG=1 -export CARGO_PROFILE_TEST_DEBUG=1 +export CARGO_PROFILE_DEV_DEBUG=line-tables-only +export CARGO_PROFILE_TEST_DEBUG=line-tables-only ptime -m cargo build --locked -p end-to-end-tests --tests --bin bootstrap \ --message-format json-render-diagnostics >/tmp/output.end-to-end.json mkdir tests From a2168a25da98355a1f60b5186a6e3b1aef458d24 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 14 May 2024 03:34:52 +0000 Subject: [PATCH 28/35] honor $GIT/$OMICRON_PACKAGE --- dev-tools/releng/src/main.rs | 48 +++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 0d54629cd1..4b61e2e813 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -127,6 +127,14 @@ struct Args { /// Output dir for TUF repo and log files #[clap(long, default_value_t = Self::default_output_dir())] output_dir: Utf8PathBuf, + + /// Path to the git binary + #[clap(long, env = "GIT", default_value = "git")] + git_bin: Utf8PathBuf, + + /// Path to a pre-built omicron-package binary (skips building if set) + #[clap(long, env = "OMICRON_PACKAGE")] + omicron_package_bin: Option, } impl Args { @@ -186,7 +194,7 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { .into(), )); - let commit = Command::new("git") + let commit = Command::new(&args.git_bin) .args(["rev-parse", "HEAD"]) .ensure_stdout(&logger) .await? @@ -244,13 +252,13 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { if args.helios_dir.exists() { if !args.ignore_helios_origin { // check that our helios clone is up to date - Command::new("git") + Command::new(&args.git_bin) .arg("-C") .arg(&args.helios_dir) .args(["fetch", "--no-write-fetch-head", "origin", "master"]) .ensure_success(&logger) .await?; - let stdout = Command::new("git") + let stdout = Command::new(&args.git_bin) .arg("-C") .arg(&args.helios_dir) .args(["rev-parse", "HEAD", "origin/master"]) @@ -272,14 +280,14 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { } } else { info!(logger, "cloning helios to {}", args.helios_dir); - Command::new("git") + Command::new(&args.git_bin) .args(["clone", "https://github.com/oxidecomputer/helios.git"]) .arg(&args.helios_dir) .ensure_success(&logger) .await?; } // Record the branch and commit in the output - Command::new("git") + Command::new(&args.git_bin) .arg("-C") .arg(&args.helios_dir) .args(["status", "--branch", "--porcelain=2"]) @@ -357,19 +365,25 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { ) .after("helios-setup"); - jobs.push_command( - "omicron-package", - Command::new("ptime").args([ - "-m", - "cargo", - "build", - "--locked", - "--release", - "--bin", + let omicron_package = if let Some(path) = &args.omicron_package_bin { + // omicron-package is provided, so don't build it. + jobs.push("omicron-package", std::future::ready(Ok(()))); + path.clone() + } else { + jobs.push_command( "omicron-package", - ]), - ); - let omicron_package = WORKSPACE_DIR.join("target/release/omicron-package"); + Command::new("ptime").args([ + "-m", + "cargo", + "build", + "--locked", + "--release", + "--bin", + "omicron-package", + ]), + ); + WORKSPACE_DIR.join("target/release/omicron-package") + }; // Generate `omicron-package stamp` jobs for a list of packages as a nested // `Jobs`. Returns the selector for the outer job. From f491f2ff421315be3d861534f1a155324b94bc34 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 14 May 2024 03:39:07 +0000 Subject: [PATCH 29/35] cancel safety in spawn_with_output --- dev-tools/releng/src/job.rs | 41 +++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/dev-tools/releng/src/job.rs b/dev-tools/releng/src/job.rs index 3b75a856f4..ce8a2de244 100644 --- a/dev-tools/releng/src/job.rs +++ b/dev-tools/releng/src/job.rs @@ -258,31 +258,33 @@ async fn spawn_with_output( tokio::io::stderr(), log_file_2, ); - match tokio::try_join!(child.wait(), stdout, stderr) { - Ok((status, (), ())) => { - let result = command.check_status(status); - info_or_error!( - logger, - result, - "[{}] process exited with {} ({:?})", - name, - status, - Instant::now().saturating_duration_since(start) - ); - result - } - Err(err) => Err(err).with_context(|| { - format!("I/O error while waiting for job {:?} to complete", name) - }), - } + + let status = child.wait().await.with_context(|| { + format!("I/O error while waiting for job {:?} to complete", name) + })?; + let result = command.check_status(status); + info_or_error!( + logger, + result, + "[{}] process exited with {} ({:?})", + name, + status, + Instant::now().saturating_duration_since(start) + ); + + // bubble up any errors from `spawn_reader` + stdout.await??; + stderr.await??; + + result } -async fn spawn_reader( +fn spawn_reader( prefix: String, reader: impl AsyncRead + Send + Unpin + 'static, mut terminal_writer: impl AsyncWrite + Send + Unpin + 'static, logfile_writer: File, -) -> std::io::Result<()> { +) -> tokio::task::JoinHandle> { let mut reader = BufReader::new(reader); let mut logfile_writer = tokio::fs::File::from(logfile_writer); let mut buf = prefix.into_bytes(); @@ -301,5 +303,4 @@ async fn spawn_reader( logfile_writer.write_all(&buf[prefix_len..]).await?; } }) - .await? } From bf93834c7bd45d3cae20fff92ea0f1248c79e6ea Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 14 May 2024 06:03:55 +0000 Subject: [PATCH 30/35] handle alternate target directories --- Cargo.lock | 1 + dev-tools/releng/Cargo.toml | 1 + dev-tools/releng/src/main.rs | 17 ++++++++++++++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d22c9acd49..882fbc9076 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5589,6 +5589,7 @@ dependencies = [ "anyhow", "camino", "camino-tempfile", + "cargo_metadata", "chrono", "clap", "fs-err", diff --git a/dev-tools/releng/Cargo.toml b/dev-tools/releng/Cargo.toml index 3d25b60cc4..e1e250d0a9 100644 --- a/dev-tools/releng/Cargo.toml +++ b/dev-tools/releng/Cargo.toml @@ -8,6 +8,7 @@ license = "MPL-2.0" anyhow.workspace = true camino.workspace = true camino-tempfile.workspace = true +cargo_metadata = "0.18.1" chrono.workspace = true clap.workspace = true fs-err = { workspace = true, features = ["tokio"] } diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 4b61e2e813..438edfacdb 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -171,6 +171,13 @@ fn main() -> Result<()> { std::env::set_current_dir(&*WORKSPACE_DIR) .context("failed to change working directory to workspace root")?; + // Determine the target directory. + let target_dir = cargo_metadata::MetadataCommand::new() + .no_deps() + .exec() + .context("failed to get cargo metadata")? + .target_directory; + // Unset `$CARGO`, `$CARGO_MANIFEST_DIR`, and `$RUSTUP_TOOLCHAIN` (all // set by cargo or its rustup proxy), which will interfere with various // tools we're about to run. (This needs to come _after_ we read from @@ -183,11 +190,15 @@ fn main() -> Result<()> { // Now that we're done mucking about with our environment (something that's // not necessarily safe in multi-threaded programs), create a Tokio runtime // and call `do_run`. - do_run(logger, args) + do_run(logger, args, target_dir) } #[tokio::main] -async fn do_run(logger: Logger, args: Args) -> Result<()> { +async fn do_run( + logger: Logger, + args: Args, + target_dir: Utf8PathBuf, +) -> Result<()> { let permits = Arc::new(Semaphore::new( std::thread::available_parallelism() .context("couldn't get available parallelism")? @@ -382,7 +393,7 @@ async fn do_run(logger: Logger, args: Args) -> Result<()> { "omicron-package", ]), ); - WORKSPACE_DIR.join("target/release/omicron-package") + target_dir.join("release/omicron-package") }; // Generate `omicron-package stamp` jobs for a list of packages as a nested From 0453fd10ed4541c93939cfebb97e5c9935b21c0c Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 14 May 2024 06:08:24 +0000 Subject: [PATCH 31/35] workspace dep cargo_metadata --- Cargo.toml | 1 + dev-tools/releng/Cargo.toml | 2 +- dev-tools/xtask/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7f7869ddd0..8c3cbc52ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -228,6 +228,7 @@ bytes = "1.6.0" camino = { version = "1.1", features = ["serde1"] } camino-tempfile = "1.1.1" cancel-safe-futures = "0.1.5" +cargo_metadata = "0.18.1" chacha20poly1305 = "0.10.1" ciborium = "0.2.2" cfg-if = "1.0" diff --git a/dev-tools/releng/Cargo.toml b/dev-tools/releng/Cargo.toml index e1e250d0a9..19ede6c24d 100644 --- a/dev-tools/releng/Cargo.toml +++ b/dev-tools/releng/Cargo.toml @@ -8,7 +8,7 @@ license = "MPL-2.0" anyhow.workspace = true camino.workspace = true camino-tempfile.workspace = true -cargo_metadata = "0.18.1" +cargo_metadata.workspace = true chrono.workspace = true clap.workspace = true fs-err = { workspace = true, features = ["tokio"] } diff --git a/dev-tools/xtask/Cargo.toml b/dev-tools/xtask/Cargo.toml index 11fcf405bd..2aecde57e5 100644 --- a/dev-tools/xtask/Cargo.toml +++ b/dev-tools/xtask/Cargo.toml @@ -11,7 +11,7 @@ workspace = true anyhow.workspace = true camino.workspace = true cargo_toml = "0.20" -cargo_metadata = "0.18" +cargo_metadata.workspace = true clap.workspace = true macaddr.workspace = true serde.workspace = true From aef238cde2bb25209c5ca356660db7abc1b58e80 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 14 May 2024 16:35:51 +0000 Subject: [PATCH 32/35] replace the terrible std::mem::replace hack with more indirection --- dev-tools/releng/src/cmd.rs | 194 ++++++++++++++++++++--------------- dev-tools/releng/src/job.rs | 19 ++-- dev-tools/releng/src/main.rs | 3 +- 3 files changed, 123 insertions(+), 93 deletions(-) diff --git a/dev-tools/releng/src/cmd.rs b/dev-tools/releng/src/cmd.rs index bb2ce82477..fd51f1383b 100644 --- a/dev-tools/releng/src/cmd.rs +++ b/dev-tools/releng/src/cmd.rs @@ -3,7 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use std::ffi::OsStr; -use std::fmt::Write; +use std::path::Path; use std::process::ExitStatus; use std::process::Output; use std::process::Stdio; @@ -14,117 +14,149 @@ use anyhow::Context; use anyhow::Result; use slog::debug; use slog::Logger; -use tokio::io::AsyncWriteExt; -use tokio::process::Command; -pub(crate) trait CommandExt { - fn check_status(&self, status: ExitStatus) -> Result<()>; - fn to_string(&self) -> String; - - async fn is_success(&mut self, logger: &Logger) -> Result; - async fn ensure_success(&mut self, logger: &Logger) -> Result<()>; - async fn ensure_stdout(&mut self, logger: &Logger) -> Result; +/// Wrapper for `tokio::process::Command` where the builder methods take/return +/// `self`, plus a number of convenience methods. +pub(crate) struct Command { + inner: tokio::process::Command, } -impl CommandExt for Command { - fn check_status(&self, status: ExitStatus) -> Result<()> { - ensure!( - status.success(), - "command `{}` exited with {}", - self.to_string(), - status - ); - Ok(()) +impl Command { + pub(crate) fn new(program: impl AsRef) -> Command { + Command { inner: tokio::process::Command::new(program) } + } + + pub(crate) fn arg(mut self, arg: impl AsRef) -> Command { + self.inner.arg(arg); + self + } + + pub(crate) fn args( + mut self, + args: impl IntoIterator>, + ) -> Command { + self.inner.args(args); + self + } + + pub(crate) fn current_dir(mut self, dir: impl AsRef) -> Command { + self.inner.current_dir(dir); + self + } + + pub(crate) fn env( + mut self, + key: impl AsRef, + value: impl AsRef, + ) -> Command { + self.inner.env(key, value); + self + } + + pub(crate) async fn is_success(mut self, logger: &Logger) -> Result { + self.inner + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + Ok(xtrace(&mut self, logger).await?.status.success()) + } + + pub(crate) async fn ensure_success( + mut self, + logger: &Logger, + ) -> Result<()> { + self.inner + .stdin(Stdio::null()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + let status = xtrace(&mut self, logger).await?.status; + check_status(self, status) + } + + pub(crate) async fn ensure_stdout( + mut self, + logger: &Logger, + ) -> Result { + self.inner + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()); + let output = xtrace(&mut self, logger).await?; + check_status(self, output.status)?; + String::from_utf8(output.stdout).context("command stdout was not UTF-8") } - fn to_string(&self) -> String { - let command = self.as_std(); - let mut command_str = String::new(); + pub(crate) fn into_parts(self) -> (Description, tokio::process::Command) { + (Description { str: self.to_string() }, self.inner) + } +} + +impl std::fmt::Display for Command { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let command = self.inner.as_std(); for (name, value) in command.get_envs() { if let Some(value) = value { write!( - command_str, + f, "{}={} ", shell_words::quote(&name.to_string_lossy()), shell_words::quote(&value.to_string_lossy()) - ) - .unwrap(); + )?; } } write!( - command_str, + f, "{}", - shell_words::join( - std::iter::once(command.get_program()) - .chain(command.get_args()) - .map(OsStr::to_string_lossy) - ) - ) - .unwrap(); - command_str - } - - async fn is_success(&mut self, logger: &Logger) -> Result { - Ok(xtrace(self, logger, Command::status).await?.success()) - } - - async fn ensure_success(&mut self, logger: &Logger) -> Result<()> { - let status = xtrace(self, logger, Command::status).await?; - self.check_status(status) - } - - async fn ensure_stdout(&mut self, logger: &Logger) -> Result { - let output = xtrace(self, logger, Command::output).await?; - - // Obnoxiously, `tokio::process::Command::output` overrides - // your stdout and stderr settings (because it doesn't use - // std::process::Command::output). - // - // Compensate by dumping whatever is in `output.stderr` to stderr. - tokio::io::stderr().write_all(&output.stderr).await?; - - self.check_status(output.status)?; - String::from_utf8(output.stdout).context("command stdout was not UTF-8") + shell_words::quote(&command.get_program().to_string_lossy()) + )?; + for arg in command.get_args() { + write!(f, " {}", shell_words::quote(&arg.to_string_lossy()))?; + } + Ok(()) } } -trait AsStatus { - fn as_status(&self) -> &ExitStatus; +/// Returned from [`Command::into_parts`] for use in the `job` module. +pub(crate) struct Description { + str: String, } -impl AsStatus for ExitStatus { - fn as_status(&self) -> &ExitStatus { - &self +impl Description { + pub(crate) fn check_status(&self, status: ExitStatus) -> Result<()> { + check_status(self, status) } } -impl AsStatus for Output { - fn as_status(&self) -> &ExitStatus { - &self.status +impl std::fmt::Display for Description { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.str) } } -async fn xtrace( - command: &mut Command, - logger: &Logger, - f: F, -) -> Result -where - F: FnOnce(&mut Command) -> Fut, - Fut: std::future::Future>, - T: AsStatus, -{ - command.stdin(Stdio::null()).kill_on_drop(true); - debug!(logger, "running: {}", command.to_string()); +fn check_status( + command: impl std::fmt::Display, + status: ExitStatus, +) -> Result<()> { + ensure!(status.success(), "command `{}` exited with {}", command, status); + Ok(()) +} + +async fn xtrace(command: &mut Command, logger: &Logger) -> Result { + command.inner.stdin(Stdio::null()).kill_on_drop(true); + debug!(logger, "running: {}", command); let start = Instant::now(); - let result = f(command) + let output = command + .inner + .spawn() + .with_context(|| format!("failed to exec `{}`", command))? + .wait_with_output() .await - .with_context(|| format!("failed to exec `{}`", command.to_string()))?; + .with_context(|| format!("failed to wait on `{}`", command))?; debug!( logger, "process exited with {} ({:?})", - result.as_status(), + output.status, Instant::now().saturating_duration_since(start) ); - Ok(result) + Ok(output) } diff --git a/dev-tools/releng/src/job.rs b/dev-tools/releng/src/job.rs index ce8a2de244..dcb58a0b92 100644 --- a/dev-tools/releng/src/job.rs +++ b/dev-tools/releng/src/job.rs @@ -38,12 +38,11 @@ use tokio::io::AsyncRead; use tokio::io::AsyncWrite; use tokio::io::AsyncWriteExt; use tokio::io::BufReader; -use tokio::process::Command; use tokio::sync::oneshot; use tokio::sync::oneshot::error::RecvError; use tokio::sync::Semaphore; -use crate::cmd::CommandExt; +use crate::cmd::Command; // We want these two jobs to run without delay because they take the longest // amount of time, so we allow them to run without taking a permit first. @@ -108,7 +107,7 @@ impl Jobs { pub(crate) fn push_command( &mut self, name: impl AsRef, - command: &mut Command, + command: Command, ) -> Selector<'_> { let name = name.as_ref().to_owned(); assert!(!self.map.contains_key(&name), "duplicate job name {}", name); @@ -116,9 +115,7 @@ impl Jobs { name.clone(), Job { future: spawn_with_output( - // terrible hack to deal with the `Command` builder - // returning &mut - std::mem::replace(command, Command::new("false")), + command, self.logger.clone(), self.permits.clone(), name.clone(), @@ -223,7 +220,7 @@ async fn run_job( } async fn spawn_with_output( - mut command: Command, + command: Command, logger: Logger, permits: Arc, name: String, @@ -233,10 +230,12 @@ async fn spawn_with_output( let _ = permits.acquire_owned().await?; } + let (command_desc, mut command) = command.into_parts(); + let log_file_1 = File::create(log_path).await?; let log_file_2 = log_file_1.try_clone().await?; - info!(logger, "[{}] running: {}", name, command.to_string()); + info!(logger, "[{}] running: {}", name, command_desc); let start = Instant::now(); let mut child = command .kill_on_drop(true) @@ -244,7 +243,7 @@ async fn spawn_with_output( .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() - .with_context(|| format!("failed to exec `{}`", command.to_string()))?; + .with_context(|| format!("failed to exec `{}`", command_desc))?; let stdout = spawn_reader( format!("[{:>16}] ", name), @@ -262,7 +261,7 @@ async fn spawn_with_output( let status = child.wait().await.with_context(|| { format!("I/O error while waiting for job {:?} to complete", name) })?; - let result = command.check_status(status); + let result = command_desc.check_status(status); info_or_error!( logger, result, diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 438edfacdb..5e4464bae0 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -28,10 +28,9 @@ use slog::Drain; use slog::Logger; use slog_term::FullFormat; use slog_term::TermDecorator; -use tokio::process::Command; use tokio::sync::Semaphore; -use crate::cmd::CommandExt; +use crate::cmd::Command; use crate::job::Jobs; /// The base version we're currently building. Build information is appended to From b7664dd5753601ff38f443ed61d3ff4fbc500430 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 14 May 2024 16:45:43 +0000 Subject: [PATCH 33/35] explain $CARGO --- dev-tools/releng/src/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 5e4464bae0..37431e1ada 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -181,6 +181,14 @@ fn main() -> Result<()> { // set by cargo or its rustup proxy), which will interfere with various // tools we're about to run. (This needs to come _after_ we read from // `WORKSPACE_DIR` as it relies on `$CARGO_MANIFEST_DIR`.) + // + // We also don't respect `$CARGO` when running Cargo throughout this + // program; we need `cargo` to be the rustup proxy for various Helios + // build tools. Cargo always searches $PATH for rustc, so running + // the toolchain-specific Cargo set in `$CARGO` without a valid + // `$RUSTUP_TOOLCHAIN` leads to hilarious toolchain mismatches when + // compiling dependencies that happen to have a `rust-toolchain.toml` in + // their source directory or Git checkout. for var in ["CARGO", "CARGO_MANIFEST_DIR", "RUSTUP_TOOLCHAIN"] { debug!(logger, "unsetting ${}", var); std::env::remove_var(var); From 9e341e8626f8de4a8082d7032574b51a4df7c415 Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 14 May 2024 17:11:46 +0000 Subject: [PATCH 34/35] respect $CARGO/$CARGO_HOME --- dev-tools/releng/src/cmd.rs | 5 ++ dev-tools/releng/src/main.rs | 104 ++++++++++++++++++----------------- 2 files changed, 58 insertions(+), 51 deletions(-) diff --git a/dev-tools/releng/src/cmd.rs b/dev-tools/releng/src/cmd.rs index fd51f1383b..198eabf99f 100644 --- a/dev-tools/releng/src/cmd.rs +++ b/dev-tools/releng/src/cmd.rs @@ -53,6 +53,11 @@ impl Command { self } + pub(crate) fn env_remove(mut self, key: impl AsRef) -> Command { + self.inner.env_remove(key); + self + } + pub(crate) async fn is_success(mut self, logger: &Logger) -> Result { self.inner .stdin(Stdio::null()) diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 37431e1ada..0fa4382931 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -127,6 +127,11 @@ struct Args { #[clap(long, default_value_t = Self::default_output_dir())] output_dir: Utf8PathBuf, + /// Path to the directory containing the rustup proxy `bin/cargo` (usually + /// set by Cargo) + #[clap(long, env = "CARGO_HOME")] + cargo_home: Option, + /// Path to the git binary #[clap(long, env = "GIT", default_value = "git")] git_bin: Utf8PathBuf, @@ -157,7 +162,8 @@ impl Args { } } -fn main() -> Result<()> { +#[tokio::main] +async fn main() -> Result<()> { let args = Args::parse(); let decorator = TermDecorator::new().build(); @@ -177,35 +183,20 @@ fn main() -> Result<()> { .context("failed to get cargo metadata")? .target_directory; - // Unset `$CARGO`, `$CARGO_MANIFEST_DIR`, and `$RUSTUP_TOOLCHAIN` (all - // set by cargo or its rustup proxy), which will interfere with various - // tools we're about to run. (This needs to come _after_ we read from - // `WORKSPACE_DIR` as it relies on `$CARGO_MANIFEST_DIR`.) - // - // We also don't respect `$CARGO` when running Cargo throughout this - // program; we need `cargo` to be the rustup proxy for various Helios - // build tools. Cargo always searches $PATH for rustc, so running - // the toolchain-specific Cargo set in `$CARGO` without a valid - // `$RUSTUP_TOOLCHAIN` leads to hilarious toolchain mismatches when - // compiling dependencies that happen to have a `rust-toolchain.toml` in - // their source directory or Git checkout. - for var in ["CARGO", "CARGO_MANIFEST_DIR", "RUSTUP_TOOLCHAIN"] { - debug!(logger, "unsetting ${}", var); - std::env::remove_var(var); - } - - // Now that we're done mucking about with our environment (something that's - // not necessarily safe in multi-threaded programs), create a Tokio runtime - // and call `do_run`. - do_run(logger, args, target_dir) -} + // We build everything in Omicron with $CARGO, but we need to use the rustup + // proxy for Cargo when outside Omicron. + let rustup_cargo = match &args.cargo_home { + Some(path) => path.join("bin/cargo"), + None => Utf8PathBuf::from("cargo"), + }; + // `var_os` here is deliberate: if CARGO is set to a non-UTF-8 path we + // shouldn't do something confusing as a fallback. + let cargo = match std::env::var_os("CARGO") { + Some(path) => Utf8PathBuf::try_from(std::path::PathBuf::from(path)) + .context("$CARGO is not valid UTF-8")?, + None => rustup_cargo.clone(), + }; -#[tokio::main] -async fn do_run( - logger: Logger, - args: Args, - target_dir: Utf8PathBuf, -) -> Result<()> { let permits = Arc::new(Semaphore::new( std::thread::available_parallelism() .context("couldn't get available parallelism")? @@ -370,16 +361,20 @@ async fn do_run( // Setting `BUILD_OS` to no makes setup skip repositories we don't // need for building the OS itself (we are just building an image // from an already-built OS). - .env("BUILD_OS", "no"), + .env("BUILD_OS", "no") + .env_remove("CARGO") + .env_remove("RUSTUP_TOOLCHAIN"), ); // Download the toolchain for phbl before we get to the image build steps. // (This is possibly a micro-optimization.) jobs.push_command( "phbl-toolchain", - Command::new("cargo") + Command::new(&rustup_cargo) .arg("--version") - .current_dir(args.helios_dir.join("projects/phbl")), + .current_dir(args.helios_dir.join("projects/phbl")) + .env_remove("CARGO") + .env_remove("RUSTUP_TOOLCHAIN"), ) .after("helios-setup"); @@ -392,7 +387,7 @@ async fn do_run( "omicron-package", Command::new("ptime").args([ "-m", - "cargo", + cargo.as_str(), "build", "--locked", "--release", @@ -415,15 +410,17 @@ async fn do_run( for package in $packages { stamp_jobs.push_command( format!("stamp-{}", package), - Command::new(&omicron_package).args([ - "--target", - $target.as_str(), - "--artifacts", - $target.artifacts_path(&args).as_str(), - "stamp", - package, - &version_str, - ]), + Command::new(&omicron_package) + .args([ + "--target", + $target.as_str(), + "--artifacts", + $target.artifacts_path(&args).as_str(), + "stamp", + package, + &version_str, + ]) + .env_remove("CARGO_MANIFEST_DIR"), ); } jobs.push($name, stamp_jobs.run_all()) @@ -445,20 +442,23 @@ async fn do_run( "target", "create", ]) - .args(target.target_args()), + .args(target.target_args()) + .env_remove("CARGO_MANIFEST_DIR"), ) .after("omicron-package"); // omicron-package package jobs.push_command( format!("{}-package", target), - Command::new(&omicron_package).args([ - "--target", - target.as_str(), - "--artifacts", - artifacts_path.as_str(), - "package", - ]), + Command::new(&omicron_package) + .args([ + "--target", + target.as_str(), + "--artifacts", + artifacts_path.as_str(), + "package", + ]) + .env_remove("CARGO_MANIFEST_DIR"), ) .after(format!("{}-target", target)); @@ -518,7 +518,9 @@ async fn do_run( Target::Host => &args.host_dataset, Target::Recovery => &args.recovery_dataset, }, - ), + ) + .env_remove("CARGO") + .env_remove("RUSTUP_TOOLCHAIN"), ) .after("helios-setup") .after(format!("{}-proto", target)); From d5341bba54cc40759a126eae934ad4b22a53313a Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 14 May 2024 17:17:13 +0000 Subject: [PATCH 35/35] doc tweaks --- docs/releng.adoc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/releng.adoc b/docs/releng.adoc index 27e67af3d2..31252c9a89 100644 --- a/docs/releng.adoc +++ b/docs/releng.adoc @@ -33,6 +33,10 @@ is always functional. You can also run the process locally with == Process overview +`cargo xtask releng` performs all of these steps in parallel (with +the temporary exception of artifact downloads handled by +`tools/install_builder_prerequisites.sh`): + . `tools/install_builder_prerequisites.sh` downloads several artifacts (via the `tools/ci_*` scripts) that are necessary to build Omicron; many of these are ultimately packaged by `omicron-package`. These @@ -49,9 +53,11 @@ is always functional. You can also run the process locally with packages them into a series of illumos zones and tarballs. (It can also manage installation and uninstallation of these zones; see how-to-run.adoc.) -. Some of the packaged artifacts are installed directly in the OS - images; `cargo xtask releng` unpacks these into a temporary directory - that is overlaid onto the OS image in the next step. +. Some of the illumos zones are distributed with the OS images (because + they are reliant on OS-specific APIs), and some are distributed + separately. `cargo xtask releng` unpacks the zones for the OS image + into a temporary directory that is overlaid onto the OS image in the + next step. . `helios-build` from the https://github.com/oxidecomputer/helios[Helios] repository then builds two images: the *host* image, which is used during normal operation, and the *trampoline* (or *recovery*) image, @@ -63,10 +69,6 @@ is always functional. You can also run the process locally with can be uploaded to Wicket to perform an upgrade of the rack while the control plane is not running. -`cargo xtask releng` performs all of these steps in parallel (with -the temporary exception of artifact downloads handled by -`tools/install_builder_prerequisites.sh`.) - == Beyond `cargo xtask releng` Currently we use TUF repos generated in CI (by `cargo xtask releng`)