From ea34ce90ef58de5f7bf3223bd47f2fffc30482f6 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Thu, 25 Jan 2024 22:36:33 -0500 Subject: [PATCH 01/10] Start work on podman-api cleanup --- .github/workflows/build.yml | 10 +++++----- Cargo.lock | 22 ++++++++++++++++++++++ Cargo.toml | 2 ++ Earthfile | 5 +++++ src/commands/build.rs | 5 ++++- src/commands/template.rs | 9 +++++++++ templates/Containerfile | 4 ++-- 7 files changed, 49 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 15757646..e90a05c7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,8 +2,6 @@ name: Earthly +build on: workflow_dispatch: - merge_group: - pull_request: push: branches: - main @@ -15,17 +13,19 @@ jobs: build: permissions: packages: write - timeout-minutes: 30 + timeout-minutes: 60 runs-on: ubuntu-latest steps: - uses: earthly/actions-setup@v1 with: use-cache: true - version: v0.8.0 + version: v0.8.2 # Setup repo and add caching - uses: actions/checkout@v4 + with: + ref: main - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -35,4 +35,4 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Run build - run: earthly --push --ci -P +all + run: earthly --push --ci -P +build diff --git a/Cargo.lock b/Cargo.lock index 7c32e8cc..1c5efefa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,6 +285,7 @@ dependencies = [ "chrono", "clap", "clap-verbosity-flag", + "derivative", "derive_builder", "env_logger", "futures-util", @@ -299,6 +300,7 @@ dependencies = [ "tokio", "typed-builder", "users", + "uuid", ] [[package]] @@ -738,6 +740,17 @@ dependencies = [ "serde", ] +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive_builder" version = "0.12.0" @@ -3262,6 +3275,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "uuid" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" +dependencies = [ + "getrandom", +] + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/Cargo.toml b/Cargo.toml index 399016b5..4bb25fc9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ cfg-if = "1.0.0" chrono = "0.4" clap = { version = "4", features = ["derive"] } clap-verbosity-flag = "2.1.1" +derivative = "2.2.0" derive_builder = "0.12.0" env_logger = "0.10.1" futures-util = { version = "0.3.30", optional = true } @@ -27,6 +28,7 @@ sigstore = { version = "0.8.0", optional = true } tokio = { version = "1", features = ["full"], optional = true } typed-builder = "0.18.0" users = "0.11.0" +uuid = { version = "1.7.0", features = ["v4"] } [features] default = [] diff --git a/Earthfile b/Earthfile index 31e301f2..3778007d 100644 --- a/Earthfile +++ b/Earthfile @@ -7,6 +7,11 @@ ARG --global IMAGE=ghcr.io/blue-build/cli all: BUILD +default BUILD +nightly + BUILD +integration-tests --NIGHTLY=true --NIGHTLY=false + +build: + BUILD +default + BUILD +nightly default: ARG NIGHTLY=false diff --git a/src/commands/build.rs b/src/commands/build.rs index 3aeef382..dd907ed5 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -9,8 +9,10 @@ use std::{ use anyhow::{anyhow, bail, Result}; use clap::Args; +use derivative::Derivative; use log::{debug, error, info, trace, warn}; use typed_builder::TypedBuilder; +use uuid::{self, Uuid}; #[cfg(feature = "podman-api")] use podman_api::{ @@ -35,7 +37,8 @@ use crate::{ use super::{template::Recipe, BlueBuildCommand}; -#[derive(Debug, Clone, Args, TypedBuilder)] +#[derive(Debug, Derivative, Clone, Args, TypedBuilder)] +#[derivative(Default)] pub struct BuildCommand { /// The recipe file to build an image #[arg()] diff --git a/src/commands/template.rs b/src/commands/template.rs index 345d9e52..6f89f084 100644 --- a/src/commands/template.rs +++ b/src/commands/template.rs @@ -9,11 +9,13 @@ use anyhow::{Error, Result}; use askama::Template; use chrono::Local; use clap::Args; +use derivative::Derivative; use indexmap::IndexMap; use log::{debug, error, info, trace, warn}; use serde::{Deserialize, Serialize}; use serde_yaml::Value; use typed_builder::TypedBuilder; +use uuid::Uuid; use super::BlueBuildCommand; @@ -25,6 +27,8 @@ pub struct ContainerFileTemplate<'a> { #[builder(default)] export_script: ExportsTemplate, + + build_id: Uuid, } #[derive(Debug, Clone, Default, Template)] @@ -168,6 +172,10 @@ pub struct TemplateCommand { #[arg(short, long)] #[builder(default, setter(into, strip_option))] output: Option, + + #[clap(skip)] + #[builder(default, setter(strip_option))] + build_id: Option, } impl BlueBuildCommand for TemplateCommand { @@ -189,6 +197,7 @@ impl TemplateCommand { let template = ContainerFileTemplate::builder() .recipe(&recipe_de) .recipe_path(&self.recipe) + .build_id(self.build_id.unwrap_or(Uuid::new_v4())) .build(); let output_str = template.render()?; diff --git a/templates/Containerfile b/templates/Containerfile index c0c5062d..70f4eada 100644 --- a/templates/Containerfile +++ b/templates/Containerfile @@ -3,8 +3,8 @@ FROM {{ recipe.base_image }}:{{ recipe.image_version }} LABEL org.opencontainers.image.title="{{ recipe.name }}" LABEL org.opencontainers.image.version="{{ recipe.image_version }}" LABEL org.opencontainers.image.description="{{ recipe.description }}" -LABEL io.artifacthub.package.readme-url=https://raw.githubusercontent.com/ublue-os/startingpoint/main/README.md -LABEL io.artifacthub.package.logo-url=https://avatars.githubusercontent.com/u/120078124?s=200&v=4 +LABEL io.blue-build.build-id={{ build_id }} +LABEL io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-build/cli/main/README.md ARG RECIPE={{ recipe_path.display() }} {%- if self::running_gitlab_actions() %} From 250bbfe4236fec14b2cc295d37abca2f340a0c01 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Thu, 25 Jan 2024 22:36:47 -0500 Subject: [PATCH 02/10] Add forgotten files --- .github/workflows/build-pr.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/build-pr.yml diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml new file mode 100644 index 00000000..6f30dd66 --- /dev/null +++ b/.github/workflows/build-pr.yml @@ -0,0 +1,27 @@ +name: Earthly +build + +on: + pull_request: + +env: + FORCE_COLOR: 1 + +jobs: + build: + timeout-minutes: 60 + runs-on: ubuntu-latest + + steps: + - uses: earthly/actions-setup@v1 + with: + use-cache: true + version: v0.8.2 + + # Setup repo and add caching + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.ref }} + + - name: Run build + run: earthly --ci -P +build + From 08e84ad36eb5d0dd1f3e51e4b754e21742e0e942 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Fri, 26 Jan 2024 00:02:58 -0500 Subject: [PATCH 03/10] Start bringing in signal hooks --- Cargo.lock | 24 +++++ Cargo.toml | 6 +- src/commands/build.rs | 57 +++++++++-- src/commands/local.rs | 3 +- src/commands/template.rs | 211 +------------------------------------- src/lib.rs | 1 + src/module_recipe.rs | 213 +++++++++++++++++++++++++++++++++++++++ src/ops.rs | 3 + templates/Containerfile | 1 - 9 files changed, 300 insertions(+), 219 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c5efefa..df62d14d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -296,6 +296,8 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "signal-hook", + "signal-hook-tokio", "sigstore", "tokio", "typed-builder", @@ -2693,6 +2695,16 @@ dependencies = [ "digest", ] +[[package]] +name = "signal-hook" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +dependencies = [ + "libc", + "signal-hook-registry", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -2702,6 +2714,18 @@ dependencies = [ "libc", ] +[[package]] +name = "signal-hook-tokio" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213241f76fb1e37e27de3b6aa1b068a2c333233b59cca6634f634b80a27ecf1e" +dependencies = [ + "futures-core", + "libc", + "signal-hook", + "tokio", +] + [[package]] name = "signature" version = "2.2.0" diff --git a/Cargo.toml b/Cargo.toml index 4bb25fc9..1edf0081 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,16 +24,18 @@ podman-api = { version = "0.10.0", optional = true } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9.25" +signal-hook = { version = "0.3.17", optional = true } +signal-hook-tokio = { version = "0.3.1", features = ["futures-v0_3"], optional = true } sigstore = { version = "0.8.0", optional = true } tokio = { version = "1", features = ["full"], optional = true } typed-builder = "0.18.0" users = "0.11.0" -uuid = { version = "1.7.0", features = ["v4"] } +uuid = { version = "1.7.0", features = ["v4"], optional = true } [features] default = [] nightly = ["builtin-podman"] -builtin-podman = ["podman-api", "tokio", "futures-util"] +builtin-podman = ["podman-api", "tokio", "futures-util", "uuid", "signal-hook-tokio", "signal-hook"] tls = ["podman-api/tls", "builtin-podman"] init = [] diff --git a/src/commands/build.rs b/src/commands/build.rs index dd907ed5..743ec6d0 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -8,11 +8,13 @@ use std::{ }; use anyhow::{anyhow, bail, Result}; -use clap::Args; +use clap::{builder, Args}; use derivative::Derivative; use log::{debug, error, info, trace, warn}; use typed_builder::TypedBuilder; -use uuid::{self, Uuid}; + +#[cfg(feature = "builtin-podman")] +use std::sync::Arc; #[cfg(feature = "podman-api")] use podman_api::{ @@ -24,18 +26,28 @@ use podman_api::{ #[cfg(feature = "podman-api")] use build_strategy::BuildStrategy; +#[cfg(feature = "signal-hook")] +use signal_hook::consts::*; + +#[cfg(feature = "signal-hook-tokio")] +use signal_hook_tokio::Signals; + #[cfg(feature = "futures-util")] use futures_util::StreamExt; #[cfg(feature = "tokio")] use tokio::runtime::Runtime; +#[cfg(feature = "uuid")] +use uuid::Uuid; + use crate::{ commands::template::TemplateCommand, + module_recipe::Recipe, ops::{self, ARCHIVE_SUFFIX}, }; -use super::{template::Recipe, BlueBuildCommand}; +use super::BlueBuildCommand; #[derive(Debug, Derivative, Clone, Args, TypedBuilder)] #[derivative(Default)] @@ -162,7 +174,7 @@ impl BlueBuildCommand for BuildCommand { info!("Building image for recipe at {}", self.recipe.display()); - #[cfg(feature = "podman-api")] + #[cfg(feature = "builtin-podman")] match BuildStrategy::determine_strategy()? { BuildStrategy::Socket(socket) => { Runtime::new()?.block_on(self.build_image_podman_api(Podman::unix(socket))) @@ -170,16 +182,18 @@ impl BlueBuildCommand for BuildCommand { _ => self.build_image(), } - #[cfg(not(feature = "podman-api"))] + #[cfg(not(feature = "builtin-podman"))] self.build_image() } } impl BuildCommand { - #[cfg(feature = "podman-api")] + #[cfg(feature = "builtin-podman")] async fn build_image_podman_api(&self, client: Podman) -> Result<()> { use podman_api::opts::ImageTagOpts; + use crate::ops::BUILD_ID_LABEL; + trace!("BuildCommand::build_image({client:#?})"); let (registry, username, password) = if self.push { @@ -236,12 +250,26 @@ impl BuildCommand { }; debug!("Full tag is {first_image_name}"); + // Prepare for the signal trap + let client = Arc::new(client); + let build_id = Arc::new(Uuid::new_v4()); + + let signals = Signals::new(&[SIGTERM, SIGINT, SIGQUIT])?; + let handle = signals.handle(); + + let signals_task = tokio::spawn(handle_signals( + signals, + Arc::clone(&build_id), + Arc::clone(&client), + )); + // Get podman ready to build let opts = ImageBuildOpts::builder(".") .tag(&first_image_name) .dockerfile("Containerfile") .remove(true) .layers(true) + .labels([(BUILD_ID_LABEL, build_id.to_string())]) .pull(true) .build(); trace!("Build options: {opts:#?}"); @@ -320,6 +348,10 @@ impl BuildCommand { sign_images(&image_name, tags.first().map(String::as_str))?; } + + handle.close(); + signals_task.await?; + Ok(()) } @@ -801,3 +833,16 @@ fn check_cosign_files() -> Result<()> { } } } + +#[cfg(feature = "builtin-podman")] +async fn handle_signals(mut signals: Signals, _build_id: Arc, _client: Arc) { + while let Some(signal) = signals.next().await { + match signal { + SIGTERM | SIGINT | SIGQUIT => { + // Shutdown the system; + todo!("Clean out working containers") + } + _ => unreachable!(), + } + } +} diff --git a/src/commands/local.rs b/src/commands/local.rs index a9d3a6df..e4f8cebc 100644 --- a/src/commands/local.rs +++ b/src/commands/local.rs @@ -11,7 +11,8 @@ use typed_builder::TypedBuilder; use users::{Users, UsersCache}; use crate::{ - commands::{build::BuildCommand, template::Recipe}, + commands::build::BuildCommand, + module_recipe::Recipe, ops::{self, ARCHIVE_SUFFIX, LOCAL_BUILD}, }; diff --git a/src/commands/template.rs b/src/commands/template.rs index 6f89f084..bfe12928 100644 --- a/src/commands/template.rs +++ b/src/commands/template.rs @@ -15,7 +15,8 @@ use log::{debug, error, info, trace, warn}; use serde::{Deserialize, Serialize}; use serde_yaml::Value; use typed_builder::TypedBuilder; -use uuid::Uuid; + +use crate::module_recipe::Recipe; use super::BlueBuildCommand; @@ -27,140 +28,12 @@ pub struct ContainerFileTemplate<'a> { #[builder(default)] export_script: ExportsTemplate, - - build_id: Uuid, } #[derive(Debug, Clone, Default, Template)] #[template(path = "export.sh", escape = "none")] pub struct ExportsTemplate; -#[derive(Serialize, Clone, Deserialize, Debug, TypedBuilder)] -pub struct Recipe { - #[builder(setter(into))] - pub name: String, - - #[builder(setter(into))] - pub description: String, - - #[serde(alias = "base-image")] - #[builder(setter(into))] - pub base_image: String, - - #[serde(alias = "image-version")] - #[builder(setter(into))] - pub image_version: String, - - #[serde(alias = "blue-build-tag")] - #[builder(default, setter(into, strip_option))] - pub blue_build_tag: Option, - - #[serde(flatten)] - pub modules_ext: ModuleExt, - - #[serde(flatten)] - #[builder(setter(into))] - pub extra: IndexMap, -} - -impl Recipe { - #[must_use] - pub fn generate_tags(&self) -> Vec { - trace!("Recipe::generate_tags()"); - debug!("Generating image tags for {}", &self.name); - - let mut tags: Vec = Vec::new(); - let image_version = &self.image_version; - let timestamp = Local::now().format("%Y%m%d").to_string(); - - if let (Ok(commit_branch), Ok(default_branch), Ok(commit_sha), Ok(pipeline_source)) = ( - env::var("CI_COMMIT_REF_NAME"), - env::var("CI_DEFAULT_BRANCH"), - env::var("CI_COMMIT_SHORT_SHA"), - env::var("CI_PIPELINE_SOURCE"), - ) { - trace!("CI_COMMIT_REF_NAME={commit_branch}, CI_DEFAULT_BRANCH={default_branch},CI_COMMIT_SHORT_SHA={commit_sha}, CI_PIPELINE_SOURCE={pipeline_source}"); - warn!("Detected running in Gitlab, pulling information from CI variables"); - - if let Ok(mr_iid) = env::var("CI_MERGE_REQUEST_IID") { - trace!("CI_MERGE_REQUEST_IID={mr_iid}"); - if pipeline_source == "merge_request_event" { - debug!("Running in a MR"); - tags.push(format!("mr-{mr_iid}-{image_version}")); - } - } - - if default_branch == commit_branch { - debug!("Running on the default branch"); - tags.push(image_version.to_string()); - tags.push(format!("{image_version}-{timestamp}")); - tags.push(timestamp); - } else { - debug!("Running on branch {commit_branch}"); - tags.push(format!("{commit_branch}-{image_version}")); - } - - tags.push(format!("{commit_sha}-{image_version}")); - } else if let ( - Ok(github_event_name), - Ok(github_event_number), - Ok(github_sha), - Ok(github_ref_name), - ) = ( - env::var("GITHUB_EVENT_NAME"), - env::var("PR_EVENT_NUMBER"), - env::var("GITHUB_SHA"), - env::var("GITHUB_REF_NAME"), - ) { - trace!("GITHUB_EVENT_NAME={github_event_name},PR_EVENT_NUMBER={github_event_number},GITHUB_SHA={github_sha},GITHUB_REF_NAME={github_ref_name}"); - warn!("Detected running in Github, pulling information from GITHUB variables"); - - let mut short_sha = github_sha; - short_sha.truncate(7); - - if github_event_name == "pull_request" { - debug!("Running in a PR"); - tags.push(format!("pr-{github_event_number}-{image_version}")); - } else if github_ref_name == "live" { - tags.push(image_version.to_owned()); - tags.push(format!("{image_version}-{timestamp}")); - tags.push("latest".to_string()); - } else { - tags.push(format!("br-{github_ref_name}-{image_version}")); - } - tags.push(format!("{short_sha}-{image_version}")); - } else { - warn!("Running locally"); - tags.push(format!("{image_version}-local")); - } - info!("Finished generating tags!"); - debug!("Tags: {tags:#?}"); - tags - } -} - -#[derive(Serialize, Clone, Deserialize, Debug, Template, TypedBuilder)] -#[template(path = "Containerfile.module", escape = "none")] -pub struct ModuleExt { - #[builder(default, setter(into))] - pub modules: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder)] -pub struct Module { - #[serde(rename = "type")] - #[builder(default, setter(into, strip_option))] - pub module_type: Option, - - #[serde(rename = "from-file")] - #[builder(default, setter(into, strip_option))] - pub from_file: Option, - - #[serde(flatten)] - #[builder(default, setter(into))] - pub config: IndexMap, -} - #[derive(Debug, Clone, Args, TypedBuilder)] pub struct TemplateCommand { /// The recipe file to create a template from @@ -172,10 +45,6 @@ pub struct TemplateCommand { #[arg(short, long)] #[builder(default, setter(into, strip_option))] output: Option, - - #[clap(skip)] - #[builder(default, setter(strip_option))] - build_id: Option, } impl BlueBuildCommand for TemplateCommand { @@ -197,7 +66,6 @@ impl TemplateCommand { let template = ContainerFileTemplate::builder() .recipe(&recipe_de) .recipe_path(&self.recipe) - .build_id(self.build_id.unwrap_or(Uuid::new_v4())) .build(); let output_str = template.render()?; @@ -237,81 +105,6 @@ fn print_script(script_contents: &ExportsTemplate) -> String { ) } -fn get_containerfile_list(module: &Module) -> Option> { - if module.module_type.as_ref()? == "containerfile" { - Some( - module - .config - .get("containerfiles")? - .as_sequence()? - .iter() - .filter_map(|t| Some(t.as_str()?.to_owned())) - .collect(), - ) - } else { - None - } -} - -fn print_containerfile(containerfile: &str) -> String { - trace!("print_containerfile({containerfile})"); - debug!("Loading containerfile contents for {containerfile}"); - - let path = format!("config/containerfiles/{containerfile}/Containerfile"); - - let file = fs::read_to_string(&path).unwrap_or_else(|e| { - error!("Failed to read file {path}: {e}"); - process::exit(1); - }); - - trace!("Containerfile contents {path}:\n{file}"); - - file -} - -fn get_module_from_file(file_name: &str) -> String { - trace!("get_module_from_file({file_name})"); - - let io_err_fn = |e| { - error!("Failed to read module {file_name}: {e}"); - process::exit(1); - }; - - let file_path = PathBuf::from("config").join(file_name); - - let file = fs::read_to_string(file_path).unwrap_or_else(io_err_fn); - - let serde_err_fn = |e| { - error!("Failed to deserialize module {file_name}: {e}"); - process::exit(1); - }; - - let template_err_fn = |e| { - error!("Failed to render module {file_name}: {e}"); - process::exit(1); - }; - - serde_yaml::from_str::(file.as_str()).map_or_else( - |_| { - let module = serde_yaml::from_str::(file.as_str()).unwrap_or_else(serde_err_fn); - - ModuleExt::builder() - .modules(vec![module]) - .build() - .render() - .unwrap_or_else(template_err_fn) - }, - |module_ext| module_ext.render().unwrap_or_else(template_err_fn), - ) -} - -fn print_module_context(module: &Module) -> String { - serde_json::to_string(module).unwrap_or_else(|e| { - error!("Failed to parse module: {e}"); - process::exit(1); - }) -} - fn running_gitlab_actions() -> bool { trace!(" running_gitlab_actions()"); diff --git a/src/lib.rs b/src/lib.rs index 740a0ffc..82642f81 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,4 +9,5 @@ #![allow(clippy::module_name_repetitions)] pub mod commands; +pub mod module_recipe; mod ops; diff --git a/src/module_recipe.rs b/src/module_recipe.rs index 8b137891..818428b9 100644 --- a/src/module_recipe.rs +++ b/src/module_recipe.rs @@ -1 +1,214 @@ +use std::{env, fs, path::PathBuf, process}; +use askama::Template; +use chrono::Local; +use indexmap::IndexMap; +use log::{debug, error, info, trace, warn}; +use serde::{Deserialize, Serialize}; +use serde_yaml::Value; +use typed_builder::TypedBuilder; + +#[derive(Serialize, Clone, Deserialize, Debug, TypedBuilder)] +pub struct Recipe { + #[builder(setter(into))] + pub name: String, + + #[builder(setter(into))] + pub description: String, + + #[serde(alias = "base-image")] + #[builder(setter(into))] + pub base_image: String, + + #[serde(alias = "image-version")] + #[builder(setter(into))] + pub image_version: String, + + #[serde(alias = "blue-build-tag")] + #[builder(default, setter(into, strip_option))] + pub blue_build_tag: Option, + + #[serde(flatten)] + pub modules_ext: ModuleExt, + + #[serde(flatten)] + #[builder(setter(into))] + pub extra: IndexMap, +} + +impl Recipe { + #[must_use] + pub fn generate_tags(&self) -> Vec { + trace!("Recipe::generate_tags()"); + debug!("Generating image tags for {}", &self.name); + + let mut tags: Vec = Vec::new(); + let image_version = &self.image_version; + let timestamp = Local::now().format("%Y%m%d").to_string(); + + if let (Ok(commit_branch), Ok(default_branch), Ok(commit_sha), Ok(pipeline_source)) = ( + env::var("CI_COMMIT_REF_NAME"), + env::var("CI_DEFAULT_BRANCH"), + env::var("CI_COMMIT_SHORT_SHA"), + env::var("CI_PIPELINE_SOURCE"), + ) { + trace!("CI_COMMIT_REF_NAME={commit_branch}, CI_DEFAULT_BRANCH={default_branch},CI_COMMIT_SHORT_SHA={commit_sha}, CI_PIPELINE_SOURCE={pipeline_source}"); + warn!("Detected running in Gitlab, pulling information from CI variables"); + + if let Ok(mr_iid) = env::var("CI_MERGE_REQUEST_IID") { + trace!("CI_MERGE_REQUEST_IID={mr_iid}"); + if pipeline_source == "merge_request_event" { + debug!("Running in a MR"); + tags.push(format!("mr-{mr_iid}-{image_version}")); + } + } + + if default_branch == commit_branch { + debug!("Running on the default branch"); + tags.push(image_version.to_string()); + tags.push(format!("{image_version}-{timestamp}")); + tags.push(timestamp); + } else { + debug!("Running on branch {commit_branch}"); + tags.push(format!("{commit_branch}-{image_version}")); + } + + tags.push(format!("{commit_sha}-{image_version}")); + } else if let ( + Ok(github_event_name), + Ok(github_event_number), + Ok(github_sha), + Ok(github_ref_name), + ) = ( + env::var("GITHUB_EVENT_NAME"), + env::var("PR_EVENT_NUMBER"), + env::var("GITHUB_SHA"), + env::var("GITHUB_REF_NAME"), + ) { + trace!("GITHUB_EVENT_NAME={github_event_name},PR_EVENT_NUMBER={github_event_number},GITHUB_SHA={github_sha},GITHUB_REF_NAME={github_ref_name}"); + warn!("Detected running in Github, pulling information from GITHUB variables"); + + let mut short_sha = github_sha; + short_sha.truncate(7); + + if github_event_name == "pull_request" { + debug!("Running in a PR"); + tags.push(format!("pr-{github_event_number}-{image_version}")); + } else if github_ref_name == "live" { + tags.push(image_version.to_owned()); + tags.push(format!("{image_version}-{timestamp}")); + tags.push("latest".to_string()); + } else { + tags.push(format!("br-{github_ref_name}-{image_version}")); + } + tags.push(format!("{short_sha}-{image_version}")); + } else { + warn!("Running locally"); + tags.push(format!("{image_version}-local")); + } + info!("Finished generating tags!"); + debug!("Tags: {tags:#?}"); + tags + } +} + +#[derive(Serialize, Clone, Deserialize, Debug, Template, TypedBuilder)] +#[template(path = "Containerfile.module", escape = "none")] +pub struct ModuleExt { + #[builder(default, setter(into))] + pub modules: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder)] +pub struct Module { + #[serde(rename = "type")] + #[builder(default, setter(into, strip_option))] + pub module_type: Option, + + #[serde(rename = "from-file")] + #[builder(default, setter(into, strip_option))] + pub from_file: Option, + + #[serde(flatten)] + #[builder(default, setter(into))] + pub config: IndexMap, +} + +// ======================================================== // +// ========================= Helpers ====================== // +// ======================================================== // + +fn get_containerfile_list(module: &Module) -> Option> { + if module.module_type.as_ref()? == "containerfile" { + Some( + module + .config + .get("containerfiles")? + .as_sequence()? + .iter() + .filter_map(|t| Some(t.as_str()?.to_owned())) + .collect(), + ) + } else { + None + } +} + +fn print_containerfile(containerfile: &str) -> String { + trace!("print_containerfile({containerfile})"); + debug!("Loading containerfile contents for {containerfile}"); + + let path = format!("config/containerfiles/{containerfile}/Containerfile"); + + let file = fs::read_to_string(&path).unwrap_or_else(|e| { + error!("Failed to read file {path}: {e}"); + process::exit(1); + }); + + trace!("Containerfile contents {path}:\n{file}"); + + file +} + +fn get_module_from_file(file_name: &str) -> String { + trace!("get_module_from_file({file_name})"); + + let io_err_fn = |e| { + error!("Failed to read module {file_name}: {e}"); + process::exit(1); + }; + + let file_path = PathBuf::from("config").join(file_name); + + let file = fs::read_to_string(file_path).unwrap_or_else(io_err_fn); + + let serde_err_fn = |e| { + error!("Failed to deserialize module {file_name}: {e}"); + process::exit(1); + }; + + let template_err_fn = |e| { + error!("Failed to render module {file_name}: {e}"); + process::exit(1); + }; + + serde_yaml::from_str::(file.as_str()).map_or_else( + |_| { + let module = serde_yaml::from_str::(file.as_str()).unwrap_or_else(serde_err_fn); + + ModuleExt::builder() + .modules(vec![module]) + .build() + .render() + .unwrap_or_else(template_err_fn) + }, + |module_ext| module_ext.render().unwrap_or_else(template_err_fn), + ) +} + +fn print_module_context(module: &Module) -> String { + serde_json::to_string(module).unwrap_or_else(|e| { + error!("Failed to parse module: {e}"); + process::exit(1); + }) +} diff --git a/src/ops.rs b/src/ops.rs index dc296b09..1fc415f5 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -5,6 +5,9 @@ use std::{path::Path, process::Command}; pub const LOCAL_BUILD: &str = "/etc/blue-build"; pub const ARCHIVE_SUFFIX: &str = "tar.gz"; +#[cfg(feature = "podman-api")] +pub const BUILD_ID_LABEL: &str = "io.blue-build.build-id"; + pub fn check_command_exists(command: &str) -> Result<()> { trace!("check_command_exists({command})"); debug!("Checking if {command} exists"); diff --git a/templates/Containerfile b/templates/Containerfile index 70f4eada..84aedf5c 100644 --- a/templates/Containerfile +++ b/templates/Containerfile @@ -3,7 +3,6 @@ FROM {{ recipe.base_image }}:{{ recipe.image_version }} LABEL org.opencontainers.image.title="{{ recipe.name }}" LABEL org.opencontainers.image.version="{{ recipe.image_version }}" LABEL org.opencontainers.image.description="{{ recipe.description }}" -LABEL io.blue-build.build-id={{ build_id }} LABEL io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-build/cli/main/README.md ARG RECIPE={{ recipe_path.display() }} From 64f9f0789e0fcb164e4341b2f1f991ad7e7d0436 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Sat, 27 Jan 2024 11:42:29 -0500 Subject: [PATCH 04/10] Fix clippy errors --- src/commands/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/build.rs b/src/commands/build.rs index 743ec6d0..7fea520a 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -254,7 +254,7 @@ impl BuildCommand { let client = Arc::new(client); let build_id = Arc::new(Uuid::new_v4()); - let signals = Signals::new(&[SIGTERM, SIGINT, SIGQUIT])?; + let signals = Signals::new([SIGTERM, SIGINT, SIGQUIT])?; let handle = signals.handle(); let signals_task = tokio::spawn(handle_signals( @@ -842,7 +842,7 @@ async fn handle_signals(mut signals: Signals, _build_id: Arc, _client: Arc // Shutdown the system; todo!("Clean out working containers") } - _ => unreachable!(), + _ => (), } } } From f2613d39ab88dc6bde6a8fa290d9553e00514253 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Sat, 27 Jan 2024 12:14:36 -0500 Subject: [PATCH 05/10] Remove derivative crate --- Cargo.lock | 12 ------------ Cargo.toml | 1 - src/commands/build.rs | 4 +--- src/commands/template.rs | 1 - 4 files changed, 1 insertion(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3baed9ee..814ffb78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,7 +285,6 @@ dependencies = [ "chrono", "clap", "clap-verbosity-flag", - "derivative", "derive_builder", "env_logger", "futures-util", @@ -742,17 +741,6 @@ dependencies = [ "serde", ] -[[package]] -name = "derivative" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "derive_builder" version = "0.12.0" diff --git a/Cargo.toml b/Cargo.toml index 3bd59425..971c695d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,6 @@ cfg-if = "1.0.0" chrono = "0.4" clap = { version = "4", features = ["derive"] } clap-verbosity-flag = "2.1.1" -derivative = "2.2.0" derive_builder = "0.12.0" env_logger = "0.10.1" futures-util = { version = "0.3.30", optional = true } diff --git a/src/commands/build.rs b/src/commands/build.rs index 7fea520a..c852d33d 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -9,7 +9,6 @@ use std::{ use anyhow::{anyhow, bail, Result}; use clap::{builder, Args}; -use derivative::Derivative; use log::{debug, error, info, trace, warn}; use typed_builder::TypedBuilder; @@ -49,8 +48,7 @@ use crate::{ use super::BlueBuildCommand; -#[derive(Debug, Derivative, Clone, Args, TypedBuilder)] -#[derivative(Default)] +#[derive(Debug, Clone, Args, TypedBuilder)] pub struct BuildCommand { /// The recipe file to build an image #[arg()] diff --git a/src/commands/template.rs b/src/commands/template.rs index bfe12928..8d7aa222 100644 --- a/src/commands/template.rs +++ b/src/commands/template.rs @@ -9,7 +9,6 @@ use anyhow::{Error, Result}; use askama::Template; use chrono::Local; use clap::Args; -use derivative::Derivative; use indexmap::IndexMap; use log::{debug, error, info, trace, warn}; use serde::{Deserialize, Serialize}; From 14e1b6f4891f71f193a1a683f0e159a1a0147361 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Sat, 27 Jan 2024 12:52:15 -0500 Subject: [PATCH 06/10] Remove duplicate build target --- Earthfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Earthfile b/Earthfile index 4170955b..c924376f 100644 --- a/Earthfile +++ b/Earthfile @@ -8,11 +8,6 @@ all: BUILD +build BUILD +integration-tests --NIGHTLY=true --NIGHTLY=false -build: - BUILD +default - BUILD +nightly - BUILD +integration-tests --NIGHTLY=true --NIGHTLY=false - build: BUILD +default BUILD +nightly From f39fbea3a12d70520f99763f063b9d84f3b2d7f7 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Sun, 28 Jan 2024 15:17:31 -0500 Subject: [PATCH 07/10] Convert templates to use Cow --- src/commands/template.rs | 9 ++++++--- src/module_recipe.rs | 32 ++++++++++++++++---------------- src/ops.rs | 3 +-- templates/Containerfile | 1 + 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/commands/template.rs b/src/commands/template.rs index 8d7aa222..9ec31bfb 100644 --- a/src/commands/template.rs +++ b/src/commands/template.rs @@ -1,4 +1,5 @@ use std::{ + borrow::Cow, collections::HashMap, env, fs, path::{Path, PathBuf}, @@ -22,8 +23,10 @@ use super::BlueBuildCommand; #[derive(Debug, Clone, Template, TypedBuilder)] #[template(path = "Containerfile")] pub struct ContainerFileTemplate<'a> { - recipe: &'a Recipe, - recipe_path: &'a Path, + recipe: Recipe<'a>, + + #[builder(setter(into))] + recipe_path: Cow<'a, Path>, #[builder(default)] export_script: ExportsTemplate, @@ -63,7 +66,7 @@ impl TemplateCommand { trace!("recipe_de: {recipe_de:#?}"); let template = ContainerFileTemplate::builder() - .recipe(&recipe_de) + .recipe(recipe_de) .recipe_path(&self.recipe) .build(); diff --git a/src/module_recipe.rs b/src/module_recipe.rs index 818428b9..2c4bc57e 100644 --- a/src/module_recipe.rs +++ b/src/module_recipe.rs @@ -1,4 +1,4 @@ -use std::{env, fs, path::PathBuf, process}; +use std::{borrow::Cow, env, fs, path::PathBuf, process}; use askama::Template; use chrono::Local; @@ -9,41 +9,41 @@ use serde_yaml::Value; use typed_builder::TypedBuilder; #[derive(Serialize, Clone, Deserialize, Debug, TypedBuilder)] -pub struct Recipe { +pub struct Recipe<'a> { #[builder(setter(into))] - pub name: String, + pub name: Cow<'a, str>, #[builder(setter(into))] - pub description: String, + pub description: Cow<'a, str>, #[serde(alias = "base-image")] #[builder(setter(into))] - pub base_image: String, + pub base_image: Cow<'a, str>, #[serde(alias = "image-version")] #[builder(setter(into))] - pub image_version: String, + pub image_version: Cow<'a, str>, #[serde(alias = "blue-build-tag")] #[builder(default, setter(into, strip_option))] - pub blue_build_tag: Option, + pub blue_build_tag: Option>, #[serde(flatten)] - pub modules_ext: ModuleExt, + pub modules_ext: ModuleExt<'a>, #[serde(flatten)] #[builder(setter(into))] pub extra: IndexMap, } -impl Recipe { +impl<'a> Recipe<'a> { #[must_use] pub fn generate_tags(&self) -> Vec { trace!("Recipe::generate_tags()"); debug!("Generating image tags for {}", &self.name); let mut tags: Vec = Vec::new(); - let image_version = &self.image_version; + let image_version = self.image_version.as_ref(); let timestamp = Local::now().format("%Y%m%d").to_string(); if let (Ok(commit_branch), Ok(default_branch), Ok(commit_sha), Ok(pipeline_source)) = ( @@ -95,7 +95,7 @@ impl Recipe { debug!("Running in a PR"); tags.push(format!("pr-{github_event_number}-{image_version}")); } else if github_ref_name == "live" { - tags.push(image_version.to_owned()); + tags.push(image_version.to_string()); tags.push(format!("{image_version}-{timestamp}")); tags.push("latest".to_string()); } else { @@ -114,20 +114,20 @@ impl Recipe { #[derive(Serialize, Clone, Deserialize, Debug, Template, TypedBuilder)] #[template(path = "Containerfile.module", escape = "none")] -pub struct ModuleExt { +pub struct ModuleExt<'a> { #[builder(default, setter(into))] - pub modules: Vec, + pub modules: Cow<'a, [Module<'a>]>, } #[derive(Serialize, Deserialize, Debug, Clone, TypedBuilder)] -pub struct Module { +pub struct Module<'a> { #[serde(rename = "type")] #[builder(default, setter(into, strip_option))] - pub module_type: Option, + pub module_type: Option>, #[serde(rename = "from-file")] #[builder(default, setter(into, strip_option))] - pub from_file: Option, + pub from_file: Option>, #[serde(flatten)] #[builder(default, setter(into))] diff --git a/src/ops.rs b/src/ops.rs index 1fc415f5..6a316605 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -5,8 +5,7 @@ use std::{path::Path, process::Command}; pub const LOCAL_BUILD: &str = "/etc/blue-build"; pub const ARCHIVE_SUFFIX: &str = "tar.gz"; -#[cfg(feature = "podman-api")] -pub const BUILD_ID_LABEL: &str = "io.blue-build.build-id"; +pub const BUILD_ID_LABEL: &str = "org.blue-build.build-id"; pub fn check_command_exists(command: &str) -> Result<()> { trace!("check_command_exists({command})"); diff --git a/templates/Containerfile b/templates/Containerfile index 84aedf5c..69a2ec6c 100644 --- a/templates/Containerfile +++ b/templates/Containerfile @@ -1,5 +1,6 @@ FROM {{ recipe.base_image }}:{{ recipe.image_version }} +LABEL {{ crate::ops::BUILD_ID_LABEL }}="" LABEL org.opencontainers.image.title="{{ recipe.name }}" LABEL org.opencontainers.image.version="{{ recipe.image_version }}" LABEL org.opencontainers.image.description="{{ recipe.description }}" From caee0fe8f2c7982252963e9beaebb537d381f52d Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Sun, 28 Jan 2024 17:05:51 -0500 Subject: [PATCH 08/10] Handle cleanup --- Cargo.toml | 4 +- src/commands/build.rs | 117 ++++++++++++++++++++++++++++----------- src/commands/template.rs | 17 +++++- src/ops.rs | 1 - templates/Containerfile | 2 +- 5 files changed, 105 insertions(+), 36 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 971c695d..22d7d3be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,12 +29,12 @@ sigstore = { version = "0.8.0", optional = true } tokio = { version = "1", features = ["full"], optional = true } typed-builder = "0.18.0" users = "0.11.0" -uuid = { version = "1.7.0", features = ["v4"], optional = true } +uuid = { version = "1.7.0", features = ["v4"] } [features] default = [] nightly = ["builtin-podman"] -builtin-podman = ["podman-api", "tokio", "futures-util", "uuid", "signal-hook-tokio", "signal-hook"] +builtin-podman = ["podman-api", "tokio", "futures-util", "signal-hook-tokio", "signal-hook"] tls = ["podman-api/tls", "builtin-podman"] init = [] diff --git a/src/commands/build.rs b/src/commands/build.rs index 7d78dd47..f848958c 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -11,6 +11,7 @@ use anyhow::{anyhow, bail, Result}; use clap::{builder, Args}; use log::{debug, error, info, trace, warn}; use typed_builder::TypedBuilder; +use uuid::Uuid; #[cfg(feature = "builtin-podman")] use std::sync::Arc; @@ -32,15 +33,15 @@ use signal_hook_tokio::Signals; use futures_util::StreamExt; #[cfg(feature = "tokio")] -use tokio::runtime::Runtime; - -#[cfg(feature = "uuid")] -use uuid::Uuid; +use tokio::{ + runtime::Runtime, + sync::oneshot::{self, Sender}, +}; use crate::{ commands::template::TemplateCommand, module_recipe::Recipe, - ops::{self, ARCHIVE_SUFFIX}, + ops::{self, ARCHIVE_SUFFIX, BUILD_ID_LABEL}, }; use super::BlueBuildCommand; @@ -151,6 +152,8 @@ impl BlueBuildCommand for BuildCommand { fn try_run(&mut self) -> Result<()> { trace!("BuildCommand::try_run()"); + let build_id = Uuid::new_v4(); + if self.push && self.archive.is_some() { bail!("You cannot use '--archive' and '--push' at the same time"); } @@ -171,6 +174,7 @@ impl BlueBuildCommand for BuildCommand { TemplateCommand::builder() .recipe(self.recipe.clone()) .output(PathBuf::from("Containerfile")) + .build_id(build_id) .build() .try_run()?; @@ -178,9 +182,8 @@ impl BlueBuildCommand for BuildCommand { #[cfg(feature = "builtin-podman")] match BuildStrategy::determine_strategy()? { - BuildStrategy::Socket(socket) => { - Runtime::new()?.block_on(self.build_image_podman_api(Podman::unix(socket))) - } + BuildStrategy::Socket(socket) => Runtime::new()? + .block_on(self.build_image_podman_api(Podman::unix(socket), build_id)), _ => self.build_image(), } @@ -191,7 +194,7 @@ impl BlueBuildCommand for BuildCommand { impl BuildCommand { #[cfg(feature = "builtin-podman")] - async fn build_image_podman_api(&self, client: Podman) -> Result<()> { + async fn build_image_podman_api(&self, client: Podman, build_id: Uuid) -> Result<()> { use podman_api::opts::ImageTagOpts; use signal_hook::consts::{SIGINT, SIGQUIT, SIGTERM}; @@ -222,15 +225,17 @@ impl BuildCommand { // Prepare for the signal trap let client = Arc::new(client); - let build_id = Arc::new(Uuid::new_v4()); let signals = Signals::new([SIGTERM, SIGINT, SIGQUIT])?; let handle = signals.handle(); + let (kill_tx, mut kill_rx) = oneshot::channel::<()>(); + let signals_task = tokio::spawn(handle_signals( signals, - Arc::clone(&build_id), - Arc::clone(&client), + kill_tx, + build_id.clone(), + client.clone(), )); // Get podman ready to build @@ -246,20 +251,25 @@ impl BuildCommand { info!("Building image {first_image_name}"); match client.images().build(&opts) { - Ok(mut build_stream) => { - while let Some(chunk) = build_stream.next().await { - match chunk { - Ok(chunk) => chunk - .stream - .trim() - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .for_each(|line| info!("{line}")), - Err(e) => bail!("{e}"), + Ok(mut build_stream) => loop { + tokio::select! { + Some(chunk) = build_stream.next() => { + match chunk { + Ok(chunk) => chunk + .stream + .trim() + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .for_each(|line| info!("{line}")), + Err(e) => bail!("{e}"), + } + }, + _ = &mut kill_rx => { + break; } } - } + }, Err(e) => bail!("{e}"), }; @@ -299,7 +309,7 @@ impl BuildCommand { } handle.close(); - signals_task.await?; + signals_task.await??; Ok(()) } @@ -739,18 +749,63 @@ fn check_cosign_files() -> Result<()> { } #[cfg(feature = "builtin-podman")] -async fn handle_signals(mut signals: Signals, _build_id: Arc, _client: Arc) { - use signal_hook::consts::{SIGINT, SIGQUIT, SIGTERM}; +async fn handle_signals( + mut signals: Signals, + kill: Sender<()>, + build_id: Uuid, + client: Arc, +) -> Result<()> { + use podman_api::opts::{ + ContainerListOpts, ContainerPruneFilter, ContainerPruneOpts, ImagePruneFilter, + ImagePruneOpts, + }; + use signal_hook::consts::{SIGHUP, SIGINT}; + use tokio::time::{self, Duration}; + + trace!("handle_signals(signals, {build_id}, {client:#?})"); while let Some(signal) = signals.next().await { match signal { - SIGTERM | SIGINT | SIGQUIT => { - // Shutdown the system; - todo!("Clean out working containers") + SIGHUP => (), + SIGINT => { + kill.send(()).unwrap(); + info!("Recieved SIGINT, cleaning up build..."); + + time::sleep(Duration::from_secs(1)).await; + + let containers = client + .containers() + .list(&ContainerListOpts::builder().sync(true).all(true).build()) + .await?; + + trace!("{containers:#?}"); + + // Prune containers from this build + let container_prune_opts = ContainerPruneOpts::builder() + .filter([ContainerPruneFilter::LabelKeyVal( + BUILD_ID_LABEL.to_string(), + build_id.to_string(), + )]) + .build(); + client.containers().prune(&container_prune_opts).await?; + debug!("Pruned containers"); + + // Prune images from this build + let image_prune_opts = ImagePruneOpts::builder() + .filter([ImagePruneFilter::LabelKeyVal( + BUILD_ID_LABEL.to_string(), + build_id.to_string(), + )]) + .build(); + client.images().prune(&image_prune_opts).await?; + debug!("Pruned images"); + process::exit(2); } - _ => (), + _ => unreachable!(), } } + + Ok(()) } fn tag_images(tags: &[String], image_name: &str, full_image: &str) -> Result<()> { diff --git a/src/commands/template.rs b/src/commands/template.rs index 9ec31bfb..0f2dc6ff 100644 --- a/src/commands/template.rs +++ b/src/commands/template.rs @@ -6,7 +6,7 @@ use std::{ process, }; -use anyhow::{Error, Result}; +use anyhow::{anyhow, Error, Result}; use askama::Template; use chrono::Local; use clap::Args; @@ -15,6 +15,7 @@ use log::{debug, error, info, trace, warn}; use serde::{Deserialize, Serialize}; use serde_yaml::Value; use typed_builder::TypedBuilder; +use uuid::Uuid; use crate::module_recipe::Recipe; @@ -28,6 +29,9 @@ pub struct ContainerFileTemplate<'a> { #[builder(setter(into))] recipe_path: Cow<'a, Path>, + #[builder(setter(into))] + build_id: Uuid, + #[builder(default)] export_script: ExportsTemplate, } @@ -47,12 +51,18 @@ pub struct TemplateCommand { #[arg(short, long)] #[builder(default, setter(into, strip_option))] output: Option, + + #[clap(skip)] + #[builder(default, setter(into, strip_option))] + build_id: Option, } impl BlueBuildCommand for TemplateCommand { fn try_run(&mut self) -> Result<()> { info!("Templating for recipe at {}", self.recipe.display()); + self.build_id.get_or_insert(Uuid::new_v4()); + self.template_file() } } @@ -65,9 +75,14 @@ impl TemplateCommand { let recipe_de = serde_yaml::from_str::(fs::read_to_string(&self.recipe)?.as_str())?; trace!("recipe_de: {recipe_de:#?}"); + let build_id = self + .build_id + .ok_or(anyhow!("Build ID should have been generated by now"))?; + let template = ContainerFileTemplate::builder() .recipe(recipe_de) .recipe_path(&self.recipe) + .build_id(build_id) .build(); let output_str = template.render()?; diff --git a/src/ops.rs b/src/ops.rs index 6a316605..de9e1730 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -4,7 +4,6 @@ use std::{path::Path, process::Command}; pub const LOCAL_BUILD: &str = "/etc/blue-build"; pub const ARCHIVE_SUFFIX: &str = "tar.gz"; - pub const BUILD_ID_LABEL: &str = "org.blue-build.build-id"; pub fn check_command_exists(command: &str) -> Result<()> { diff --git a/templates/Containerfile b/templates/Containerfile index 69a2ec6c..1de6a240 100644 --- a/templates/Containerfile +++ b/templates/Containerfile @@ -1,6 +1,6 @@ FROM {{ recipe.base_image }}:{{ recipe.image_version }} -LABEL {{ crate::ops::BUILD_ID_LABEL }}="" +LABEL {{ crate::ops::BUILD_ID_LABEL }}="{{ build_id }}" LABEL org.opencontainers.image.title="{{ recipe.name }}" LABEL org.opencontainers.image.version="{{ recipe.image_version }}" LABEL org.opencontainers.image.description="{{ recipe.description }}" From a86043829e5dc5b1a0cfd697898f5381c5650925 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Sun, 28 Jan 2024 17:14:13 -0500 Subject: [PATCH 09/10] Fix lint --- src/commands/build.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/commands/build.rs b/src/commands/build.rs index f848958c..3f7e79d2 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -231,12 +231,7 @@ impl BuildCommand { let (kill_tx, mut kill_rx) = oneshot::channel::<()>(); - let signals_task = tokio::spawn(handle_signals( - signals, - kill_tx, - build_id.clone(), - client.clone(), - )); + let signals_task = tokio::spawn(handle_signals(signals, kill_tx, build_id, client.clone())); // Get podman ready to build let opts = ImageBuildOpts::builder(".") From 0773c40c5d9ccd7a060463318cd59673a0bf5c23 Mon Sep 17 00:00:00 2001 From: Gerald Pinder Date: Wed, 14 Feb 2024 14:14:38 -0500 Subject: [PATCH 10/10] Move signal handle close to right after build --- src/commands/build.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/commands/build.rs b/src/commands/build.rs index 0df0ddde..b6d8aa53 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -279,6 +279,8 @@ impl BuildCommand { }, Err(e) => bail!("{e}"), }; + handle.close(); + signals_task.await??; if self.push { debug!("Pushing is enabled"); @@ -315,9 +317,6 @@ impl BuildCommand { sign_images(&image_name, tags.first().map(String::as_str))?; } - handle.close(); - signals_task.await??; - Ok(()) }