From f4be22b97964c53e33d121c78cc787da9dd673c8 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Fri, 12 Jul 2024 15:34:07 -0500 Subject: [PATCH] feat: use hcl lists for cherry-picks, allow undefined --- .gitlab-ci.yml | 8 +++++-- containers/jug/dev.hcl | 24 +++++++------------ eic-spack.sh => eic-spack.hcl | 4 ++-- key4hep-spack.sh => key4hep-spack.hcl | 4 ++-- spack.hcl | 24 +++++++++++++++++++ spack.sh | 33 --------------------------- 6 files changed, 42 insertions(+), 55 deletions(-) rename eic-spack.sh => eic-spack.hcl (56%) rename key4hep-spack.sh => key4hep-spack.hcl (55%) create mode 100644 spack.hcl delete mode 100644 spack.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 76fe87f0..5e1c56fd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -342,17 +342,21 @@ eic: fi - apk add envsubst - source spack.sh ; - source key4hep-spack.sh ; - source eic-spack.sh ; export SPACK_VERSION ; cat mirrors.yaml.in | envsubst > mirrors.yaml - docker buildx bake ${BUILD_OPTIONS} --print + --file spack.hcl + --file eic-spack.hcl + --file key4hep-spack.hcl --file containers/variables.hcl --file containers/jug/dev.hcl - while ! docker buildx bake ${BUILD_OPTIONS} --push + --file spack.hcl + --file eic-spack.hcl + --file key4hep-spack.hcl --file containers/variables.hcl --file containers/jug/dev.hcl 2>&1 | tee build.log diff --git a/containers/jug/dev.hcl b/containers/jug/dev.hcl index 956f935e..3b867e6a 100644 --- a/containers/jug/dev.hcl +++ b/containers/jug/dev.hcl @@ -9,14 +9,6 @@ variable "DOCKER_REGISTRY" { default = join("/", [ CI_REGISTRY, CI_PROJECT_PATH, variable "BUILDER_IMAGE" { default = null } variable "RUNTIME_IMAGE" { default = null } -variable "SPACK_ORGREPO" { default = null } -variable "SPACK_VERSION" { default = null } -variable "SPACK_CHERRYPICKS" { default = null } -variable "SPACK_CHERRYPICKS_FILES" { default = null } -variable "KEY4HEPSPACK_ORGREPO" { default = null } -variable "KEY4HEPSPACK_VERSION" { default = null } -variable "EICSPACK_ORGREPO" { default = null } -variable "EICSPACK_VERSION" { default = null } variable "S3_ACCESS_KEY" { default = null } variable "S3_SECRET_KEY" { default = null } @@ -56,14 +48,14 @@ target "default" { BUILDER_IMAGE = BUILDER_IMAGE RUNTIME_IMAGE = RUNTIME_IMAGE INTERNAL_TAG = INTERNAL_TAG - SPACK_ORGREPO = SPACK_ORGREPO - SPACK_VERSION = SPACK_VERSION - SPACK_CHERRYPICKS = SPACK_CHERRYPICKS - SPACK_CHERRYPICKS_FILES = SPACK_CHERRYPICKS_FILES - KEY4HEPSPACK_ORGREPO = KEY4HEPSPACK_ORGREPO - KEY4HEPSPACK_VERSION = KEY4HEPSPACK_VERSION - EICSPACK_ORGREPO = EICSPACK_ORGREPO - EICSPACK_VERSION = EICSPACK_VERSION + SPACK_ORGREPO = try(SPACK_ORGREPO, null) + SPACK_VERSION = try(SPACK_VERSION, null) + SPACK_CHERRYPICKS = join("\n", try(SPACK_CHERRYPICKS, [])) + SPACK_CHERRYPICKS_FILES = join("\n", try(SPACK_CHERRYPICKS_FILES, [])) + KEY4HEPSPACK_ORGREPO = try(KEY4HEPSPACK_ORGREPO, null) + KEY4HEPSPACK_VERSION = try(KEY4HEPSPACK_VERSION, null) + EICSPACK_ORGREPO = try(EICSPACK_ORGREPO, null) + EICSPACK_VERSION = try(EICSPACK_VERSION, null) S3_ACCESS_KEY = S3_ACCESS_KEY S3_SECRET_KEY = S3_SECRET_KEY EDM4EIC_VERSION = BUILD_TYPE == "default" ? EDM4EIC_VERSION : "main" diff --git a/eic-spack.sh b/eic-spack.hcl similarity index 56% rename from eic-spack.sh rename to eic-spack.hcl index a9cb0313..c8f542f0 100644 --- a/eic-spack.sh +++ b/eic-spack.hcl @@ -1,6 +1,6 @@ ## EIC spack organization and repository, e.g. eic/eic-spack -EICSPACK_ORGREPO="eic/eic-spack" +variable "EICSPACK_ORGREPO" { default = "eic/eic-spack" } ## EIC spack commit hash or github version, e.g. v0.19.7 ## note: nightly builds could use a branch e.g. releases/v0.19 -EICSPACK_VERSION="c8429aea4cb76c3dc40b301b4a5b375976fce674" +variable "EICSPACK_VERSION" { default = "c8429aea4cb76c3dc40b301b4a5b375976fce674" } diff --git a/key4hep-spack.sh b/key4hep-spack.hcl similarity index 55% rename from key4hep-spack.sh rename to key4hep-spack.hcl index 90f2a1a3..da6434cd 100644 --- a/key4hep-spack.sh +++ b/key4hep-spack.hcl @@ -1,6 +1,6 @@ ## Key4HEP spack organization and repository, e.g. key4hep/key4hep-spack -KEY4HEPSPACK_ORGREPO="key4hep/key4hep-spack" +variable "KEY4HEPSPACK_ORGREPO" { default = "key4hep/key4hep-spack" } ## Key4HEP spack commit hash or github version, e.g. v0.19.7 ## note: nightly builds could use a branch e.g. releases/v0.19 -KEY4HEPSPACK_VERSION="ce50d74dba7c665157cbef63cd5c9cc4d7758f72" +variable "KEY4HEPSPACK_VERSION" { default = "ce50d74dba7c665157cbef63cd5c9cc4d7758f72" } diff --git a/spack.hcl b/spack.hcl new file mode 100644 index 00000000..87c2bfe7 --- /dev/null +++ b/spack.hcl @@ -0,0 +1,24 @@ +## Spack organization and repository, e.g. spack/spack +variable "SPACK_ORGREPO" { default = "spack/spack" } + +## Spack github version, e.g. v0.18.1 or commit hash +## note: nightly builds will use e.g. releases/v0.19 +variable "SPACK_VERSION" { default = "v0.22.0" } + +## Space-separated list of spack cherry-picks +variable "SPACK_CHERRYPICKS" { + default = [ + "09f75ee426a2e05e0543570821582480ff823ba5", # setup-env.sh: if exe contains qemu, use /proc/$$/comm instead + "f6d50f790ee8b123f7775429f6ca6394170e6de9", # gaudi: Add version 38.1 + "63f6e6079aacc99078386e5c8ff06173841b9595", # gaudi: upstream patch when @38.1 for missing #include + "9bcc43c4c158639fa6cb575c6106595a34682081", # protobuf: update hash for patch needed when="@3.4:3.21" + "9f3e45ddbee24aaa7993e575297827e0aed2e6fe", # acts: pass cuda_arch to CMAKE_CUDA_ARCHITECTURES + "85f13442d2a7486daba81fdd9a3b6a1182ba11f6", # Consolidate concretization output for environments + "f73d7d2dce226857cbc774e942454bad2992969e", # dd4hep: cleanup recipe, remove deprecated versions and patches + "cbab451c1a342523ed75e9be1098615a597a9b59", # dd4hep: Add version 1.29 + ] +} +## Optional hash table with comma-separated file list +variable "SPACK_CHERRYPICKS_FILES" { + default = [] +} diff --git a/spack.sh b/spack.sh deleted file mode 100644 index d431e6f9..00000000 --- a/spack.sh +++ /dev/null @@ -1,33 +0,0 @@ -## Spack organization and repository, e.g. spack/spack -SPACK_ORGREPO="spack/spack" - -## Spack github version, e.g. v0.18.1 or commit hash -## note: nightly builds will use e.g. releases/v0.19 -SPACK_VERSION="v0.22.0" - -## Space-separated list of spack cherry-picks -read -r -d '' SPACK_CHERRYPICKS <<- \ ---- || true -09f75ee426a2e05e0543570821582480ff823ba5 -f6d50f790ee8b123f7775429f6ca6394170e6de9 -63f6e6079aacc99078386e5c8ff06173841b9595 -9bcc43c4c158639fa6cb575c6106595a34682081 -9f3e45ddbee24aaa7993e575297827e0aed2e6fe -85f13442d2a7486daba81fdd9a3b6a1182ba11f6 -f73d7d2dce226857cbc774e942454bad2992969e -cbab451c1a342523ed75e9be1098615a597a9b59 ---- -## Optional hash table with comma-separated file list -read -r -d '' SPACK_CHERRYPICKS_FILES <<- \ ---- || true ---- -## Ref: https://github.com/spack/spack/commit/[hash] -## [hash]: [description] -## 09f75ee426a2e05e0543570821582480ff823ba5: setup-env.sh: if exe contains qemu, use /proc/$$/comm instead -## f6d50f790ee8b123f7775429f6ca6394170e6de9: gaudi: Add version 38.1 -## 63f6e6079aacc99078386e5c8ff06173841b9595: gaudi: upstream patch when @38.1 for missing #include -## 9bcc43c4c158639fa6cb575c6106595a34682081: protobuf: update hash for patch needed when="@3.4:3.21" -## 9f3e45ddbee24aaa7993e575297827e0aed2e6fe: acts: pass cuda_arch to CMAKE_CUDA_ARCHITECTURES -## 85f13442d2a7486daba81fdd9a3b6a1182ba11f6: Consolidate concretization output for environments -## f73d7d2dce226857cbc774e942454bad2992969e: dd4hep: cleanup recipe, remove deprecated versions and patches -## cbab451c1a342523ed75e9be1098615a597a9b59: dd4hep: Add version 1.29