get rid of toolsets, add extension

- add hashicorp.hcl to extensions - remove toolset json files, use build args instead - remove nuget download from powershell stage - verify powershell version after installation - split powershell and powershell-modules stages - run sbom scan in final stage only, since currently all have same base - fix copy job from golang stage in pulumi stage - ensure build-arg TARGETARCH is used in all stages
mauwii · Sep 29, 2023 · d2a25ab · d2a25ab
1 parent 8ee7707
commit d2a25ab
Show file tree

Hide file tree

Showing 10 changed files with 76 additions and 720 deletions.
diff --git a/.cspell.json b/.cspell.json
@@ -14,11 +14,11 @@
     "actrc",
     "aliyun",
     "aquasecurity",
-    "ASPNET",
+    "aspnet",
     "assumeyes",
     "automake",
     "azcliextensions",
-    "BASEPATH",
+    "basepath",
     "binfmt",
     "binutils",
     "brotli",
@@ -28,7 +28,7 @@
     "buildx",
     "catthehacker",
     "cmdline",
-    "CODEOWNERS",
+    "codeowners",
     "commandlinetools",
     "containerd",
     "containerimage",
@@ -79,7 +79,7 @@
     "libxss",
     "libyaml",
     "localedef",
-    "MARKDOWNLINT",
+    "markdownlint",
     "mauwii",
     "mediainfo",
     "mediatypes",
@@ -92,15 +92,15 @@
     "nerdbank",
     "netcat",
     "nofile",
-    "NOLOGO",
+    "nologo",
     "noninteractive",
     "noto",
     "nugetpackages",
     "nupkg",
     "oldci",
-    "ONBUILD",
+    "onbuild",
     "opencontainers",
-    "OPTOUT",
+    "optout",
     "patchelf",
     "pigz",
     "pipefail",
@@ -122,17 +122,15 @@
     "texinfo",
     "tonistiigi",
     "toolcache",
-    "TOOLSDIRECTORY",
-    "toolset",
-    "toolsets",
+    "toolsdirectory",
     "trivy",
     "trivyignore",
     "tzdata",
     "venv",
     "vercel",
     "vuln",
-    "WORKDIR",
-    "XMLDOC",
+    "workdir",
+    "xmldoc",
     "xorriso",
     "xvfb",
     "Zrmnxj",

diff --git a/.dockerignore b/.dockerignore
@@ -1,2 +1 @@
 *
-!linux/ubuntu/toolsets/*.json
diff --git a/.jscpd.json b/.jscpd.json
@@ -12,7 +12,6 @@
     "**/report/**",
     "**/*.svg",
     "**/tmp/**",
-    "**/megalinter-reports/**",
-    "**/linux/*/toolsets/*.json"
+    "**/megalinter-reports/**"
   ]
 }
diff --git a/.secretlintignore b/.secretlintignore
@@ -1 +1,2 @@
-**/megalinter-reports/**
+**/megalinter-reports/**
+**/cspell.txt
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
@@ -5,9 +5,10 @@
     "exiasr.hadolint",
     "foxundermoon.shell-format",
     "github.vscode-github-actions",
+    "hashicorp.hcl",
     "mads-hartmann.bash-ide-vscode",
     "ms-azuretools.vscode-docker",
-    "MS-CST-E.vscode-devskim",
+    "ms-cst-e.vscode-devskim",
     "streetsidesoftware.code-spell-checker"
   ]
 }
diff --git a/docker-bake.hcl b/docker-bake.hcl
@@ -45,6 +45,11 @@ variable "BICEP_VERSION" {
 variable "DOTNET_SDK_VERSION" {
   default = "6.0.414"
 }
+
+variable "DEPENDENCIES" {
+  default = "[\"acl\",\"apt-transport-https\",\"aria2\",\"bison\",\"brotli\",\"dbus\",\"dnsutils\",\"fakeroot\",\"flex\",\"fonts-noto-color-emoji\",\"ftp\",\"gawk\",\"gnupg-agent\",\"gnupg2\",\"haveged\",\"iproute2\",\"iputils-ping\",\"libc++-dev\",\"libc++abi-dev\",\"libc6-dev\",\"libgbm-dev\",\"libgconf-2-4\",\"libgsl-dev\",\"libgtk-3-0\",\"libmagic-dev\",\"libsecret-1-dev\",\"libssl-dev\",\"libunwind8\",\"libxkbfile-dev\",\"libxss1\",\"libyaml-dev\",\"lz4\",\"mediainfo\",\"net-tools\",\"netcat\",\"p7zip-full\",\"p7zip-rar\",\"parallel\",\"pass\",\"patchelf\",\"pigz\",\"pollinate\",\"python-is-python3\",\"rpm\",\"rsync\",\"shellcheck\",\"software-properties-common\",\"sphinxsearch\",\"sqlite3\",\"ssh\",\"sshpass\",\"subversion\",\"sudo\",\"swig\",\"telnet\",\"texinfo\",\"time\",\"tk\",\"unzip\",\"upx\",\"xorriso\",\"xvfb\",\"xz-utils\",\"zip\",\"zstd\",\"zsync\"]"
+}
+
 variable "GO_VERSION" {
   default = "1.20.8"
 }
@@ -62,16 +67,25 @@ variable "NODE_VERSION" {
 }
 
 variable "PULUMI_VERSION" {
-  default = "3.83.0"
+  default = "3.86.0"
+}
+
+variable "POWERSHELL_AZ_MODULE_VERSIONS" {
+  default = "[\"9.3.0\"]"
 }
 
 variable "POWERSHELL_VERSION" {
   default = "7.2.13"
 }
 
+variable "POWERSHELL_MODULES" {
+  default = "[\"MarkdownPS\",\"Microsoft.Graph\",\"Pester\",\"PSScriptAnalyzer\"]"
+}
+
 group "default" {
-  targets = ["ubuntu"]
-  context = "."
+  targets = [
+    "ubuntu"
+  ]
 }
 
 target "ubuntu" {
@@ -92,19 +106,22 @@ target "ubuntu" {
     ]
   }
   args = {
-    BICEP_VERSION              = BICEP_VERSION
-    CODENAME                   = release.codename
-    DISTRO                     = "ubuntu"
-    DOTNET_SDK_VERSION         = DOTNET_SDK_VERSION
-    FROM_IMAGE                 = FROM_IMAGE
-    FROM_VERSION_MAJOR         = release.major
-    FROM_VERSION_MINOR         = release.minor
-    GO_VERSION                 = GO_VERSION
-    GOLANG_GITHUB_SHA256_amd64 = GOLANG_GITHUB_SHA256_amd64
-    GOLANG_GITHUB_SHA256_arm64 = GOLANG_GITHUB_SHA256_arm64
-    NODE_VERSION               = NODE_VERSION
-    PULUMI_VERSION             = PULUMI_VERSION
-    TOOL_PATH_PWSH             = "/usr/share/powershell"
+    BICEP_VERSION                 = BICEP_VERSION
+    CODENAME                      = release.codename
+    DEPENDENCIES                  = DEPENDENCIES
+    DISTRO                        = "ubuntu"
+    DOTNET_SDK_VERSION            = DOTNET_SDK_VERSION
+    FROM_IMAGE                    = FROM_IMAGE
+    FROM_VERSION_MAJOR            = release.major
+    FROM_VERSION_MINOR            = release.minor
+    GO_VERSION                    = GO_VERSION
+    GOLANG_GITHUB_SHA256_amd64    = GOLANG_GITHUB_SHA256_amd64
+    GOLANG_GITHUB_SHA256_arm64    = GOLANG_GITHUB_SHA256_arm64
+    NODE_VERSION                  = NODE_VERSION
+    POWERSHELL_AZ_MODULE_VERSIONS = POWERSHELL_AZ_MODULE_VERSIONS
+    POWERSHELL_MODULES            = POWERSHELL_MODULES
+    PULUMI_VERSION                = PULUMI_VERSION
+    TOOL_PATH_PWSH                = "/usr/share/powershell"
   }
   name = "ubuntu-act-${release.codename}"
   cache-from = [
@@ -114,7 +131,7 @@ target "ubuntu" {
     notequal(REF_NAME, "local") ? "${REGISTRY}/${GITHUB_REPOSITORY_OWNER}/ubuntu-act:cache-${release.codename}" : ""
   ]
   tags = [
-    "${REGISTRY}/${GITHUB_REPOSITORY_OWNER}/ubuntu-act:${release.major}.${release.minor}-${and(notequal(REF_NAME, ""), notequal(REF_NAME, null)) ? REF_NAME : "local"}",
+    "${REGISTRY}/${GITHUB_REPOSITORY_OWNER}/ubuntu-act:${release.major}.${release.minor}-${REF_NAME}",
     and(notequal(GITHUB_SHA, null), equal("${REF_NAME}", "main")) ? "${REGISTRY}/${GITHUB_REPOSITORY_OWNER}/ubuntu-act:${release.major}.${release.minor}-${substr(GITHUB_SHA, 0, 7)}" : "",
     equal("${REF_NAME}", "main") ? "${REGISTRY}/${GITHUB_REPOSITORY_OWNER}/ubuntu-act:${release.major}.${release.minor}" : "",
     and(equal("${REF_NAME}", "main"), equal(release.codename, "jammy")) ? "${REGISTRY}/${GITHUB_REPOSITORY_OWNER}/ubuntu-act:latest" : "",

diff --git a/linux/ubuntu/Dockerfile b/linux/ubuntu/Dockerfile
@@ -7,7 +7,6 @@ ARG FROM_VERSION=${FROM_VERSION_MAJOR}.${FROM_VERSION_MINOR}
 FROM ${FROM_IMAGE}:${FROM_VERSION} as base
 
 FROM base as bicep
-ARG BUILDKIT_SBOM_SCAN_STAGE=true
 ARG TARGETARCH
 ARG BICEP_VERSION=v0.21.1
 RUN export targetarch="${TARGETARCH}" \
@@ -18,7 +17,6 @@ RUN export targetarch="${TARGETARCH}" \
     && bicep --version
 
 FROM base as golang
-ARG BUILDKIT_SBOM_SCAN_STAGE=true
 ARG TARGETARCH
 SHELL [ "/bin/bash", "--login", "-e", "-o", "pipefail", "-c" ]
 ENV PATH=/usr/local/go/bin:${PATH}
@@ -34,9 +32,9 @@ RUN mkdir -p /tmp/go \
     && [[ $(go version) =~ ${GOLANG_VERSION} ]]
 
 FROM base as pulumi
-ARG BUILDKIT_SBOM_SCAN_STAGE=true
+ARG TARGETARCH
 SHELL [ "/bin/bash", "--login", "-e", "-o", "pipefail", "-c" ]
-COPY --from=golang /tmp/go /usr/local/
+COPY --from=golang /usr/local/go /usr/local/go
 ENV PATH=/usr/local/go/bin:${PATH}
 ARG PULUMI_VERSION=3.83.0
 RUN curl -fsSL https://get.pulumi.com \
@@ -103,8 +101,10 @@ RUN ssh-keyscan github.com >>/etc/ssh/ssh_known_hosts \
     && ssh-keyscan ssh.dev.azure.com >>/etc/ssh/ssh_known_hosts
 
 # Install dependencies
+ARG DEPENDENCIES
 RUN apt-get -y update &>/dev/null \
-    && packages=(gawk sudo jq gnupg-agent ca-certificates software-properties-common apt-transport-https zstd zip unzip xz-utils) \
+    && apt-get -y install --no-install-recommends jq && packages=() \
+    && while IFS='' read -r pkg; do echo "adding $pkg" && packages+=("$pkg"); done < <(echo "${DEPENDENCIES}" | jq -r '.[]') \
     && apt-get -y install --no-install-recommends \
         "${packages[@]}" \
     && apt-get clean \
@@ -220,7 +220,7 @@ RUN [[ $(curl -sL https://packages.microsoft.com/repos/azure-cli/dists/) =~ $(ls
         && rm -rf /var/lib/apt/lists/*) \
     || echo "Azure CLI not available for this distribution"
 
-# Install default NodeJS and toolset node_modules
+# Install default NodeJS
 ARG NODE_VERSION=20
 RUN curl -fsSL https://raw.githubusercontent.com/tj/n/master/bin/n -o ~/n \
     && bash ~/n "${NODE_VERSION}" \
@@ -273,72 +273,43 @@ RUN apt-get -y clean \
 
 # kics-scan ignore-line
 FROM act-base as powershell
-ARG BUILDKIT_SBOM_SCAN_STAGE=true
-ARG FROM_VERSION_MAJOR=22
-ARG FROM_VERSION_MINOR=04
-ARG FROM_VERSION=${FROM_VERSION_MAJOR}.${FROM_VERSION_MINOR}
-ARG DISTRO=ubuntu
 SHELL [ "/bin/bash", "--login", "-e", "-o", "pipefail", "-c" ]
-
 # Install PowerShell global tool
 ARG TARGETARCH
 ARG POWERSHELL_VERSION=7.2.13
 ARG TOOL_PATH_PWSH=/usr/share/powershell
-RUN export targetarch=${TARGETARCH} \
-    && if [ ${targetarch} = "amd64" ]; then export targetarch="x64"; fi \
-    && pwsh_nupkg="PowerShell.Linux.${targetarch}.${POWERSHELL_VERSION}.nupkg" \
-    && export pwsh_nupkg \
-    && curl -fsSLO "https://pwshtool.blob.core.windows.net/tool/${POWERSHELL_VERSION}/${pwsh_nupkg}" \
-    && dotnet tool install \
-        --add-source / \
-        --tool-path "${TOOL_PATH_PWSH}" \
-        --version "${POWERSHELL_VERSION}" \
-        "PowerShell.Linux.${targetarch}" \
-    && dotnet nuget locals all --clear \
-    && rm "${pwsh_nupkg}" \
-    && ln -s "${TOOL_PATH_PWSH}/pwsh" /usr/bin/pwsh \
-    && chmod 755 "${TOOL_PATH_PWSH}/pwsh"
-
-# Add toolset.json
-ARG TOOLSET=/imagegeneration/toolset.json
-COPY linux/${DISTRO}/toolsets/${FROM_VERSION_MAJOR}${FROM_VERSION_MINOR}.json ${TOOLSET}
+RUN dotnet tool install \
+    --tool-path "${TOOL_PATH_PWSH}" \
+    --version "${POWERSHELL_VERSION}" \
+    PowerShell \
+    && find "${TOOL_PATH_PWSH}" -print | grep -i '.*[.]nupkg$' | xargs rm \
+    && ln -s "${TOOL_PATH_PWSH}/pwsh" /usr/local/bin/pwsh \
+    && chmod 755 "${TOOL_PATH_PWSH}/pwsh" \
+    && [[ "$(pwsh --version)" =~ ${POWERSHELL_VERSION} ]]
 
+FROM powershell as powershell-modules
+ARG TARGETARCH
+SHELL [ "/bin/bash", "--login", "-e", "-o", "pipefail", "-c" ]
+ARG POWERSHELL_MODULES
+ARG POWERSHELL_AZ_MODULE_VERSIONS
 # Trust PSGallery and install Powershell-/Azure Modules
 RUN pwsh -NonInteractive -Command "Set-PSRepository -Name PSGallery -InstallationPolicy Trusted" \
-    && while IFS='' read -r pwshModule; do echo "installing ${pwshModule}" && pwsh -NonInteractive -Command "Install-Module -Name ${pwshModule} -Scope AllUsers -Repository PSGallery"; done < <(jq -r '.powershellModules[].name' ${TOOLSET}) \
-    && while IFS='' read -r azVersion; do echo "installing Az ${azVersion}" && pwsh -NonInteractive -Command "Install-Module -Name Az -RequiredVersion ${azVersion} -Scope AllUsers -Repository PSGallery"; done < <(jq -r '.azureModules[].versions[]' ${TOOLSET})
+    && while IFS='' read -r pwshModule; do echo "installing ${pwshModule}" && pwsh -NonInteractive -Command "Install-Module -Name ${pwshModule} -Scope AllUsers -Repository PSGallery"; done < <(echo "${POWERSHELL_MODULES}" | jq -r '.[]') \
+    && while IFS='' read -r azVersion; do echo "installing Az ${azVersion}" && pwsh -NonInteractive -Command "Install-Module -Name Az -RequiredVersion ${azVersion} -Scope AllUsers -Repository PSGallery"; done < <(echo "${POWERSHELL_AZ_MODULE_VERSIONS}" | jq -r '.[]')
 
 # kics-scan ignore-line
 FROM act-base as act
-ARG BUILDKIT_SBOM_SCAN_STAGE=true
+ARG TARGETARCH
 SHELL [ "/bin/bash", "--login", "-e", "-o", "pipefail", "-c" ]
 WORKDIR /tmp
 
-# Add toolset.json
-ARG FROM_VERSION_MAJOR=22
-ARG FROM_VERSION_MINOR=04
-ARG DISTRO=ubuntu
-ARG TOOLSET=/imagegeneration/toolset.json
-COPY linux/${DISTRO}/toolsets/${FROM_VERSION_MAJOR}${FROM_VERSION_MINOR}.json ${TOOLSET}
-
-# install apt packages from toolset
-# hadolint ignore=SC2207
-RUN apt-get -y update &>/dev/null \
-    && packages=() \
-    && while IFS='' read -r line; do packages+=("${line}"); done < <(jq -r '.apt|.vital_packages[],.common_packages[],.cmd_packages[]' ${TOOLSET}) \
-    && apt-get -y install --no-install-recommends \
-        "${packages[@]}" \
-    && apt-get clean \
-    && rm -rf /etc/apt/sources.list.d/* \
-    && rm -rf /var/lib/apt/lists/*
-
-# add Powershell and Modules
+# add PowerShell
 ARG TOOL_PATH_PWSH=/usr/share/powershell
-COPY --from=powershell /usr/local/share/powershell /usr/local/share/powershell
 COPY --from=powershell ${TOOL_PATH_PWSH} ${TOOL_PATH_PWSH}
-RUN ln -s "${TOOL_PATH_PWSH}/pwsh" /usr/bin/pwsh \
-    && chmod 755 "${TOOL_PATH_PWSH}/pwsh" \
-    && find "${TOOL_PATH_PWSH}" -print | grep -i '.*[.]nupkg$' | xargs rm
+RUN ln -s "${TOOL_PATH_PWSH}/pwsh" /usr/local/bin/pwsh
+
+# add  PowerShell-modules
+COPY --from=powershell-modules /usr/local/share/powershell /usr/local/share/powershell
 
 # add go
 COPY --from=golang /usr/local/go /usr/local/go