Skip to content

Commit

Permalink
Multistage dockerfile (#52)
Browse files Browse the repository at this point in the history
* use multistage Dockerfile to help speedup builds

* move step to free up disk space before build

* update registry login logic

* setup buildx before qemu, no qemu in test job

* update qemu/buildx images

* no options for qemu/buildx, qemu before buildx

* get rid of toolsets, add extension
- add hashicorp.hcl to extensions
- remove toolset json files, use build args instead
- remove nuget download from powershell stage
- verify powershell version after installation
- split powershell and powershell-modules stages
- run sbom scan in final stage only, since currently all have same base
- fix copy job from golang stage in pulumi stage
- ensure build-arg TARGETARCH is used in all stages

* fix megalinter issues
  • Loading branch information
mauwii authored Sep 29, 2023
1 parent d66b34b commit 83c2030
Show file tree
Hide file tree
Showing 11 changed files with 215 additions and 773 deletions.
30 changes: 17 additions & 13 deletions .cspell.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,19 @@
"**/tmp/**",
"**/megalinter-reports/**",
"**/.editorconfig",
"**.hcl"
"**/docker-bake.hcl"
],
"language": "en,en-GB",
"language": "en,en-US",
"words": [
"actrc",
"aliyun",
"aquasecurity",
"ASPNET",
"aspnet",
"assumeyes",
"automake",
"azcliextensions",
"BASEPATH",
"basepath",
"binfmt",
"binutils",
"brotli",
"buildkit",
Expand All @@ -27,7 +28,7 @@
"buildx",
"catthehacker",
"cmdline",
"CODEOWNERS",
"codeowners",
"commandlinetools",
"containerd",
"containerimage",
Expand All @@ -37,6 +38,7 @@
"dearmor",
"dearmour",
"didnt",
"distro",
"dnsutils",
"dockerhub",
"dpkg",
Expand All @@ -54,6 +56,7 @@
"imagename",
"imagetools",
"iputils",
"keyrings",
"keyscan",
"kics",
"libc",
Expand All @@ -76,27 +79,28 @@
"libxss",
"libyaml",
"localedef",
"MARKDOWNLINT",
"markdownlint",
"mauwii",
"mediainfo",
"mediatypes",
"mergify",
"moby",
"multiarch",
"myrepo",
"nbgv",
"nektos",
"nerdbank",
"netcat",
"nofile",
"NOLOGO",
"nologo",
"noninteractive",
"noto",
"nugetpackages",
"nupkg",
"oldci",
"ONBUILD",
"onbuild",
"opencontainers",
"OPTOUT",
"optout",
"patchelf",
"pigz",
"pipefail",
Expand All @@ -116,17 +120,17 @@
"stefanzweifel",
"targetproc",
"texinfo",
"tonistiigi",
"toolcache",
"TOOLSDIRECTORY",
"toolsets",
"toolsdirectory",
"trivy",
"trivyignore",
"tzdata",
"venv",
"vercel",
"vuln",
"WORKDIR",
"XMLDOC",
"workdir",
"xmldoc",
"xorriso",
"xvfb",
"Zrmnxj",
Expand Down
1 change: 0 additions & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
*
!linux/ubuntu/toolsets/*.json
43 changes: 21 additions & 22 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ on:
push:
paths:
- '**/Dockerfile'
- '**/toolsets/*.json'
- '**/.github/workflows/ci.yml'
- '**/docker-bake.hcl'
pull_request:
Expand All @@ -20,31 +19,26 @@ permissions:
pull-requests: write

env:
REGISTRY: ${{ vars.DOCKERHUB_USERNAME && 'docker.io' || 'ghcr.io' }}
REGISTRY_USER: ${{ vars.DOCKERHUB_USERNAME || github.repository_owner }}
REGISTRY: ${{ (vars.DOCKERHUB_USERNAME != '' && secrets.DOCKERHUB_TOKEN != '') && 'docker.io' || 'ghcr.io' }}
REGISTRY_USER: ${{ (vars.DOCKERHUB_USERNAME != '' && secrets.DOCKERHUB_TOKEN != '') && vars.DOCKERHUB_USERNAME || github.repository_owner }}
GITHUB_SHA: ${{ github.event.pull_request.head.sha || github.event.after || github.sha }}
BUILDKIT_PROGRESS: plain

jobs:
test:
runs-on: ubuntu-latest
env:
REGISTRY: ${{ vars.DOCKERHUB_USERNAME && 'docker.io' || 'ghcr.io' }}
outputs:
targets: ${{ steps.matrix.outputs.targets }}
steps:
- name: Checkout code
# kics-scan ignore-line
uses: actions/[email protected]

- name: Setup QEMU
# kics-scan ignore-line
uses: docker/[email protected]

- name: Setup Docker Buildx
# kics-scan ignore-line
uses: docker/[email protected]

# get docker binary path to use it with env -i
- name: Get the docker binary path
id: docker-binary-path
run: echo "DOCKER_BINARY_PATH=$(which docker)" >> "$GITHUB_OUTPUT"
Expand Down Expand Up @@ -80,26 +74,23 @@ jobs:
matrix:
targets: ${{ fromJson(needs.test.outputs.targets) }}
steps:
- uses: actions/[email protected]

- name: Free up disk space
- name: Checkout code
# kics-scan ignore-line
uses: ./.github/actions/free-space
with:
deleteDotnet: 'true'
deleteAndroid: 'true'
uses: actions/[email protected]

- name: Setup QEMU
# kics-scan ignore-line
uses: docker/[email protected]
# with:
# image: 'tonistiigi/binfmt:buildkit-latest'

- name: Setup Docker Buildx
# kics-scan ignore-line
uses: docker/[email protected]
with:
driver: docker-container
driver-opts: image=moby/buildkit:v0.12.2
# driver: ${{ github.event_name == 'pull_request' && 'docker' || 'docker-container' }}
# with:
# driver-opts: |
# image=moby/buildkit:latest
# # driver: ${{ github.event_name == 'pull_request' && 'docker' || 'docker-container' }}

# Login against a container registry
# https://github.com/docker/login-action
Expand All @@ -108,10 +99,18 @@ jobs:
name: Login to ${{ env.REGISTRY }}
with:
registry: ${{ env.REGISTRY }}
username: ${{ vars.DOCKERHUB_USERNAME || github.repository_owner }}
password: ${{ secrets.DOCKERHUB_TOKEN || secrets.GITHUB_TOKEN }}
username: ${{ env.REGISTRY_USER }}
password: ${{ env.REGISTRY == 'docker.io' && secrets.DOCKERHUB_TOKEN || secrets.GITHUB_TOKEN }}
logout: true

# Free up space on the runner since the image is huge
# kics-scan ignore-line
- uses: ./.github/actions/free-space
name: Free up disk space
with:
deleteDotnet: 'true'
deleteAndroid: 'true'

# Bake the image
# kics-scan ignore-line
- uses: docker/[email protected]
Expand Down
3 changes: 1 addition & 2 deletions .jscpd.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
"**/report/**",
"**/*.svg",
"**/tmp/**",
"**/megalinter-reports/**",
"**/linux/*/toolsets/*.json"
"**/megalinter-reports/**"
]
}
3 changes: 2 additions & 1 deletion .secretlintignore
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
**/megalinter-reports/**
**/megalinter-reports/**
**/cspell.txt
3 changes: 2 additions & 1 deletion .vscode/extensions.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@
"exiasr.hadolint",
"foxundermoon.shell-format",
"github.vscode-github-actions",
"hashicorp.hcl",
"mads-hartmann.bash-ide-vscode",
"ms-azuretools.vscode-docker",
"MS-CST-E.vscode-devskim",
"ms-cst-e.vscode-devskim",
"streetsidesoftware.code-spell-checker"
]
}
87 changes: 66 additions & 21 deletions docker-bake.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@ variable "REGISTRY" {
default = "docker.io"
}

variable "REPOSITORY_OWNER" {
variable "GITHUB_REPOSITORY_OWNER" {
default = "mauwii"
}

variable "REPOSITORY" {
default = "${REPOSITORY_OWNER}/act-docker-images"
default = "${GITHUB_REPOSITORY_OWNER}/act-docker-images"
}

variable "REPOSITORY_URL" {
Expand All @@ -30,10 +30,26 @@ variable "GITHUB_BASE_REF" {
default = null
}

variable "GITHUB_ACTOR" {
default = GITHUB_REPOSITORY_OWNER
}

variable "FROM_IMAGE" {
default = "buildpack-deps"
}

variable "BICEP_VERSION" {
default = "v0.21.1"
}

variable "DOTNET_SDK_VERSION" {
default = "6.0.414"
}

variable "DEPENDENCIES" {
default = "[\"acl\",\"apt-transport-https\",\"aria2\",\"bison\",\"brotli\",\"dbus\",\"dnsutils\",\"fakeroot\",\"flex\",\"fonts-noto-color-emoji\",\"ftp\",\"gawk\",\"gnupg-agent\",\"gnupg2\",\"haveged\",\"iproute2\",\"iputils-ping\",\"libc++-dev\",\"libc++abi-dev\",\"libc6-dev\",\"libgbm-dev\",\"libgconf-2-4\",\"libgsl-dev\",\"libgtk-3-0\",\"libmagic-dev\",\"libsecret-1-dev\",\"libssl-dev\",\"libunwind8\",\"libxkbfile-dev\",\"libxss1\",\"libyaml-dev\",\"lz4\",\"mediainfo\",\"net-tools\",\"netcat\",\"p7zip-full\",\"p7zip-rar\",\"parallel\",\"pass\",\"patchelf\",\"pigz\",\"pollinate\",\"python-is-python3\",\"rpm\",\"rsync\",\"shellcheck\",\"software-properties-common\",\"sphinxsearch\",\"sqlite3\",\"ssh\",\"sshpass\",\"subversion\",\"sudo\",\"swig\",\"telnet\",\"texinfo\",\"time\",\"tk\",\"unzip\",\"upx\",\"xorriso\",\"xvfb\",\"xz-utils\",\"zip\",\"zstd\",\"zsync\"]"
}

variable "GO_VERSION" {
default = "1.20.8"
}
Expand All @@ -46,9 +62,30 @@ variable "GOLANG_GITHUB_SHA256_arm64" {
default = "15ab379c6a2b0d086fe3e74be4599420e66549edf7426a300ee0f3809500f89e"
}

variable "NODE_VERSION" {
default = "20"
}

variable "PULUMI_VERSION" {
default = "3.86.0"
}

variable "POWERSHELL_AZ_MODULE_VERSIONS" {
default = "[\"9.3.0\"]"
}

variable "POWERSHELL_VERSION" {
default = "7.2.13"
}

variable "POWERSHELL_MODULES" {
default = "[\"MarkdownPS\",\"Microsoft.Graph\",\"Pester\",\"PSScriptAnalyzer\"]"
}

group "default" {
targets = ["ubuntu"]
context = "."
targets = [
"ubuntu"
]
}

target "ubuntu" {
Expand All @@ -69,38 +106,46 @@ target "ubuntu" {
]
}
args = {
CODENAME = release.codename
DISTRO = "ubuntu"
FROM_IMAGE = "${FROM_IMAGE}"
FROM_VERSION_MAJOR = release.major
FROM_VERSION_MINOR = release.minor
GO_VERSION = "${GO_VERSION}"
GOLANG_GITHUB_SHA256_amd64 = "${GOLANG_GITHUB_SHA256_amd64}"
GOLANG_GITHUB_SHA256_arm64 = "${GOLANG_GITHUB_SHA256_arm64}"
BICEP_VERSION = BICEP_VERSION
CODENAME = release.codename
DEPENDENCIES = DEPENDENCIES
DISTRO = "ubuntu"
DOTNET_SDK_VERSION = DOTNET_SDK_VERSION
FROM_IMAGE = FROM_IMAGE
FROM_VERSION_MAJOR = release.major
FROM_VERSION_MINOR = release.minor
GO_VERSION = GO_VERSION
GOLANG_GITHUB_SHA256_amd64 = GOLANG_GITHUB_SHA256_amd64
GOLANG_GITHUB_SHA256_arm64 = GOLANG_GITHUB_SHA256_arm64
NODE_VERSION = NODE_VERSION
POWERSHELL_AZ_MODULE_VERSIONS = POWERSHELL_AZ_MODULE_VERSIONS
POWERSHELL_MODULES = POWERSHELL_MODULES
PULUMI_VERSION = PULUMI_VERSION
TOOL_PATH_PWSH = "/usr/share/powershell"
}
name = "ubuntu-act-${release.codename}"
cache-from = [
"${REGISTRY}/${REPOSITORY_OWNER}/ubuntu-act:cache-${release.codename}"
"${REGISTRY}/${GITHUB_REPOSITORY_OWNER}/ubuntu-act:cache-${release.codename}"
]
cache-to = [
notequal(REF_NAME, "local") ? "${REGISTRY}/${REPOSITORY_OWNER}/ubuntu-act:cache-${release.codename}" : ""
notequal(REF_NAME, "local") ? "${REGISTRY}/${GITHUB_REPOSITORY_OWNER}/ubuntu-act:cache-${release.codename}" : ""
]
tags = [
"${REGISTRY}/${REPOSITORY_OWNER}/ubuntu-act:${release.major}.${release.minor}-${and(notequal(REF_NAME, ""), notequal(REF_NAME, null)) ? REF_NAME : "local"}",
and(notequal(GITHUB_SHA, null), equal("${REF_NAME}", "main")) ? "${REGISTRY}/${REPOSITORY_OWNER}/ubuntu-act:${release.major}.${release.minor}-${substr(GITHUB_SHA, 0, 7)}" : "",
equal("${REF_NAME}", "main") ? "${REGISTRY}/${REPOSITORY_OWNER}/ubuntu-act:${release.major}.${release.minor}" : "",
and(equal("${REF_NAME}", "main"), equal(release.codename, "jammy")) ? "${REGISTRY}/${REPOSITORY_OWNER}/ubuntu-act:latest" : "",
"${REGISTRY}/${GITHUB_REPOSITORY_OWNER}/ubuntu-act:${release.major}.${release.minor}-${REF_NAME}",
and(notequal(GITHUB_SHA, null), equal("${REF_NAME}", "main")) ? "${REGISTRY}/${GITHUB_REPOSITORY_OWNER}/ubuntu-act:${release.major}.${release.minor}-${substr(GITHUB_SHA, 0, 7)}" : "",
equal("${REF_NAME}", "main") ? "${REGISTRY}/${GITHUB_REPOSITORY_OWNER}/ubuntu-act:${release.major}.${release.minor}" : "",
and(equal("${REF_NAME}", "main"), equal(release.codename, "jammy")) ? "${REGISTRY}/${GITHUB_REPOSITORY_OWNER}/ubuntu-act:latest" : "",
]
labels = {
"org.opencontainers.image.authors" = REPOSITORY_OWNER
"org.opencontainers.image.authors" = notequal(GITHUB_REPOSITORY_OWNER, GITHUB_ACTOR) ? "${GITHUB_REPOSITORY_OWNER}, ${GITHUB_ACTOR}" : GITHUB_REPOSITORY_OWNER
"org.opencontainers.image.created" = timestamp()
"org.opencontainers.image.description" = "This Image is made to be used with Nektos/act to run your GH-Workflows locally"
"org.opencontainers.image.documentation" = REPOSITORY_URL
"org.opencontainers.image.revision" = GITHUB_SHA
"org.opencontainers.image.source" = and(and(notequal(REPOSITORY_URL, ""), notequal(REPOSITORY_URL, null)), and(notequal(GITHUB_SHA, ""), notequal(GITHUB_SHA, null))) ? "${REPOSITORY_URL}/blob/${GITHUB_SHA}/linux/ubuntu/Dockerfile" : null
"org.opencontainers.image.title" = "ubuntu-act-${release.codename}"
"org.opencontainers.image.url" = equal("${REGISTRY}", "docker.io") ? "https://hub.docker.com/r/${REPOSITORY_OWNER}/ubuntu-act" : equal("${REGISTRY}", "ghcr.io") ? "https://github.com/${REPOSITORY}/pkgs/container/ubuntu-act" : null
"org.opencontainers.image.vendor" = "${REPOSITORY_OWNER}"
"org.opencontainers.image.url" = equal(REGISTRY, "docker.io") ? "https://hub.docker.com/r/${GITHUB_REPOSITORY_OWNER}/ubuntu-act" : equal("${REGISTRY}", "ghcr.io") ? "https://github.com/${REPOSITORY}/pkgs/container/ubuntu-act" : null
"org.opencontainers.image.vendor" = GITHUB_REPOSITORY_OWNER
}
}

Expand Down
Loading

0 comments on commit 83c2030

Please sign in to comment.