Skip to content

Commit

Permalink
Merge pull request #14932 from ajayk/revert-ci-build-gh-runners
Browse files Browse the repository at this point in the history
  • Loading branch information
@ajayk
ajayk authored Mar 13, 2024
2 parents ba9627c + 52765ae commit 2b577dd
Showing 1 changed file with 73 additions and 95 deletions.
168 changes: 73 additions & 95 deletions .github/workflows/ci-build.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: CI build action

on:
pull_request:
branches: ["main"]
branches: ['main']
push:
branches:
- gh-readonly-queue/main/**
Expand Down Expand Up @@ -64,17 +64,24 @@ jobs:
name: Test building of packages
strategy:
matrix:
arch: ["x86_64", "aarch64"]
include:
- arch: x86_64
runner: ubuntu-latest-16-cores
oci: amd64
- arch: aarch64
runner: ubuntu-arm-16-cores
oci: arm64
arch: [ "x86_64", "aarch64" ]
fail-fast: false
runs-on: ${{ matrix.runner }}
runs-on:
group: wolfi-builder-${{ matrix.arch }}
needs: changes
container:
image: ghcr.io/wolfi-dev/sdk:latest@sha256:b8778a18b4aa16bc5302aababbe5dfd6fccb659c5478ef46b5b4550d8bc3aa76
options: |
--cap-add NET_ADMIN --cap-add SYS_ADMIN --security-opt seccomp=unconfined --security-opt apparmor:unconfined
volumes:
# GHA assumes we mount /var/run/docker.sock for dind, which means we're
# working with host bind mounts, so we need a volume on the host that
# we can leverage for other bind mounts created by melange commands
# using the docker runner.
#
# This is named `temp` on purpose so we don't interfere with the host's
# /tmp or the inner containers /tmp.
- "/temp:/temp"
outputs:
packages_were_built: ${{ steps.file_check.outputs.exists }}

Expand All @@ -89,82 +96,55 @@ jobs:
rm -rf /usr/share/dotnet
rm -rf "$AGENT_TOOLSDIRECTORY"
- uses: actions/checkout@v4
- name: Setup Docker
- name: 'Trust the github workspace'
run: |
# Add Docker's official GPG key:
sudo apt-get update -y
sudo apt-get install ca-certificates curl -y
sudo install -m 0755 -d /etc/apt/keyrings
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
sudo chmod a+r /etc/apt/keyrings/docker.asc
# Add the repository to Apt sources:
echo \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update -y
sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
sudo usermod -aG docker $USER
sudo apt-get install acl
sudo setfacl --modify user:$USER:rw /var/run/docker.sock
- name: "Generate local signing key"
uses: ./.github/actions/docker-run
with:
run: |
make MELANGE="melange" local-melange.rsa
# This is to avoid fatal errors about "dubious ownership" because we are
# running inside of a container action with the workspace mounted in.
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: "Build Wolfi"
uses: ./.github/actions/docker-run
with:
opts: "-v /temp:/temp -v /var/run/docker.sock:/var/run/docker.sock"
run: |
# Setup the melange cache dir on the host so we can use that in subsequent builds
mkdir ../.melangecache
for package in ${{needs.changes.outputs.packages}}; do
make MELANGE_EXTRA_OPTS="--create-build-log --cache-dir=\$(pwd)/../.melangecache" REPO="./packages" package/\$package -j1
TMPDIR="/temp" make REPO="./packages" MELANGE_EXTRA_OPTS="--runner docker" test/\$package -j1
done
- name: "Check that packages can be installed with apk add"
uses: ./.github/actions/docker-run
with:
run: |
set -x
# Create a fake linux fs under /tmp/emptyroot to pass to `apk --root`.
mkdir -p /tmp/emptyroot/etc/apk
cp -r /etc/apk/* /tmp/emptyroot/etc/apk/
cat /dev/null > /tmp/emptyroot/etc/apk/world
mkdir -p /tmp/emptyroot/lib/apk/db
touch /tmp/emptyroot/lib/apk/db/{installed,lock,scripts.tar,triggers}
mkdir -p /tmp/emptyroot/var/cache/apk
apk update --root /tmp/emptyroot
# Find .apk files and add them to the string
for f in \$(find packages -name '*.apk'); do
tar -Oxf \$f .PKGINFO
apk add --root /tmp/emptyroot --repository "./packages" --allow-untrusted --simulate \$f
done
- name: 'Generate local signing key'
run: |
make MELANGE="melange" local-melange.rsa
- name: Reset file permissions
- name: 'Build Wolfi'
run: |
sudo chown -R $(id -u):$(id -g) .
# Setup the melange cache dir on the host so we can use that in subsequent builds
mkdir ../.melangecache
for package in ${{needs.changes.outputs.packages}}; do
make MELANGE_EXTRA_OPTS="--create-build-log --cache-dir=$(pwd)/../.melangecache" REPO="$GITHUB_WORKSPACE/packages" package/$package -j1
TMPDIR="/temp" make REPO="$GITHUB_WORKSPACE/packages" MELANGE_EXTRA_OPTS="--runner docker" test/$package -j1
done
- name: 'Check that packages can be installed with apk add'
run: |
# Create a fake linux fs under /tmp/emptyroot to pass to `apk --root`.
mkdir -p /tmp/emptyroot/etc/apk
cp -r /etc/apk/* /tmp/emptyroot/etc/apk/
cat /dev/null > /tmp/emptyroot/etc/apk/world
mkdir -p /tmp/emptyroot/lib/apk/db
touch /tmp/emptyroot/lib/apk/db/{installed,lock,scripts.tar,triggers}
mkdir -p /tmp/emptyroot/var/cache/apk
apk update --root /tmp/emptyroot
# Find .apk files and add them to the string
for f in $(find packages -name '*.apk'); do
tar -Oxf $f .PKGINFO
apk add --root /tmp/emptyroot --repository "$GITHUB_WORKSPACE/packages" --allow-untrusted --simulate $f
done
- name: Check SBOMs
uses: ./.github/actions/docker-run
with:
run: |
apk add py3-ntia-conformance-checker
for f in \$(find packages -name '*.apk'); do
echo ==== Checking SBOM for \$f ====
tar -Oxf \$f var/lib/db/sbom/ > sbom.json
echo ::group::sbom.json
cat sbom.json
echo ::endgroup::
ntia-checker -v --file sbom.json
done
run: |
apk add py3-ntia-conformance-checker
for f in $(find packages -name '*.apk'); do
echo ==== Checking SBOM for $f ====
tar -Oxf $f var/lib/db/sbom/ > sbom.json
echo ::group::sbom.json
cat sbom.json
echo ::endgroup::
ntia-checker -v --file sbom.json
done
- name: Check for file
id: file_check
Expand All @@ -181,10 +161,8 @@ jobs:
if: steps.file_check.outputs.exists == 'true'
# Let's not fail the whole job if this step fails as it is for improved UX rather than an enforced check
continue-on-error: true
uses: ./.github/actions/docker-run
with:
run: |
wolfictl check diff
run: |
wolfictl check diff
- name: Check for diff file
id: diff_file_check
Expand All @@ -206,7 +184,7 @@ jobs:
filePath: diff.log
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: "Upload built packages to GitHub artifacts"
- name: 'Upload built packages to GitHub artifacts'
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
path: |
Expand All @@ -228,19 +206,19 @@ jobs:
if: needs.build.outputs.packages_were_built == 'true'

steps:
- name: "Retrieve x86_64 packages"
- name: 'Retrieve x86_64 packages'
uses: actions/download-artifact@21e5c25de9cf2ee24742cd3e822327f3be6dd2a3 # v4.1.1
with:
name: packages-x86_64
path: /tmp/artifacts-1/

- name: "Retrieve aarch64 packages"
- name: 'Retrieve aarch64 packages'
uses: actions/download-artifact@21e5c25de9cf2ee24742cd3e822327f3be6dd2a3 # v4.1.1
with:
name: packages-aarch64
path: /tmp/artifacts-2/

- name: "Collect packages from all architectures into one place"
- name: 'Collect packages from all architectures into one place'
run: |
cd /tmp/artifacts-1
Expand Down Expand Up @@ -269,19 +247,19 @@ jobs:
timeout-minutes: 30

steps:
- name: "Retrieve x86_64 packages"
- name: 'Retrieve x86_64 packages'
uses: actions/download-artifact@21e5c25de9cf2ee24742cd3e822327f3be6dd2a3 # v4.1.1
with:
name: packages-x86_64
path: /tmp/artifacts-1/

- name: "Retrieve aarch64 packages"
- name: 'Retrieve aarch64 packages'
uses: actions/download-artifact@21e5c25de9cf2ee24742cd3e822327f3be6dd2a3 # v4.1.1
with:
name: packages-aarch64
path: /tmp/artifacts-2/

- name: "Collect packages from all architectures into one place"
- name: 'Collect packages from all architectures into one place'
run: |
cd /tmp/artifacts-1
Expand All @@ -292,11 +270,11 @@ jobs:
cat /tmp/artifacts-2/packages.log >> ./packages.log
fi
- name: "Retrieve Wolfi advisory data"
- name: 'Retrieve Wolfi advisory data'
uses: actions/checkout@v4
with:
repository: "wolfi-dev/advisories"
path: "data/wolfi-advisories"
repository: 'wolfi-dev/advisories'
path: 'data/wolfi-advisories'

- name: Scan for CVEs
run: |
Expand All @@ -306,4 +284,4 @@ jobs:
--advisory-filter 'resolved' \
--require-zero \
/tmp/artifacts-1 \
2> /dev/null # The error message renders strangely on GitHub Actions, and the important information is already being sent to stdout.
2> /dev/null # The error message renders strangely on GitHub Actions, and the important information is already being sent to stdout.

0 comments on commit 2b577dd

Please sign in to comment.