Merge pull request #13363 from wolfi-dev/wolfictl-8c6b34b6-99f5-48e9-… #5380
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Wolfi OS | |
| on: | |
| push: | |
| branches: ['main'] | |
| paths-ignore: | |
| - '**.md' | |
| - '**.txt' | |
| workflow_dispatch: | |
| # Only run one build at a time to prevent out of sync signatures. | |
| concurrency: build | |
| jobs: | |
| build: | |
| name: Build packages | |
| if: github.repository == 'wolfi-dev/os' | |
| strategy: | |
| matrix: | |
| arch: [ "x86_64", "aarch64" ] | |
| fail-fast: false | |
| runs-on: | |
| group: wolfi-os-builder-${{ matrix.arch }} | |
| permissions: | |
| contents: read | |
| container: | |
| image: ghcr.io/wolfi-dev/sdk:latest@sha256:110c4bc0a8941606034ee7af12f1197b4a6b6f6434fd4b4bbf61de501e18ffd1 | |
| # TODO: Deprivilege | |
| options: | | |
| --cap-add NET_ADMIN --cap-add SYS_ADMIN --device /dev/fuse --security-opt seccomp=unconfined --security-opt apparmor:unconfined | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: 'Trust the github workspace' | |
| run: | | |
| # This is to avoid fatal errors about "dubious ownership" because we are | |
| # running inside of a container action with the workspace mounted in. | |
| git config --global --add safe.directory "$(pwd)" | |
| # Build with a local key, we'll resign this with the real key later | |
| - name: 'Generate local signing key' | |
| run: | | |
| make local-melange.rsa | |
| # Touch it with the epoch date to convince `make` that we don't need to | |
| # rebuild the targets that depend on this (all) | |
| touch -d @0 local-melange.rsa | |
| - name: 'Prepare package repository' | |
| run: | | |
| # yay wolfi! | |
| apk add gcsfuse google-cloud-sdk | |
| # Set up a gcsfuse RO mount to the public bucket. This is a cheap and | |
| # cheerful way to recreate the make targets (class A HEADs) locally | |
| # without syncing the whole bucket (class A+B). | |
| mkdir -p /gcsfuse/wolfi-registry | |
| gcsfuse -o ro --implicit-dirs --only-dir os wolfi-production-registry-destination /gcsfuse/wolfi-registry | |
| mkdir -p ./packages/${{ matrix.arch }} | |
| # Symlink the gcsfuse mount to ./packages/ to workaround the Makefile CWD assumptions | |
| for f in /gcsfuse/wolfi-registry/${{ matrix.arch }}/*.apk; do | |
| ln -s "$f" ./packages/${{ matrix.arch }}/ | |
| done | |
| # Make a copy of the APKINDEX.* since we'll need to write to it on package builds | |
| cp /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.* ./packages/${{ matrix.arch }}/ | |
| # TODO: Replace this with wolfictl build, since the current make build | |
| # method doesn't trigger new builds for dependent updates. | |
| - name: 'Build Wolfi' | |
| run: | | |
| make \ | |
| ARCH=${{ matrix.arch }} \ | |
| MELANGE_EXTRA_OPTS="--keyring-append=/gcsfuse/wolfi-registry/wolfi-signing.rsa.pub" \ | |
| all -j1 | |
| # Always run this step for https://github.com/wolfi-dev/os/issues/8698 | |
| - if: ${{ always() }} | |
| name: 'Create artifacts tarball' | |
| run: | | |
| set -x | |
| set -e | |
| set -o pipefail | |
| # Pick up any stragglers that didn't get uploaded in previous builds. | |
| cat ./packages/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX | awk -F':' '$1 == "P" {printf "%s-", $2} $1 == "V" {printf "%s.apk\n", $2}' | sort > indexed.txt | |
| # TODO: Figure out why ls through gcsfuse is so slow. | |
| gcloud storage ls gs://wolfi-production-registry-destination/os/${{ matrix.arch }} | grep ".apk$" | xargs -n1 basename | sort > uploaded.txt | |
| # Lines that are only in uploaded.txt and not indexed.txt. | |
| comm -13 indexed.txt uploaded.txt > missing.txt | |
| # Clean up the symlinks to keep only packages we built. | |
| find ./packages/${{ matrix.arch }} -type l -exec rm -f {} \; | |
| # Merge any missing APKs into our new index. | |
| for missed in $(cat missing.txt); do | |
| # We could do this in one command instead of a loop, but it takes things on argv, which is a bit annoying. | |
| melange index --merge \ | |
| --source ./packages/${{ matrix.arch }}/APKINDEX.tar.gz \ | |
| --output new.tar.gz \ | |
| /gcsfuse/wolfi-registry/${{ matrix.arch }}/${missed} | |
| # Overwrite what we're going to upload (and for the next loop). | |
| mv new.tar.gz ./packages/${{ matrix.arch }}/APKINDEX.tar.gz | |
| done | |
| diff \ | |
| <(cat /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX) \ | |
| <(cat ./packages/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX) || true | |
| # Create an archive for uploading | |
| tar -cvzf /tmp/packages-${{ matrix.arch }}.tar.gz ./packages/${{ matrix.arch }} | |
| # Always run this step for https://github.com/wolfi-dev/os/issues/8698 | |
| - if: ${{ always() }} | |
| name: 'Upload built packages archive to GitHub Artifacts' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: packages-${{ matrix.arch }} | |
| path: /tmp/packages-${{ matrix.arch }}.tar.gz | |
| retention-days: 1 # Low ttl since this is just an intermediary used once | |
| if-no-files-found: warn | |
| upload-packages: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| # Always run this job for https://github.com/wolfi-dev/os/issues/8698 | |
| if: ${{ always() }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| container: | |
| # NOTE: This step only signs and uploads, so it doesn't need any privileges | |
| image: ghcr.io/wolfi-dev/sdk:latest@sha256:110c4bc0a8941606034ee7af12f1197b4a6b6f6434fd4b4bbf61de501e18ffd1 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: 'Trust the github workspace' | |
| run: | | |
| # This is to avoid fatal errors about "dubious ownership" because we are | |
| # running inside of a container action with the workspace mounted in. | |
| git config --global --add safe.directory "$(pwd)" | |
| - name: 'Download x86_64 package archives' | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: /tmp/artifacts/ | |
| name: packages-x86_64 | |
| - name: 'Download aarch64 package archives' | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: /tmp/artifacts/ | |
| name: packages-aarch64 | |
| - name: 'Authenticate to Google Cloud' | |
| id: auth | |
| uses: google-github-actions/auth@f6de81663f7788d05bd15bcce18f0e57f23f0846 # v2.0.1 | |
| with: | |
| workload_identity_provider: "projects/618116202522/locations/global/workloadIdentityPools/prod-shared-e350/providers/prod-shared-gha" | |
| service_account: "[email protected]" | |
| - uses: google-github-actions/setup-gcloud@5a5f7b85fca43e76e53463acaa9d408a03c98d3a # v2.0.1 | |
| with: | |
| project_id: "prod-images-c6e5" | |
| - uses: 'google-github-actions/get-secretmanager-secrets@ae0d4054c32840e2ced71207a9df55161ae3debc' # v2.0.0 | |
| id: secrets | |
| with: | |
| secrets: |- | |
| token:prod-images-c6e5/melange-signing-key | |
| - run: echo "${{ steps.secrets.outputs.token }}" > ./wolfi-signing.rsa | |
| - run: | | |
| mkdir -p /etc/apk/keys | |
| cp ./wolfi-signing.rsa.pub /etc/apk/keys/wolfi-signing.rsa.pub | |
| - name: 'Update the APKINDEX' | |
| run: | | |
| for arch in "x86_64" "aarch64"; do | |
| mkdir -p ./packages/${arch} | |
| # Consolidate with the built artifacts | |
| tar xvf /tmp/artifacts/packages-${arch}.tar.gz | |
| # Sign the APK index | |
| melange sign-index -f --signing-key ./wolfi-signing.rsa packages/${arch}/APKINDEX.tar.gz | |
| # Only attempt to sign when *.apk's exist. | |
| apks=$(ls ./packages/${arch}/*.apk 2>/dev/null || true) | |
| if [ -n "$apks" ]; then | |
| melange sign --signing-key ./wolfi-signing.rsa ./packages/${arch}/*.apk | |
| fi | |
| done | |
| # Clean up the signing key before uploading to storage out | |
| # of an abundance of caution. | |
| - run: rm ./wolfi-signing.rsa | |
| - name: 'Upload packages to GCS' | |
| run: | | |
| for arch in "x86_64" "aarch64"; do | |
| # Only attempt to upload when *.apk's exist | |
| apks=$(ls ./packages/${arch}/*.apk 2>/dev/null || true) | |
| if [ -n "$apks" ]; then | |
| # apks will be cached in CDN for an hour by default. | |
| # Don't upload the object if it already exists. | |
| gcloud --quiet storage cp \ | |
| --no-clobber \ | |
| "./packages/${arch}/*.apk" "gs://wolfi-production-registry-destination/os/${arch}/" | |
| fi | |
| done | |
| - name: 'Create APKINDEX tarball' | |
| run: | | |
| # Tar up any 'APKINDEX.*' files {aarch64,x86_64} x {tar.gz,json} | |
| find ./packages/ -name 'APKINDEX.*' > to-include | |
| tar -cvzf /tmp/indexes.tar.gz --files-from to-include | |
| - name: 'Upload APKINDEX archive to GitHub Artifacts' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: indexes | |
| path: /tmp/indexes.tar.gz | |
| retention-days: 1 # Low ttl since this is just an intermediary used once | |
| if-no-files-found: warn | |
| upload-index: | |
| runs-on: ubuntu-latest | |
| needs: upload-packages | |
| permissions: | |
| id-token: write | |
| contents: read | |
| container: | |
| # NOTE: This step only signs and uploads, so it doesn't need any privileges | |
| image: ghcr.io/wolfi-dev/sdk:latest@sha256:110c4bc0a8941606034ee7af12f1197b4a6b6f6434fd4b4bbf61de501e18ffd1 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: 'Trust the github workspace' | |
| run: | | |
| # This is to avoid fatal errors about "dubious ownership" because we are | |
| # running inside of a container action with the workspace mounted in. | |
| git config --global --add safe.directory "$(pwd)" | |
| - id: auth | |
| name: 'Authenticate to Google Cloud' | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| workload_identity_provider: "projects/618116202522/locations/global/workloadIdentityPools/prod-shared-e350/providers/prod-shared-gha" | |
| service_account: "[email protected]" | |
| - uses: google-github-actions/setup-gcloud@v1 | |
| with: | |
| project_id: prod-images-c6e5 | |
| - name: 'Download index archive' | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: /tmp/artifacts/ | |
| name: indexes | |
| - name: 'Upload indexes to GCS' | |
| run: | | |
| tar xvf /tmp/artifacts/indexes.tar.gz | |
| for arch in "x86_64" "aarch64"; do | |
| # Don't cache the APKINDEX. | |
| gcloud --quiet storage cp \ | |
| --cache-control=no-store \ | |
| "./packages/${arch}/APKINDEX.tar.gz" "gs://wolfi-production-registry-destination/os/${arch}/" | |
| gcloud --quiet storage cp \ | |
| --cache-control=no-store \ | |
| "./packages/${arch}/APKINDEX.json" "gs://wolfi-production-registry-destination/os/${arch}/" | |
| done | |
| postrun: | |
| name: Notify Slack | |
| runs-on: ubuntu-latest | |
| if: failure() | |
| needs: [build, upload-packages, upload-index] | |
| steps: | |
| - uses: rtCamp/[email protected] | |
| env: | |
| SLACK_ICON: http://github.com/chainguard-dev.png?size=48 | |
| SLACK_USERNAME: guardian | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_CHANNEL: C047DK5BUNP | |
| SLACK_MSG_AUTHOR: wolfi-bot | |
| SLACK_COLOR: '#8E1600' | |
| MSG_MINIMAL: 'true' | |
| SLACK_TITLE: '[build-wolfi-os] failure: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' | |
| SLACK_MESSAGE: | | |
| https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} |