build.yaml

name: Build Wolfi OS

on:
  push:
    branches: ['main']
    paths-ignore:
      - '**.md'
      - '**.txt'

  workflow_dispatch:

# Only run one build at a time to prevent out of sync signatures.
concurrency: build

jobs:
  build:
    name: Build packages
    if: github.repository == 'wolfi-dev/os'

    strategy:
      matrix:
        arch: [ "x86_64", "aarch64" ]
      fail-fast: false

    runs-on:
      group: wolfi-os-builder-${{ matrix.arch }}

    permissions:
      contents: read

    container:
      image: ghcr.io/wolfi-dev/sdk:latest@sha256:0bfd338bd2a3bb228c3f8c57da88b5052b44b42f2f1d90c2489e020aed5f4fbb
      # TODO: Deprivilege
      options: |
        --cap-add NET_ADMIN --cap-add SYS_ADMIN --device /dev/fuse --security-opt seccomp=unconfined --security-opt apparmor:unconfined

    steps:
      - uses: actions/checkout@v4

      - name: 'Trust the github workspace'
        run: |
          # This is to avoid fatal errors about "dubious ownership" because we are
          # running inside of a container action with the workspace mounted in.
          git config --global --add safe.directory "$(pwd)"

      # Build with a local key, we'll resign this with the real key later
      - name: 'Generate local signing key'
        run: |
          make local-melange.rsa

          # Touch it with the epoch date to convince `make` that we don't need to
          # rebuild the targets that depend on this (all)
          touch -d @0 local-melange.rsa

      - name: 'Prepare package repository'
        run: |
          # yay wolfi!
          apk add gcsfuse google-cloud-sdk

          # Set up a gcsfuse RO mount to the public bucket. This is a cheap and
          # cheerful way to recreate the make targets (class A HEADs) locally
          # without syncing the whole bucket (class A+B).
          mkdir -p /gcsfuse/wolfi-registry
          gcsfuse -o ro --implicit-dirs --only-dir os wolfi-production-registry-destination /gcsfuse/wolfi-registry

          mkdir -p ./packages/${{ matrix.arch }}
          # Symlink the gcsfuse mount to ./packages/ to workaround the Makefile CWD assumptions
          for f in /gcsfuse/wolfi-registry/${{ matrix.arch }}/*.apk; do
            ln -s "$f" ./packages/${{ matrix.arch }}/
          done

          # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
          cp /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.* ./packages/${{ matrix.arch }}/

      # TODO: Replace this with wolfictl build, since the current make build
      # method doesn't trigger new builds for dependent updates.
      - name: 'Build Wolfi'
        run: |
          make \
            ARCH=${{ matrix.arch }} \
            MELANGE_EXTRA_OPTS="--keyring-append=/gcsfuse/wolfi-registry/wolfi-signing.rsa.pub" \
            all -j1

      # Always run this step for https://github.com/wolfi-dev/os/issues/8698
      - if: ${{ always() }}
        name: 'Create artifacts tarball'
        run: |
          set -x
          set -e
          set -o pipefail

          # Pick up any stragglers that didn't get uploaded in previous builds.
          cat ./packages/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX | awk -F':' '$1 == "P" {printf "%s-", $2} $1 == "V" {printf "%s.apk\n", $2}' | sort > indexed.txt
          # TODO: Figure out why ls through gcsfuse is so slow.
          gcloud storage ls gs://wolfi-production-registry-destination/os/${{ matrix.arch }} | grep ".apk$" | xargs -n1 basename | sort > uploaded.txt

          # Lines that are only in uploaded.txt and not indexed.txt.
          comm -13 indexed.txt uploaded.txt > missing.txt

          # Clean up the symlinks to keep only packages we built.
          find ./packages/${{ matrix.arch }} -type l -exec rm -f {} \;

          # Merge any missing APKs into our new index.
          for missed in $(cat missing.txt); do
            # We could do this in one command instead of a loop, but it takes things on argv, which is a bit annoying.
            melange index --merge \
              --source ./packages/${{ matrix.arch }}/APKINDEX.tar.gz \
              --output new.tar.gz \
              /gcsfuse/wolfi-registry/${{ matrix.arch }}/${missed}

            # Overwrite what we're going to upload (and for the next loop).
            mv new.tar.gz ./packages/${{ matrix.arch }}/APKINDEX.tar.gz
          done

          diff \
            <(cat /gcsfuse/wolfi-registry/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX) \
            <(cat ./packages/${{ matrix.arch }}/APKINDEX.tar.gz | tar -Oxz APKINDEX) || true

          # Create an archive for uploading
          tar -cvzf /tmp/packages-${{ matrix.arch }}.tar.gz ./packages/${{ matrix.arch }}

      # Always run this step for https://github.com/wolfi-dev/os/issues/8698
      - if: ${{ always() }}
        name: 'Upload built packages archive to GitHub Artifacts'
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
        with:
          name: packages-${{ matrix.arch }}
          path: /tmp/packages-${{ matrix.arch }}.tar.gz
          retention-days: 1 # Low ttl since this is just an intermediary used once
          if-no-files-found: warn

  upload-packages:
    runs-on: ubuntu-latest
    needs: build

    # Always run this job for https://github.com/wolfi-dev/os/issues/8698
    if: ${{ always() }}

    permissions:
      id-token: write
      contents: read

    container:
      # NOTE: This step only signs and uploads, so it doesn't need any privileges
      image: ghcr.io/wolfi-dev/sdk:latest@sha256:0bfd338bd2a3bb228c3f8c57da88b5052b44b42f2f1d90c2489e020aed5f4fbb

    steps:
      - uses: actions/checkout@v4

      - name: 'Trust the github workspace'
        run: |
          # This is to avoid fatal errors about "dubious ownership" because we are
          # running inside of a container action with the workspace mounted in.
          git config --global --add safe.directory "$(pwd)"

      - name: 'Download x86_64 package archives'
        uses: actions/download-artifact@21e5c25de9cf2ee24742cd3e822327f3be6dd2a3 # v4.1.1
        with:
          path: /tmp/artifacts/
          name: packages-x86_64

      - name: 'Download aarch64 package archives'
        uses: actions/download-artifact@21e5c25de9cf2ee24742cd3e822327f3be6dd2a3 # v4.1.1
        with:
          path: /tmp/artifacts/
          name: packages-aarch64

      # This is managed here: https://github.com/chainguard-dev/secrets/blob/main/wolfi-dev.tf
      - uses: google-github-actions/auth@a6e2e39c0a0331da29f7fd2c2a20a427e8d3ad1f # v2.1.1
        id: auth
        with:
          workload_identity_provider: "projects/12758742386/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
          service_account: "wolfi-dev@chainguard-github-secrets.iam.gserviceaccount.com"
      - uses: google-github-actions/setup-gcloud@5a5f7b85fca43e76e53463acaa9d408a03c98d3a # v2.0.1
        with:
          project_id: "chainguard-github-secrets"
      - uses: 'google-github-actions/get-secretmanager-secrets@ae0d4054c32840e2ced71207a9df55161ae3debc' # v2.0.0
        id: secrets
        with:
          secrets: |-
            token:chainguard-github-secrets/wolfi-dev-signing-key

      - run: echo "${{ steps.secrets.outputs.token }}" > ./wolfi-signing.rsa
      - run: |
          mkdir -p /etc/apk/keys
          cp ./wolfi-signing.rsa.pub /etc/apk/keys/wolfi-signing.rsa.pub

      - name: 'Update the APKINDEX'
        run: |
          for arch in "x86_64" "aarch64"; do
            mkdir -p ./packages/${arch}

            # Consolidate with the built artifacts
            tar xvf /tmp/artifacts/packages-${arch}.tar.gz

            # Sign the APK index
            melange sign-index -f --signing-key ./wolfi-signing.rsa packages/${arch}/APKINDEX.tar.gz

            # Only attempt to sign when *.apk's exist.
            apks=$(ls ./packages/${arch}/*.apk 2>/dev/null || true)
            if [ -n "$apks" ]; then
              melange sign --signing-key ./wolfi-signing.rsa ./packages/${arch}/*.apk
            fi
          done

      # Clean up the signing key before uploading to storage out
      # of an abundance of caution.
      - run: rm ./wolfi-signing.rsa

      # We use a different GSA for our interaction with GCS.
      - uses: google-github-actions/auth@f6de81663f7788d05bd15bcce18f0e57f23f0846 # v2.0.1
        with:
          workload_identity_provider: "projects/618116202522/locations/global/workloadIdentityPools/prod-shared-e350/providers/prod-shared-gha"
          service_account: "prod-images-ci@prod-images-c6e5.iam.gserviceaccount.com"
      - uses: google-github-actions/setup-gcloud@5a5f7b85fca43e76e53463acaa9d408a03c98d3a # v2.0.1
        with:
          project_id: "prod-images-c6e5"

      - name: 'Upload packages to GCS'
        run: |
          for arch in "x86_64" "aarch64"; do
            # Only attempt to upload when *.apk's exist
            apks=$(ls ./packages/${arch}/*.apk 2>/dev/null || true)
            if [ -n "$apks" ]; then
              # apks will be cached in CDN for an hour by default.
              # Don't upload the object if it already exists.
              gcloud --quiet storage cp \
                  --no-clobber \
                  "./packages/${arch}/*.apk" "gs://wolfi-production-registry-destination/os/${arch}/"
            fi
          done

      - name: 'Create APKINDEX tarball'
        run: |
          # Tar up any 'APKINDEX.*' files {aarch64,x86_64} x {tar.gz,json}
          find ./packages/ -name 'APKINDEX.*' > to-include
          tar -cvzf /tmp/indexes.tar.gz --files-from to-include

      - name: 'Upload APKINDEX archive to GitHub Artifacts'
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
        with:
          name: indexes
          path: /tmp/indexes.tar.gz
          retention-days: 1 # Low ttl since this is just an intermediary used once
          if-no-files-found: warn

  upload-index:
    runs-on: ubuntu-latest
    needs: upload-packages

    permissions:
      id-token: write
      contents: read

    container:
      # NOTE: This step only signs and uploads, so it doesn't need any privileges
      image: ghcr.io/wolfi-dev/sdk:latest@sha256:0bfd338bd2a3bb228c3f8c57da88b5052b44b42f2f1d90c2489e020aed5f4fbb

    steps:
      - uses: actions/checkout@v4

      - name: 'Trust the github workspace'
        run: |
          # This is to avoid fatal errors about "dubious ownership" because we are
          # running inside of a container action with the workspace mounted in.
          git config --global --add safe.directory "$(pwd)"

      - id: auth
        name: 'Authenticate to Google Cloud'
        uses: google-github-actions/auth@v1
        with:
          workload_identity_provider: "projects/618116202522/locations/global/workloadIdentityPools/prod-shared-e350/providers/prod-shared-gha"
          service_account: "prod-images-ci@prod-images-c6e5.iam.gserviceaccount.com"

      - uses: google-github-actions/setup-gcloud@v1
        with:
          project_id: prod-images-c6e5

      - name: 'Download index archive'
        uses: actions/download-artifact@21e5c25de9cf2ee24742cd3e822327f3be6dd2a3 # v4.1.1
        with:
          path: /tmp/artifacts/
          name: indexes

      - name: 'Upload indexes to GCS'
        run: |
          tar xvf /tmp/artifacts/indexes.tar.gz

          for arch in "x86_64" "aarch64"; do
            # Don't cache the APKINDEX.
            gcloud --quiet storage cp \
                --cache-control=no-store \
                "./packages/${arch}/APKINDEX.tar.gz" "gs://wolfi-production-registry-destination/os/${arch}/"

            gcloud --quiet storage cp \
                --cache-control=no-store \
                "./packages/${arch}/APKINDEX.json" "gs://wolfi-production-registry-destination/os/${arch}/"
          done

  postrun:
    name: Notify Slack
    runs-on: ubuntu-latest
    if: failure()
    needs: [build, upload-packages, upload-index]
    steps:
      - uses: rtCamp/action-slack-notify@v2.2.1
        env:
          SLACK_ICON: http://github.com/chainguard-dev.png?size=48
          SLACK_USERNAME: guardian
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_CHANNEL: C047DK5BUNP
          SLACK_MSG_AUTHOR: wolfi-bot
          SLACK_COLOR: '#8E1600'
          MSG_MINIMAL: 'true'
          SLACK_TITLE: '[build-wolfi-os] failure: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
          SLACK_MESSAGE: |
            https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}