docker-build-and-test.yml

---
# Test built docker images by building simple projects inside them

name: dagger
on:
  pull_request:
    paths:
      - '.dagger-ci'
      - '.github/workflows/docker-build-and-test.yml'
      - 'docker/**'
      - 'tests/**'
  push:
    branches: ['main']
    paths:
      - '.dagger-ci'
      - '.github/workflows/docker-build-and-test.yml'
      - 'docker/**'
      - 'tests/**'
  release:
  schedule:
    # First day of the month at midnight
    - cron: '0 0 1 * 0'
  workflow_dispatch:

env:
  REGISTRY: ghcr.io

permissions:
  contents: read

jobs:
  #=============================
  # Dynamically generate matrix
  #=============================

  get-matrix:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: get-matrix
        id: get-matrix
        run: |
          # Disable SC2046: Quote this to prevent word splitting
          # I can't quote this, just look at it
          # shellcheck disable=SC2046
          echo matrix=$( yq --input-format yaml --output-format json '.services | keys[]' docker/compose.yaml | sed 's/"//g' | jq -Rs 'split("\n") | del(.[-1])' | jq -c ) >> "${GITHUB_OUTPUT}"
      - name: Check
        run: |
          jq . <<< '${{ steps.get-matrix.outputs.matrix }}'
    outputs:
      matrix: ${{ steps.get-matrix.outputs.matrix }}

  get-matrix-coreboot:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: get-matrix
        id: get-matrix
        run: |
          # shellcheck disable=SC2046
          echo matrix=$( yq --input-format yaml --output-format json '.services | keys[] | select(. | test("coreboot.*"))' docker/compose.yaml | sed 's/"//g' | jq -Rs 'split("\n") | del(.[-1])' | jq -c ) >> "${GITHUB_OUTPUT}"
      - name: Check
        run: |
          jq . <<< '${{ steps.get-matrix.outputs.matrix }}'
    outputs:
      matrix: ${{ steps.get-matrix.outputs.matrix }}

  #=====================
  # Coreboot toolchains
  #=====================

  build-coreboot-toolchains:
    timeout-minutes: 120
    needs:
      - get-matrix-coreboot
    strategy:
      fail-fast: false
      matrix:
        arch: ['amd64', 'arm64']
        dockerfile: ${{ fromJson(needs.get-matrix-coreboot.outputs.matrix) }}
    runs-on: ${{ matrix.arch == 'arm64' && 'ARM64' || 'ubuntu-latest' }}
    container:
      # At the time of writing (2024-10) we cannot use ubuntu:noble as it is broken
      image: ubuntu:jammy
    env:
      DEBIAN_FRONTEND: noninteractive
      # Use coreboot mirrors
      BUILDGCC_OPTIONS: -m
    steps:
      - name: Install dependencies for CI
        run: |
          apt-get update
          apt-get install -y --no-install-recommends \
            build-essential \
            ca-certificates \
            curl \
            git \
            git-lfs \
            jq \
            sudo \
            tzdata \
            upx-ucl \
            wget
          update-ca-certificates
      - name: Get yq
        # the --no-check-certificate is needed because GitHub
        run: |
          wget -q --no-check-certificate -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${{ matrix.arch }}
          chmod 755 /usr/local/bin/yq
      - name: Configure tzdata
        run: |
          dpkg-reconfigure --frontend noninteractive tzdata
      - name: Checkout
        uses: actions/checkout@v4
      - name: Get coreboot version
        id: version
        run: |
          yq -r '.services.["${{ matrix.dockerfile }}"].build.args[] | select(test("COREBOOT_VERSION=.*"))' docker/compose.yaml >> "${GITHUB_OUTPUT}"

      - name: Clone coreboot
        run: |
          git clone --depth 1 "https://review.coreboot.org/coreboot.git" -b "${{ steps.version.outputs.COREBOOT_VERSION }}"

      - name: Get coreboot commit hash
        id: coreboot-hash
        run: |
          cd coreboot
          COREBOOT_HASH="$( git rev-parse --short HEAD )"
          echo "${COREBOOT_HASH}"
          echo "COREBOOT_HASH=${COREBOOT_HASH}" >> "${GITHUB_OUTPUT}"

      - name: Check if toolchain is stored in firmware-action-toolchains repo
        continue-on-error: true
        run: |
          # Check if the toolchain exists without downloading it
          wget --no-verbose --tries=3 "https://github.com/9elements/firmware-action-toolchains/raw/refs/heads/main/coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar.sha256"
      - name: Check if toolchain exist
        id: toolchains-exist
        run: |
          if [ -f "${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar.sha256" ]; then
            echo "toolchain is stored in firmware-action-toolchains repository, skipping rest of the job"
            echo "EXIST=true" >> "${GITHUB_OUTPUT}"
          else
            echo "toolchain is NOT stored in firmware-action-toolchains repository, will build it"
            echo "EXIST=false" >> "${GITHUB_OUTPUT}"
          fi

      - name: Cache key
        id: cache-key
        run: |
          CACHE_KEY="coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc"
          echo "${CACHE_KEY}"
          echo "CACHE_KEY=${CACHE_KEY}" >> "${GITHUB_OUTPUT}"
      - name: Tar filename
        id: tar-filename
        run: |
          TAR_FILENAME="${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar"
          echo "${TAR_FILENAME}"
          echo "TAR_FILENAME=${TAR_FILENAME}" >> "${GITHUB_OUTPUT}"

      - name: Restore cached toolchains
        id: cache-toolchains
        uses: actions/cache/restore@v4
        if: steps.toolchains-exist.outputs.EXIST == 'false'
        with:
          path: |
            ${{ steps.tar-filename.outputs.TAR_FILENAME }}
            ${{ steps.tar-filename.outputs.TAR_FILENAME }}.sha256
          key: ${{ steps.cache-key.outputs.CACHE_KEY }}

      - name: Install dependencies if needed
        # != 'true' because on miss the cache-hit is empty
        if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
        run: |
          apt-get install -y --no-install-recommends \
            acpica-tools \
            bc \
            bison \
            bsdmainutils \
            build-essential \
            flex \
            gnat \
            imagemagick \
            libelf-dev \
            libncurses5-dev \
            libnss3-dev \
            libssl-dev \
            m4 \
            nasm \
            openssh-client \
            pkgconf \
            python-is-python3 \
            python3-pip \
            qemu-system-x86 \
            upx-ucl \
            uuid-dev \
            zlib1g-dev
      - name: Install dependencies if needed (amd64)
        if: matrix.arch == 'amd64' && steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
        run: |
          apt-get install -y --no-install-recommends \
            iucode-tool

      - name: Build coreboot toolchains
        if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
        run: |
          cd coreboot
          make crossgcc CPUS="$(nproc)"
      - name: Compress toolchain binaries
        # This step should shrink the size of single toolchain from 1.5 GB down to around 700 MB
        # I think it is save to compress all binaries except libraries, hence the '-wholename'
        if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
        run: |
          cd coreboot/util/crossgcc/xgcc
          # shellcheck disable=SC2016
          find . -type f -wholename '*/bin/*' -exec bash -c 'upx-ucl -9 "$1"' shell {} \; || true

      - name: Tar toolchain to prevent permission loss
        if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
        run: |
          # Docs: https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss
          mv "coreboot/util/crossgcc/xgcc" "coreboot/util/crossgcc/${{ matrix.arch }}-xgcc"
          tar -cf "${{ steps.tar-filename.outputs.TAR_FILENAME }}" "coreboot/util/crossgcc/${{ matrix.arch }}-xgcc"
          sha256sum "${{ steps.tar-filename.outputs.TAR_FILENAME }}" > "${{ steps.tar-filename.outputs.TAR_FILENAME }}.sha256"

      # Store toolchains and utils in cache
      - name: Cache toolchains
        uses: actions/cache/save@v4
        if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
        with:
          path: |
            ${{ steps.tar-filename.outputs.TAR_FILENAME }}
            ${{ steps.tar-filename.outputs.TAR_FILENAME }}.sha256
          key: ${{ steps.cache-key.outputs.CACHE_KEY }}


      - name: Checkout
        uses: actions/checkout@v4
        if: steps.toolchains-exist.outputs.EXIST == 'false'
        with:
          repository: '9elements/firmware-action-toolchains'
          path: 'firmware-action-toolchains'
          ref: 'main'
          lfs: false
          token: ${{ secrets.GH_PAT_TOOLCHAINS }}
      - name: Set up Git
        if: steps.toolchains-exist.outputs.EXIST == 'false'
        run: |
          cd firmware-action-toolchains
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git lfs install
      - name: Prepare files for firmware-action-toolchains repository
        if: steps.toolchains-exist.outputs.EXIST == 'false'
        run: |
          # Clone repo without downloading LFS items
          cd firmware-action-toolchains
          mkdir -p "coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/"
          mv "../${{ steps.tar-filename.outputs.TAR_FILENAME }}" "coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/"
          mv "../${{ steps.tar-filename.outputs.TAR_FILENAME }}.sha256" "coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/"
      - name: Create pull request in firmware-action-toolchains repository
        uses: peter-evans/create-pull-request@v7
        if: steps.toolchains-exist.outputs.EXIST == 'false'
        with:
          path: 'firmware-action-toolchains'
          token: ${{ secrets.GH_PAT_TOOLCHAINS }}
          add-paths: |
            coreboot/**
          branch: "feat/${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}"
          commit-message: "feat: add toolchain for coreboot ${{ steps.version.outputs.COREBOOT_VERSION }}"


  #=========================
  # Build Docker containers
  #=========================

  build:
    name: build_test_publish
    runs-on: ubuntu-latest
    timeout-minutes: 60
    needs:
      - get-matrix
      - build-coreboot-toolchains
    strategy:
      fail-fast: false
      matrix:
        dockerfile: ${{ fromJson(needs.get-matrix.outputs.matrix) }}
    permissions:
      contents: read
      packages: write
    steps:
      # We have to use my own fork of actions/delete-package-versions at the moment
      #   to have access to 'dry-run' and 'ignore-versions-include-tags' features
      # We can switch to upstream whe following PRs get merged:
      #   - [dry-run](https://github.com/actions/delete-package-versions/pull/119/commits)
      #   - [tags](https://github.com/actions/delete-package-versions/pull/104
      - name: Delete old packages
        uses: AtomicFS/delete-package-versions@main
        continue-on-error:
          true
          # we have continue-on-error because when I make a fork of this repo to debug something,
          # the Docker containers would not build because this step fails to fetch existing containers
          # (in fresh fork there are none)
        with:
          package-name: firmware-action/${{ matrix.dockerfile }}
          package-type: container
          min-versions-to-keep: 5
          ignore-versions:
            '^(main|latest|v(\d+\.?)+)$'
            # ignore:
            # - main
            # - latest
            # - vX
            # - vX.X
            # - vX.X.X
          dry-run: false
          ignore-versions-include-tags: true

      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: '3.x'

      - name: Checkout
        uses: actions/checkout@v4

      - name: Get coreboot version
        id: version
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: yq -r '.services.["${{ matrix.dockerfile }}"].build.args[] | select(test("COREBOOT_VERSION=.*"))' docker/compose.yaml >> "${GITHUB_OUTPUT}"
      - name: Clone coreboot
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          git clone --depth 1 "https://review.coreboot.org/coreboot.git" -b "${{ steps.version.outputs.COREBOOT_VERSION }}"
      - name: Get coreboot commit hash
        id: coreboot-hash
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          cd coreboot
          COREBOOT_HASH="$( git rev-parse --short HEAD )"
          echo "${COREBOOT_HASH}"
          echo "COREBOOT_HASH=${COREBOOT_HASH}" >> "${GITHUB_OUTPUT}"

      #=================================
      # Download artifacts for coreboot
      #=================================

      - name: Download coreboot toolchains from firmware-action-toolchains repository
        id: firmware-action-toolchains
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          for arch in "amd64" "arm64"; do
            wget --continue --no-verbose --tries=3 "https://github.com/9elements/firmware-action-toolchains/raw/refs/heads/main/coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${arch}-xgcc.tar"
            wget --continue --no-verbose --tries=3 "https://github.com/9elements/firmware-action-toolchains/raw/refs/heads/main/coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${arch}-xgcc.tar.sha256"
            sha256sum -c "${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${arch}-xgcc.tar.sha256";
          done

      - name: Prepare toolchains
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          mkdir -p docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}
          for f in ${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-*-xgcc.tar; do
            ARCH=$( basename "${f}" | sed -E 's/[a-z0-9]{8}-([a-z0-9_-]*)-xgcc.*/\1/g' )
            echo "extracting ${f} -> ${{ steps.version.outputs.COREBOOT_VERSION }} / ${ARCH}"
            mkdir -p "${f}.dir/"
            tar -xf "${f}" -C "${f}.dir/"
            mv "${f}.dir/coreboot/util/crossgcc/${ARCH}-xgcc" "docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-${ARCH}"
            rm -rf "${f}"
          done

      - name: Debug list artifacts
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/
      - name: Debug list xgcc (amd64)
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-*
      - name: Debug list xgcc/bin (amd64)
        if: startsWith(matrix.dockerfile, 'coreboot')
        run: |
          ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-*/bin

      #============================
      # Build the docker container
      #============================

      - name: Setup docker-compose
        uses: KengoTODA/actions-setup-docker-compose@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Validate compose file
        run: docker-compose -f docker/compose.yaml config

      - name: Install python dependencies
        run: pip install -r ./.dagger-ci/daggerci/requirements.txt

      - name: Run dagger pipeline
        timeout-minutes: 60
        run: |
          if [[ "${GITHUB_EVENT_NAME}" == 'release' ]] || [[ "${GITHUB_REF}" == *'main' ]] || [[ "${GITHUB_REF_TYPE}" == 'tag' ]]; then
            echo "Enable publishing"
            python .dagger-ci/daggerci/main.py -d ${{ matrix.dockerfile }} --publish
          else
            echo "Disable publishing"
            python .dagger-ci/daggerci/main.py -d ${{ matrix.dockerfile }}
          fi
        shell: bash
        env:
          GITHUB_REGISTRY: ${{ env.REGISTRY }}
          GITHUB_ACTOR: ${{ github.actor }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}