Skip to content
dagger

Ci/store coreboot toolchains in new repository #1238

Sign in to view logs

Ci/store coreboot toolchains in new repository

Ci/store coreboot toolchains in new repository #1238

Workflow file for this run

.github/workflows/docker-build-and-test.yml at a520826
---
# Test built docker images by building simple projects inside them
name: dagger
on:
pull_request:
paths:
- '.dagger-ci'
- '.github/workflows/docker-build-and-test.yml'
- 'docker/**'
- 'tests/**'
push:
branches: ['main']
paths:
- '.dagger-ci'
- '.github/workflows/docker-build-and-test.yml'
- 'docker/**'
- 'tests/**'
release:
schedule:
# First day of the month at midnight
- cron: '0 0 1 * 0'
workflow_dispatch:
env:
REGISTRY: ghcr.io
permissions:
contents: read
jobs:
#=============================
# Dynamically generate matrix
#=============================
get-matrix:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: get-matrix
id: get-matrix
run: |
# Disable SC2046: Quote this to prevent word splitting
# I can't quote this, just look at it
# shellcheck disable=SC2046
echo matrix=$( yq --input-format yaml --output-format json '.services | keys[]' docker/compose.yaml | sed 's/"//g' | jq -Rs 'split("\n") | del(.[-1])' | jq -c ) >> "${GITHUB_OUTPUT}"
- name: Check
run: |
jq . <<< '${{ steps.get-matrix.outputs.matrix }}'
outputs:
matrix: ${{ steps.get-matrix.outputs.matrix }}
get-matrix-coreboot:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: get-matrix
id: get-matrix
run: |
# shellcheck disable=SC2046
echo matrix=$( yq --input-format yaml --output-format json '.services | keys[] | select(. | test("coreboot.*"))' docker/compose.yaml | sed 's/"//g' | jq -Rs 'split("\n") | del(.[-1])' | jq -c ) >> "${GITHUB_OUTPUT}"
- name: Check
run: |
jq . <<< '${{ steps.get-matrix.outputs.matrix }}'
outputs:
matrix: ${{ steps.get-matrix.outputs.matrix }}
#=====================
# Coreboot toolchains
#=====================
build-coreboot-toolchains:
timeout-minutes: 120
needs:
- get-matrix-coreboot
strategy:
fail-fast: false
matrix:
arch: ['amd64', 'arm64']
dockerfile: ${{ fromJson(needs.get-matrix-coreboot.outputs.matrix) }}
runs-on: ${{ matrix.arch == 'arm64' && 'ARM64' || 'ubuntu-latest' }}
container:
# At the time of writing (2024-10) we cannot use ubuntu:noble as it is broken
image: ubuntu:jammy
env:
DEBIAN_FRONTEND: noninteractive
# Use coreboot mirrors
BUILDGCC_OPTIONS: -m
steps:
- name: Install dependencies for CI
run: |
apt-get update
apt-get install -y --no-install-recommends \
build-essential \
ca-certificates \
curl \
git \
git-lfs \
jq \
sudo \
tzdata \
upx-ucl \
wget
update-ca-certificates
- name: Get yq
# the --no-check-certificate is needed because GitHub
run: |
wget -q --no-check-certificate -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${{ matrix.arch }}
chmod 755 /usr/local/bin/yq
- name: Configure tzdata
run: |
dpkg-reconfigure --frontend noninteractive tzdata
- name: Checkout
uses: actions/checkout@v4
- name: Get coreboot version
id: version
run: |
yq -r '.services.["${{ matrix.dockerfile }}"].build.args[] | select(test("COREBOOT_VERSION=.*"))' docker/compose.yaml >> "${GITHUB_OUTPUT}"
- name: Clone coreboot
run: |
git clone --depth 1 "https://review.coreboot.org/coreboot.git" -b "${{ steps.version.outputs.COREBOOT_VERSION }}"
- name: Get coreboot commit hash
id: coreboot-hash
run: |
cd coreboot
COREBOOT_HASH="$( git rev-parse --short HEAD )"
echo "${COREBOOT_HASH}"
echo "COREBOOT_HASH=${COREBOOT_HASH}" >> "${GITHUB_OUTPUT}"
- name: Check if toolchain is stored in firmware-action-toolchains repo
continue-on-error: true
run: |
# Check if the toolchain exists without downloading it
wget --spider "https://raw.githubusercontent.com/9elements/firmware-action-toolchains/refs/heads/main/coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar"
touch .firmware-action-toolchains-exist
- name: Check if toolchain exist
id: toolchains-exist
run: |
if [ -f ".firmware-action-toolchains-exist" ]; then
echo "toolchain is stored in firmware-action-toolchains repository, skipping rest of the job"
echo "EXIST=true" >> "${GITHUB_OUTPUT}"
else
echo "toolchain is NOT stored in firmware-action-toolchains repository, will build it"
echo "EXIST=false" >> "${GITHUB_OUTPUT}"
fi
- name: Cache key
id: cache-key
run: |
CACHE_KEY="coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc"
echo "${CACHE_KEY}"
echo "CACHE_KEY=${CACHE_KEY}" >> "${GITHUB_OUTPUT}"
- name: Tar filename
id: tar-filename
run: |
TAR_FILENAME="${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar"
echo "${TAR_FILENAME}"
echo "TAR_FILENAME=${TAR_FILENAME}" >> "${GITHUB_OUTPUT}"
- name: Restore cached toolchains
id: cache-toolchains
uses: actions/cache/restore@v4
with:
path: |
${{ steps.tar-filename.outputs.TAR_FILENAME }}
${{ steps.tar-filename.outputs.TAR_FILENAME }}.sha256
key: ${{ steps.cache-key.outputs.CACHE_KEY }}
- name: Install dependencies if needed
# != 'true' because on miss the cache-hit is empty
if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
run: |
apt-get install -y --no-install-recommends \
acpica-tools \
bc \
bison \
bsdmainutils \
build-essential \
flex \
gnat \
imagemagick \
libelf-dev \
libncurses5-dev \
libnss3-dev \
libssl-dev \
m4 \
nasm \
openssh-client \
pkgconf \
python-is-python3 \
python3-pip \
qemu-system-x86 \
upx-ucl \
uuid-dev \
zlib1g-dev
- name: Install dependencies if needed (amd64)
if: matrix.arch == 'amd64' && steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
run: |
apt-get install -y --no-install-recommends \
iucode-tool
- name: Build coreboot toolchains
if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
run: |
cd coreboot
make crossgcc CPUS="$(nproc)"
- name: Compress toolchain binaries
# This step should shrink the size of single toolchain from 1.5 GB down to around 700 MB
# I think it is save to compress all binaries except libraries, hence the '-wholename'
if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
run: |
cd coreboot/util/crossgcc/xgcc
# shellcheck disable=SC2016
find . -type f -wholename '*/bin/*' -exec bash -c 'upx-ucl -9 "$1"' shell {} \; || true
- name: Tar toolchain to prevent permission loss
if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
run: |
# Docs: https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss
mv "coreboot/util/crossgcc/xgcc" "coreboot/util/crossgcc/${{ matrix.arch }}-xgcc"
tar -cf "${{ steps.tar-filename.outputs.TAR_FILENAME }}" "coreboot/util/crossgcc/${{ matrix.arch }}-xgcc"
sha256sum "${{ steps.tar-filename.outputs.TAR_FILENAME }}" > "${{ steps.tar-filename.outputs.TAR_FILENAME }}.sha256"
# Store toolchains and utils in cache
- name: Cache toolchains
uses: actions/cache/save@v4
if: steps.toolchains-exist.outputs.EXIST == 'false' && steps.cache-toolchains.outputs.cache-hit != 'true'
with:
path: |
${{ steps.tar-filename.outputs.TAR_FILENAME }}
${{ steps.tar-filename.outputs.TAR_FILENAME }}.sha256
key: ${{ steps.cache-key.outputs.CACHE_KEY }}
- name: Checkout
uses: actions/checkout@v4
with:
repository: '9elements/firmware-action-toolchains'
path: 'firmware-action-toolchains'
ref: 'main'
lfs: false
token: ${{ secrets.GH_PAT_TOOLCHAINS }}
- name: Set up Git
run: |
cd firmware-action-toolchains
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git lfs install
- name: Commit the toolchain into firmware-action-toolchains repository
if: steps.toolchains-exist.outputs.EXIST == 'false'
run: |
# Clone repo without downloading LFS items
cd firmware-action-toolchains
mkdir -p "coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/"
mv "../${{ steps.tar-filename.outputs.TAR_FILENAME }}" "coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/"
mv "../${{ steps.tar-filename.outputs.TAR_FILENAME }}.sha256" "coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/"
- name: Create pull request
uses: peter-evans/create-pull-request@v7
with:
path: 'firmware-action-toolchains'
token: ${{ secrets.GH_PAT_TOOLCHAINS }}
add-paths: |
coreboot/**
branch: "feat/${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}"
commit-message: "feat: add toolchain for coreboot ${{ steps.version.outputs.COREBOOT_VERSION }}"
#=========================
# Build Docker containers
#=========================
build:
name: build_test_publish
runs-on: ubuntu-latest
timeout-minutes: 60
needs:
- get-matrix
- build-coreboot-toolchains
strategy:
fail-fast: false
matrix:
dockerfile: ${{ fromJson(needs.get-matrix.outputs.matrix) }}
permissions:
contents: read
packages: write
steps:
# We have to use my own fork of actions/delete-package-versions at the moment
# to have access to 'dry-run' and 'ignore-versions-include-tags' features
# We can switch to upstream whe following PRs get merged:
# - [dry-run](https://github.com/actions/delete-package-versions/pull/119/commits)
# - [tags](https://github.com/actions/delete-package-versions/pull/104
- name: Delete old packages
uses: AtomicFS/delete-package-versions@main
continue-on-error:
true
# we have continue-on-error because when I make a fork of this repo to debug something,
# the Docker containers would not build because this step fails to fetch existing containers
# (in fresh fork there are none)
with:
package-name: firmware-action/${{ matrix.dockerfile }}
package-type: container
min-versions-to-keep: 5
ignore-versions:
'^(main|latest|v(\d+\.?)+)$'
# ignore:
# - main
# - latest
# - vX
# - vX.X
# - vX.X.X
dry-run: false
ignore-versions-include-tags: true
- name: Setup python
uses: actions/setup-python@v5
with:
python-version: '3.x'
- name: Checkout
uses: actions/checkout@v4
- name: Get coreboot version
id: version
if: startsWith(matrix.dockerfile, 'coreboot')
run: yq -r '.services.["${{ matrix.dockerfile }}"].build.args[] | select(test("COREBOOT_VERSION=.*"))' docker/compose.yaml >> "${GITHUB_OUTPUT}"
- name: Clone coreboot
if: startsWith(matrix.dockerfile, 'coreboot')
run: |
git clone --depth 1 "https://review.coreboot.org/coreboot.git" -b "${{ steps.version.outputs.COREBOOT_VERSION }}"
- name: Get coreboot commit hash
id: coreboot-hash
if: startsWith(matrix.dockerfile, 'coreboot')
run: |
cd coreboot
COREBOOT_HASH="$( git rev-parse --short HEAD )"
echo "${COREBOOT_HASH}"
echo "COREBOOT_HASH=${COREBOOT_HASH}" >> "${GITHUB_OUTPUT}"
#=================================
# Download artifacts for coreboot
#=================================
- name: Download coreboot toolchains from firmware-action-toolchains repository
id: firmware-action-toolchains
if: startsWith(matrix.dockerfile, 'coreboot')
continue-on-error: true
run: |
for arch in "amd64" "arm64"; do
wget --continue --tries=3 "https://raw.githubusercontent.com/9elements/firmware-action-toolchains/refs/heads/main/coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${arch}-xgcc.tar";
wget --continue --tries=3 "https://raw.githubusercontent.com/9elements/firmware-action-toolchains/refs/heads/main/coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${arch}-xgcc.tar.sha256";
sha256sum -c "${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${arch}-xgcc.tar.sha256";
done
- name: Prepare toolchains
if: startsWith(matrix.dockerfile, 'coreboot')
run: |
mkdir -p docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}
for f in ${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-*-xgcc/*.tar; do
ARCH=$( basename "${f}" | sed -E 's/coreboot\-[0-9\.]+\-[a-z0-9]+\-([a-z0-9]+)\-.*/\1/g' )
echo "extracting ${f} -> ${{ steps.version.outputs.COREBOOT_VERSION }} / ${ARCH}"
mkdir -p "${f}.dir/"
tar -xf "${f}" -C "${f}.dir/"
mv "${f}.dir/coreboot/util/crossgcc/${ARCH}-xgcc" "docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-${ARCH}"
rm -rf "${f}"
done
- name: Debug list artifacts
if: startsWith(matrix.dockerfile, 'coreboot')
run: |
ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/
- name: Debug list xgcc (amd64)
if: startsWith(matrix.dockerfile, 'coreboot')
run: |
ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-*
- name: Debug list xgcc/bin (amd64)
if: startsWith(matrix.dockerfile, 'coreboot')
run: |
ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-*/bin
#============================
# Build the docker container
#============================
- name: Setup docker-compose
uses: KengoTODA/actions-setup-docker-compose@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Validate compose file
run: docker-compose -f docker/compose.yaml config
- name: Install python dependencies
run: pip install -r ./.dagger-ci/daggerci/requirements.txt
- name: Run dagger pipeline
timeout-minutes: 60
run: |
if [[ "${GITHUB_EVENT_NAME}" == 'release' ]] || [[ "${GITHUB_REF}" == *'main' ]] || [[ "${GITHUB_REF_TYPE}" == 'tag' ]]; then
echo "Enable publishing"
python .dagger-ci/daggerci/main.py -d ${{ matrix.dockerfile }} --publish
else
echo "Disable publishing"
python .dagger-ci/daggerci/main.py -d ${{ matrix.dockerfile }}
fi
shell: bash
env:
GITHUB_REGISTRY: ${{ env.REGISTRY }}
GITHUB_ACTOR: ${{ github.actor }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}