Ci/store coreboot toolchains in new repository #1189

Workflow file for this run

.github/workflows/docker-build-and-test.yml at fa60d7f

	---
	# Test built docker images by building simple projects inside them

	name: dagger
	on:
	pull_request:
	paths:
	- '.dagger-ci'
	- '.github/workflows/docker-build-and-test.yml'
	- 'docker/**'
	- 'tests/**'
	push:
	branches: ['main']
	paths:
	- '.dagger-ci'
	- '.github/workflows/docker-build-and-test.yml'
	- 'docker/**'
	- 'tests/**'
	release:
	schedule:
	# First day of the month at midnight
	- cron: '0 0 1 * 0'
	workflow_dispatch:

	env:
	REGISTRY: ghcr.io

	permissions:
	contents: read

	jobs:
	#=============================
	# Dynamically generate matrix
	#=============================

	get-matrix:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: get-matrix
	id: get-matrix
	run: \|
	# Disable SC2046: Quote this to prevent word splitting
	# I can't quote this, just look at it
	# shellcheck disable=SC2046
	echo matrix=$( yq --input-format yaml --output-format json '.services \| keys[]' docker/compose.yaml \| sed 's/"//g' \| jq -Rs 'split("\n") \| del(.[-1])' \| jq -c ) >> "${GITHUB_OUTPUT}"
	- name: Check
	run: \|
	jq . <<< '${{ steps.get-matrix.outputs.matrix }}'
	outputs:
	matrix: ${{ steps.get-matrix.outputs.matrix }}

	get-matrix-coreboot:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: get-matrix
	id: get-matrix
	run: \|
	# shellcheck disable=SC2046
	echo matrix=$( yq --input-format yaml --output-format json '.services \| keys[] \| select(. \| test("coreboot.*"))' docker/compose.yaml \| sed 's/"//g' \| jq -Rs 'split("\n") \| del(.[-1])' \| jq -c ) >> "${GITHUB_OUTPUT}"
	- name: Check
	run: \|
	jq . <<< '${{ steps.get-matrix.outputs.matrix }}'
	outputs:
	matrix: ${{ steps.get-matrix.outputs.matrix }}

	#=====================
	# Coreboot toolchains
	#=====================

	build-coreboot-toolchains:
	timeout-minutes: 120
	needs:
	- get-matrix-coreboot
	strategy:
	fail-fast: false
	matrix:
	arch: ['amd64', 'arm64']
	dockerfile: ${{ fromJson(needs.get-matrix-coreboot.outputs.matrix) }}
	runs-on: ${{ matrix.arch == 'arm64' && 'ARM64' \|\| 'ubuntu-latest' }}
	container:
	# At the time of writing (2024-10) we cannot use ubuntu:noble as it is broken
	image: ubuntu:jammy
	env:
	DEBIAN_FRONTEND: noninteractive
	# Use coreboot mirrors
	BUILDGCC_OPTIONS: -m
	steps:
	- name: Install dependencies for CI
	run: \|
	apt-get update
	apt-get install -y --no-install-recommends \
	build-essential \
	ca-certificates \
	curl \
	git \
	git-lfs \
	jq \
	sudo \
	tzdata \
	upx-ucl \
	wget
	update-ca-certificates
	- name: Get yq
	# the --no-check-certificate is needed because GitHub
	run: \|
	wget -q --no-check-certificate -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${{ matrix.arch }}
	chmod 755 /usr/local/bin/yq
	- name: Configure tzdata
	run: \|
	dpkg-reconfigure --frontend noninteractive tzdata
	- name: Checkout
	uses: actions/checkout@v4
	- name: Get coreboot version
	id: version
	run: \|
	yq -r '.services.["${{ matrix.dockerfile }}"].build.args[] \| select(test("COREBOOT_VERSION=.*"))' docker/compose.yaml >> "${GITHUB_OUTPUT}"

	- name: Clone coreboot
	run: \|
	git clone --depth 1 "https://review.coreboot.org/coreboot.git" -b "${{ steps.version.outputs.COREBOOT_VERSION }}"

	- name: Get coreboot commit hash
	id: coreboot-hash
	run: \|
	cd coreboot
	COREBOOT_HASH="$( git rev-parse --short HEAD )"
	echo "${COREBOOT_HASH}"
	echo "COREBOOT_HASH=${COREBOOT_HASH}" >> "${GITHUB_OUTPUT}"
	- name: Artefact and cache key
	id: cache-key
	run: \|
	CACHE_KEY="coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}"
	echo "${CACHE_KEY}"
	echo "CACHE_KEY=${CACHE_KEY}" >> "${GITHUB_OUTPUT}"

	- name: Restore cached toolchains
	id: cache-toolchains
	uses: actions/cache/restore@v4
	with:
	path: coreboot/util/crossgcc/xgcc
	key: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc

	- name: Check if toolchain is stored in firmware-action-toolchains repo
	id: toolchains-exist
	continue-on-error: true
	run: \|
	# Check if the toolchain exists without downloading it
	wget --spider "https://raw.githubusercontent.com/9elements/firmware-action-toolchains/refs/heads/main/coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar"
	- name: Debug
	run: \|
	if [ "${{ steps.toolchains-exist.conclusion }}" == "success" ]; then
	echo "toolchain is stored in firmware-action-toolchains repository, skipping rest of the job"
	else
	echo "toolchain is NOT stored in firmware-action-toolchains repository, will build it"
	fi

	- name: Install dependencies if needed
	if: steps.toolchains-exist.conclusion == 'failure'
	run: \|
	apt-get install -y --no-install-recommends \
	acpica-tools \
	bc \
	bison \
	bsdmainutils \
	build-essential \
	flex \
	gnat \
	imagemagick \
	libelf-dev \
	libncurses5-dev \
	libnss3-dev \
	libssl-dev \
	m4 \
	nasm \
	openssh-client \
	pkgconf \
	python-is-python3 \
	python3-pip \
	qemu-system-x86 \
	upx-ucl \
	uuid-dev \
	zlib1g-dev
	- name: Install dependencies if needed (amd64)
	if: matrix.arch == 'amd64' && steps.toolchains-exist.conclusion == 'failure'
	run: \|
	apt-get install -y --no-install-recommends \
	iucode-tool

	- name: Build coreboot toolchains
	if: steps.toolchains-exist.conclusion == 'failure'
	run: \|
	cd coreboot
	make crossgcc CPUS="$(nproc)"
	- name: Compress toolchain binaries
	# This step should shrink the size of single toolchain from 1.5 GB down to around 700 MB
	# I think it is save to compress all binaries except libraries, hence the '-wholename'
	if: steps.toolchains-exist.conclusion == 'failure'
	run: \|
	cd coreboot/util/crossgcc/xgcc
	# shellcheck disable=SC2016
	find . -type f -wholename '/bin/' -exec bash -c 'upx-ucl -9 "$1"' shell {} \; \|\| true

	# Tar toolchain to prevent permission loss
	# Docs: https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss
	mv "coreboot/util/crossgcc/xgcc" "coreboot/util/crossgcc/${{ matrix.arch }}-xgcc"
	tar -cf "${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar" "coreboot/util/crossgcc/${{ matrix.arch }}-xgcc"
	sha256sum "${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar" > "${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar.sha256"

	# Store toolchains and utils in cache
	- name: Cache toolchains
	uses: actions/cache/save@v4
	if: steps.cache-toolchains.outputs.cache-hit != 'true'
	with:
	path: coreboot/util/crossgcc/xgcc
	key: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc

	- name: Commit the toolchain into firmware-action-toolchains repository
	if: steps.toolchains-exist.conclusion == 'failure'
	run: \|
	# Clone repo without downloading LFS items
	GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/9elements/firmware-action-toolchains.git
	cd firmware-action-toolchains
	git branch "feat/${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}"
	mv "../${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar" "coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/"
	mv "../${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar.sha256" "coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/"
	git add .
	git commit -m "feat: add toolchain for coreboot ${{ steps.version.outputs.COREBOOT_VERSION }}"
	git push --set-upstream "feat/${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}"



	#=========================
	# Build Docker containers
	#=========================

	build:
	name: build_test_publish
	runs-on: ubuntu-latest
	timeout-minutes: 60
	needs:
	- get-matrix
	- build-coreboot-toolchains
	strategy:
	fail-fast: false
	matrix:
	dockerfile: ${{ fromJson(needs.get-matrix.outputs.matrix) }}
	permissions:
	contents: read
	packages: write
	steps:
	# We have to use my own fork of actions/delete-package-versions at the moment
	# to have access to 'dry-run' and 'ignore-versions-include-tags' features
	# We can switch to upstream whe following PRs get merged:
	# - [dry-run](https://github.com/actions/delete-package-versions/pull/119/commits)
	# - [tags](https://github.com/actions/delete-package-versions/pull/104
	- name: Delete old packages
	uses: AtomicFS/delete-package-versions@main
	continue-on-error:
	true
	# we have continue-on-error because when I make a fork of this repo to debug something,
	# the Docker containers would not build because this step fails to fetch existing containers
	# (in fresh fork there are none)
	with:
	package-name: firmware-action/${{ matrix.dockerfile }}
	package-type: container
	min-versions-to-keep: 5
	ignore-versions:
	'^(main\|latest\|v(\d+\.?)+)$'
	# ignore:
	# - main
	# - latest
	# - vX
	# - vX.X
	# - vX.X.X
	dry-run: false
	ignore-versions-include-tags: true

	- name: Setup python
	uses: actions/setup-python@v5
	with:
	python-version: '3.x'

	- name: Checkout
	uses: actions/checkout@v4

	- name: Get coreboot version
	id: version
	if: startsWith(matrix.dockerfile, 'coreboot')
	run: yq -r '.services.["${{ matrix.dockerfile }}"].build.args[] \| select(test("COREBOOT_VERSION=.*"))' docker/compose.yaml >> "${GITHUB_OUTPUT}"
	- name: Clone coreboot
	if: startsWith(matrix.dockerfile, 'coreboot')
	run: \|
	git clone --depth 1 "https://review.coreboot.org/coreboot.git" -b "${{ steps.version.outputs.COREBOOT_VERSION }}"
	- name: Get coreboot commit hash
	id: coreboot-hash
	if: startsWith(matrix.dockerfile, 'coreboot')
	run: \|
	cd coreboot
	COREBOOT_HASH="$( git rev-parse --short HEAD )"
	echo "${COREBOOT_HASH}"
	echo "COREBOOT_HASH=${COREBOOT_HASH}" >> "${GITHUB_OUTPUT}"

	#=================================
	# Download artifacts for coreboot
	#=================================

	- name: Download coreboot toolchains from firmware-action-toolchains repository
	id: firmware-action-toolchains
	if: startsWith(matrix.dockerfile, 'coreboot')
	continue-on-error: true
	run: \|
	wget --continue --tries=3 "https://raw.githubusercontent.com/9elements/firmware-action-toolchains/refs/heads/main/coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar"
	wget --continue --tries=3 "https://raw.githubusercontent.com/9elements/firmware-action-toolchains/refs/heads/main/coreboot/${{ steps.version.outputs.COREBOOT_VERSION }}/${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar.sha256"
	sha256sum -c ${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}-xgcc.tar.sha256

	- name: Prepare toolchains
	if: startsWith(matrix.dockerfile, 'coreboot')
	run: \|
	mkdir -p docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}
	for f in ${{ steps.cache-key.outputs.CACHE_KEY }}--xgcc/.tar; do
	ARCH=$( basename "${f}" \| sed -E 's/coreboot\-[0-9\.]+\-[a-z0-9]+\-([a-z0-9]+)\-.*/\1/g' )
	echo "extracting ${f} -> ${{ steps.version.outputs.COREBOOT_VERSION }} / ${ARCH}"
	mkdir -p "${f}.dir/"
	tar -xf "${f}" -C "${f}.dir/"
	mv "${f}.dir/coreboot/util/crossgcc/${ARCH}-xgcc" "docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-${ARCH}"
	done
	- name: Prepare utils
	if: startsWith(matrix.dockerfile, 'coreboot')
	run: \|
	for f in ${{ steps.cache-key.outputs.CACHE_KEY }}-*-utils; do
	ARCH=$( basename "${f}" \| sed -E 's/coreboot\-[0-9\.]+\-[a-z0-9]+\-([a-z0-9]+)\-.*/\1/g' )
	echo "${f} -> ${{ steps.version.outputs.COREBOOT_VERSION }} / ${ARCH}"
	chmod +rx "${f}"/*
	mv "${f}" "docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/utils-${ARCH}"
	done

	- name: Debug list artifacts
	if: startsWith(matrix.dockerfile, 'coreboot')
	run: \|
	ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/
	- name: Debug list xgcc (amd64)
	if: startsWith(matrix.dockerfile, 'coreboot')
	run: \|
	ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-*
	- name: Debug list xgcc/bin (amd64)
	if: startsWith(matrix.dockerfile, 'coreboot')
	run: \|
	ls -a1lh docker/coreboot/coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}/xgcc-*/bin

	#============================
	# Build the docker container
	#============================

	- name: Setup docker-compose
	uses: KengoTODA/actions-setup-docker-compose@v1
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	- name: Validate compose file
	run: docker-compose -f docker/compose.yaml config

	- name: Install python dependencies
	run: pip install -r ./.dagger-ci/daggerci/requirements.txt

	- name: Run dagger pipeline
	timeout-minutes: 60
	run: \|
	if [[ "${GITHUB_EVENT_NAME}" == 'release' ]] \|\| [[ "${GITHUB_REF}" == *'main' ]] \|\| [[ "${GITHUB_REF_TYPE}" == 'tag' ]]; then
	echo "Enable publishing"
	python .dagger-ci/daggerci/main.py -d ${{ matrix.dockerfile }} --publish
	else
	echo "Disable publishing"
	python .dagger-ci/daggerci/main.py -d ${{ matrix.dockerfile }}
	fi
	shell: bash
	env:
	GITHUB_REGISTRY: ${{ env.REGISTRY }}
	GITHUB_ACTOR: ${{ github.actor }}
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Ci/store coreboot toolchains in new repository #1189

Workflow file

Ci/store coreboot toolchains in new repository #1189

Jobs

Run details

Workflow file for this run