diff --git a/.dockerignore b/.dockerignore index 7b0c22ca2..eedae8c71 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,7 @@ +# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. +# +# Generated on 2023-11-02T09:50:52Z by kres 3a2980e-dirty. + * +!README.md !pkg.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 41713a60e..dfaf4d11e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,8 +1,12 @@ +# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. +# +# Generated on 2023-11-02T09:53:23Z by kres 3a2980e-dirty. + name: default concurrency: group: ${{ github.head_ref || github.run_id }} cancel-in-progress: true -on: +"on": push: branches: - main @@ -15,20 +19,23 @@ on: - release-* jobs: default: - if: (!startsWith(github.head_ref, 'renovate/') && !startsWith(github.head_ref, 'dependabot/')) permissions: + actions: read + contents: write packages: write + pull-requests: read runs-on: - self-hosted - pkgs + if: (!startsWith(github.head_ref, 'renovate/') && !startsWith(github.head_ref, 'dependabot/')) outputs: labels: ${{ steps.workflow-run-info.outputs.pullRequestLabels }} services: buildkitd: - image: moby/buildkit:buildx-stable-1 + image: moby/buildkit:v0.12.2 + options: --privileged ports: - 1234:1234 - options: --privileged volumes: - /var/lib/buildkit/${{ github.repository }}:/var/lib/buildkit - /usr/etc/buildkit/buildkitd.toml:/etc/buildkit/buildkitd.toml @@ -41,17 +48,14 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: - driver: remote - endpoint: tcp://localhost:1234 append: | - endpoint: tcp://buildkit-arm64.ci.svc.cluster.local:1234 platforms: linux/arm64 - - name: build + driver: remote + endpoint: tcp://localhost:1234 + - name: Build run: | make - - name: build nonfree - run: | - make nonfree - name: Login to registry if: github.event_name != 'pull_request' uses: docker/login-action@v3 @@ -62,13 +66,22 @@ jobs: - name: Push to registry if: github.event_name != 'pull_request' run: | - make PUSH=true - make nonfree PUSH=true + make PUSH=true - name: Retrieve workflow info id: workflow-run-info uses: potiuk/get-workflow-origin@v1_5 with: token: ${{ secrets.GITHUB_TOKEN }} + - name: release-notes + if: startsWith(github.ref, 'refs/tags/') + run: | + make release-notes + - name: Release + if: startsWith(github.ref, 'refs/tags/') + uses: crazy-max/ghaction-github-release@v2 + with: + body_path: _out/RELEASE_NOTES.md + draft: "true" reproducibility: runs-on: - self-hosted @@ -78,10 +91,10 @@ jobs: - default services: buildkitd: - image: moby/buildkit:buildx-stable-1 + image: moby/buildkit:v0.12.2 + options: --privileged ports: - 1234:1234 - options: --privileged volumes: - /var/lib/buildkit/${{ github.repository }}:/var/lib/buildkit - /usr/etc/buildkit/buildkitd.toml:/etc/buildkit/buildkitd.toml @@ -94,11 +107,11 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: - driver: remote - endpoint: tcp://localhost:1234 append: | - endpoint: tcp://buildkit-arm64.ci.svc.cluster.local:1234 platforms: linux/arm64 - - name: reproducibility + driver: remote + endpoint: tcp://localhost:1234 + - name: reproducibility-test run: | make reproducibility-test diff --git a/.github/workflows/slack-notify.yaml b/.github/workflows/slack-notify.yaml index 218fc0314..ef1ff0bdf 100644 --- a/.github/workflows/slack-notify.yaml +++ b/.github/workflows/slack-notify.yaml @@ -1,3 +1,7 @@ +# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. +# +# Generated on 2023-11-02T09:53:23Z by kres 3a2980e-dirty. + name: slack-notify "on": workflow_run: @@ -10,6 +14,7 @@ jobs: slack-notify: runs-on: - self-hosted + - generic if: ${{ github.event.workflow_run.conclusion != 'skipped' }} steps: - name: Retrieve Workflow Run Info @@ -34,7 +39,7 @@ jobs: "fields": [ { "type": "mrkdwn", - "text": "${{ github.event.workflow_run.event == 'pull_request' && format('*Pull Request:* {0} (`{1}`)\n<{2}/pull/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, steps.retrieve-workflow-run-info.outputs.pullRequestNumber, github.event.workflow_run.display_title) || format('*Build:* {0}#{1} (`{2}`)', github.repository, github.sha, github.ref_name) }}" + "text": "${{ github.event.workflow_run.event == 'pull_request' && format('*Pull Request:* {0} (`{1}`)\n<{2}/pull/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, steps.retrieve-workflow-run-info.outputs.pullRequestNumber, github.event.workflow_run.display_title) || format('*Build:* {0} (`{1}`)\n<{2}/commit/{3}|{4}>', github.repository, github.ref_name, github.event.repository.html_url, github.sha, github.event.workflow_run.display_title) }}" }, { "type": "mrkdwn", diff --git a/.github/workflows/weekly.yaml b/.github/workflows/weekly.yaml new file mode 100644 index 000000000..b91068490 --- /dev/null +++ b/.github/workflows/weekly.yaml @@ -0,0 +1,42 @@ +# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. +# +# Generated on 2023-11-02T09:53:23Z by kres 3a2980e-dirty. + +name: weekly +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true +"on": + schedule: + - cron: 30 1 * * 1 +jobs: + reproducibility: + runs-on: + - self-hosted + - pkgs + services: + buildkitd: + image: moby/buildkit:v0.12.2 + options: --privileged + ports: + - 1234:1234 + volumes: + - /var/lib/buildkit/${{ github.repository }}:/var/lib/buildkit + - /usr/etc/buildkit/buildkitd.toml:/etc/buildkit/buildkitd.toml + steps: + - name: checkout + uses: actions/checkout@v4 + - name: Unshallow + run: | + git fetch --prune --unshallow + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + append: | + - endpoint: tcp://buildkit-arm64.ci.svc.cluster.local:1234 + platforms: linux/arm64 + driver: remote + endpoint: tcp://localhost:1234 + - name: reproducibility-test + run: | + make reproducibility-test diff --git a/.gitignore b/.gitignore index 1bf7ec94a..6f737917f 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ -_out* +# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. +# +# Generated on 2023-11-02T09:50:52Z by kres 3a2980e-dirty. + +_out diff --git a/.kres.yaml b/.kres.yaml new file mode 100644 index 000000000..a9886f197 --- /dev/null +++ b/.kres.yaml @@ -0,0 +1,49 @@ +--- +kind: pkgfile.Build +spec: + targets: + # - non-related to the kernel, in alphabetical order + - base + - ca-certificates + - cni + - containerd + - cryptsetup + - dosfstools + - eudev + - fhs + - flannel-cni + - grub + - ipmitool + - iptables + - ipxe + - kmod + - libaio + - libinih + - libjson-c + - liblzma + - libpopt + - libseccomp + - liburcu + - linux-firmware + - lvm2 + - musl + - openssl + - raspberrypi-firmware + - runc + - sd-boot + - socat + - syslinux + - u-boot + - util-linux + - xfsprogs + # - kernel & dependent packages (out of tree kernel modules) + # kernel first, then packages in alphabetical order + - kernel + - drbd-pkg + - gasket-driver-pkg + - nvidia-open-gpu-kernel-modules-pkg + - zfs-pkg + additionalTargets: + nonfree: + - nonfree-kmod-nvidia-pkg + reproducibleTargetName: toolchain diff --git a/Makefile b/Makefile index 277171719..bb3dc8c1b 100644 --- a/Makefile +++ b/Makefile @@ -1,138 +1,171 @@ +# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. +# +# Generated on 2023-11-02T12:51:30Z by kres latest. + +# common variables + +SHA := $(shell git describe --match=none --always --abbrev=8 --dirty) +TAG := $(shell git describe --tag --always --dirty) +ABBREV_TAG := $(shell git describe --tags >/dev/null 2>/dev/null && git describe --tag --always --match v[0-9]\* --abbrev=0 || echo 'undefined') +BRANCH := $(shell git rev-parse --abbrev-ref HEAD) +ARTIFACTS := _out REGISTRY ?= ghcr.io USERNAME ?= siderolabs -SHA ?= $(shell git describe --match=none --always --abbrev=8 --dirty) -TAG ?= $(shell git describe --tag --always --dirty) -BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) -REGISTRY_AND_USERNAME := $(REGISTRY)/$(USERNAME) -# inital commit time -# git rev-list --max-parents=0 HEAD -# git log ad5ad0a513b775e597c818b25476fc59ba3e4a8c --pretty=%ct -SOURCE_DATE_EPOCH ?= "1559424892" - -# Sync bldr image with Pkgfile +REGISTRY_AND_USERNAME ?= $(REGISTRY)/$(USERNAME) +KRES_IMAGE ?= ghcr.io/siderolabs/kres:latest +CONFORMANCE_IMAGE ?= ghcr.io/siderolabs/conform:latest + +# source date epoch of first commit + +INITIAL_COMMIT_SHA := $(shell git rev-list --max-parents=0 HEAD) +SOURCE_DATE_EPOCH := $(shell git log $(INITIAL_COMMIT_SHA) --pretty=%ct) + +# sync bldr image with pkgfile + BLDR_IMAGE := ghcr.io/siderolabs/bldr:v0.2.3 -BLDR ?= docker run --rm --volume $(PWD):/tools --entrypoint=/bldr \ - $(BLDR_IMAGE) graph --root=/tools +BLDR := docker run --rm --user $(shell id -u):$(shell id -g) --volume $(PWD):/src --entrypoint=/bldr $(BLDR_IMAGE) --root=/src + +# docker build settings BUILD := docker buildx build PLATFORM ?= linux/amd64,linux/arm64 PROGRESS ?= auto PUSH ?= false -COMMON_ARGS := --file=Pkgfile +CI_ARGS ?= +COMMON_ARGS = --file=Pkgfile COMMON_ARGS += --provenance=false COMMON_ARGS += --progress=$(PROGRESS) COMMON_ARGS += --platform=$(PLATFORM) -COMMON_ARGS += --build-arg=http_proxy=$(http_proxy) -COMMON_ARGS += --build-arg=https_proxy=$(https_proxy) COMMON_ARGS += --build-arg=SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) -, := , -empty := -space = $(empty) $(empty) - -# TARGETS are split into two groups: -# - non-related to the kernel, in alphabetical order -TARGETS = \ - base \ - ca-certificates \ - cni \ - containerd \ - cryptsetup \ - dosfstools \ - eudev \ - fhs \ - flannel-cni \ - grub \ - ipmitool \ - iptables \ - ipxe \ - kmod \ - libaio \ - libinih \ - libjson-c \ - liblzma \ - libpopt \ - libseccomp \ - liburcu \ - linux-firmware \ - lvm2 \ - musl \ - openssl \ - raspberrypi-firmware \ - runc \ - sd-boot \ - socat \ - syslinux \ - u-boot \ - util-linux \ - xfsprogs - -# - kernel & dependent packages (out of tree kernel modules) -# kernel first, then packages in alphabetical order -TARGETS += \ - kernel \ - drbd-pkg \ - gasket-driver-pkg \ - nvidia-open-gpu-kernel-modules-pkg \ - zfs-pkg \ - -# Temporarily disabled until mellanox builds with Linux 6.1 -# mellanox-ofed-pkg \ - +# targets defines all the available targets + +TARGETS = base +TARGETS += ca-certificates +TARGETS += cni +TARGETS += containerd +TARGETS += cryptsetup +TARGETS += dosfstools +TARGETS += eudev +TARGETS += fhs +TARGETS += flannel-cni +TARGETS += grub +TARGETS += ipmitool +TARGETS += iptables +TARGETS += ipxe +TARGETS += kmod +TARGETS += libaio +TARGETS += libinih +TARGETS += libjson-c +TARGETS += liblzma +TARGETS += libpopt +TARGETS += libseccomp +TARGETS += liburcu +TARGETS += linux-firmware +TARGETS += lvm2 +TARGETS += musl +TARGETS += openssl +TARGETS += raspberrypi-firmware +TARGETS += runc +TARGETS += sd-boot +TARGETS += socat +TARGETS += syslinux +TARGETS += u-boot +TARGETS += util-linux +TARGETS += xfsprogs +TARGETS += kernel +TARGETS += drbd-pkg +TARGETS += gasket-driver-pkg +TARGETS += nvidia-open-gpu-kernel-modules-pkg +TARGETS += zfs-pkg NONFREE_TARGETS = nonfree-kmod-nvidia-pkg -all: $(TARGETS) ## Builds all known pkgs. +# help menu -nonfree: $(NONFREE_TARGETS) ## Builds all known non-free pkgs. +export define HELP_MENU_HEADER +# Getting Started -.PHONY: help -help: ## This help menu. - @grep -E '^[a-zA-Z%_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +To build this project, you must have the following installed: -local-%: ## Builds the specified target defined in the Dockerfile using the local output type. The build result will be output to the specified local destination. - @$(MAKE) target-$* TARGET_ARGS="--output=type=local,dest=$(DEST) $(TARGET_ARGS)" +- git +- make +- docker (19.03 or higher) + +## Creating a Builder Instance + +The build process makes use of experimental Docker features (buildx). +To enable experimental features, add 'experimental: "true"' to '/etc/docker/daemon.json' on +Linux or enable experimental features in Docker GUI for Windows or Mac. + +To create a builder instance, run: -reproducibility-test: - @$(MAKE) reproducibility-test-local-reproducibility + docker buildx create --name local --use -reproducibility-test-local-%: ## Builds the specified target defined in the Pkgfile using the local output type. The build result will be output to the specified local destination. - @rm -rf _out1/ _out2/ - @$(MAKE) local-$* DEST=_out1 - @$(MAKE) local-$* DEST=_out2 TARGET_ARGS="--no-cache" - @touch -ch -t $$(date -d @$(SOURCE_DATE_EPOCH) +%Y%m%d0000) _out1 _out2 - @diffoscope _out1 _out2 - @rm -rf _out1/ _out2/ -target-%: ## Builds the specified target defined in the Dockerfile. The build result will only remain in the build cache. - @$(BUILD) \ - --target=$* \ - $(COMMON_ARGS) \ - $(TARGET_ARGS) . +If you already have a compatible builder instance, you may use that instead. -docker-%: ## Builds the specified target defined in the Dockerfile using the docker output type. The build result will be loaded into docker. +## Artifacts + +All artifacts will be output to ./$(ARTIFACTS). Images will be tagged with the +registry "$(REGISTRY)", username "$(USERNAME)", and a dynamic tag (e.g. $(IMAGE):$(TAG)). +The registry and username can be overridden by exporting REGISTRY, and USERNAME +respectively. + +endef + +all: $(TARGETS) ## Builds all targets defined. + +.PHONY: clean +clean: ## Cleans up all artifacts. + @rm -rf $(ARTIFACTS) + +target-%: ## Builds the specified target defined in the Pkgfile. The build result will only remain in the build cache. + @$(BUILD) --target=$* $(COMMON_ARGS) $(TARGET_ARGS) $(CI_ARGS) . + +local-%: ## Builds the specified target defined in the Pkgfile using the local output type. The build result will be output to the specified local destination. + @$(MAKE) target-$* TARGET_ARGS="--output=type=local,dest=$(DEST) $(TARGET_ARGS)" + +docker-%: ## Builds the specified target defined in the Pkgfile using the docker output type. The build result will be loaded into Docker. @$(MAKE) target-$* TARGET_ARGS="$(TARGET_ARGS)" +reproducibility-test: ## Builds the reproducibility test target + @$(MAKE) reproducibility-test-local-toolchain + +reproducibility-test-local-%: ## Builds the specified target defined in the Pkgfile using the local output type with and without cahce. The build result will be output to the specified local destination + @rm -rf $(ARTIFACTS)/build-a $(ARTIFACTS)/build-b + @$(MAKE) local-$* DEST=$(ARTIFACTS)/build-a + @$(MAKE) local-$* DEST=$(ARTIFACTS)/build-b TARGET_ARGS="--no-cache" + @touch -ch -t $$(date -d @$(SOURCE_DATE_EPOCH) +%Y%m%d0000) $(ARTIFACTS)/build-a $(ARTIFACTS)/build-b + @diffoscope $(ARTIFACTS)/build-a $(ARTIFACTS)/build-b + @rm -rf $(ARTIFACTS)/build-a $(ARTIFACTS)/build-b + +nonfree: $(NONFREE_TARGETS) ## Builds all nonfree targets defined. + .PHONY: $(TARGETS) $(NONFREE_TARGETS) $(TARGETS) $(NONFREE_TARGETS): - @$(MAKE) docker-$@ TARGET_ARGS="--tag=$(REGISTRY)/$(USERNAME)/$@:$(TAG) --push=$(PUSH)" + @$(MAKE) docker-$@ TARGET_ARGS="--tag=$(REGISTRY_AND_USERNAME)/$@:$(TAG) --push=$(PUSH)" .PHONY: deps.png -deps.png: - @$(BLDR) graph | dot -Tpng > deps.png +deps.png: ## Generates a dependency graph of the Pkgfile. + @$(BLDR) graph | dot -Tpng -o deps.png -kernel-olddefconfig: - @$(MAKE) local-kernel-build TARGET_ARGS="--build-arg=KERNEL_TARGET=olddefconfig" PLATFORM=linux/amd64 DEST="kernel/build" - @$(MAKE) local-kernel-build TARGET_ARGS="--build-arg=KERNEL_TARGET=olddefconfig" PLATFORM=linux/arm64 DEST="kernel/build" +.PHONY: rekres +rekres: + @docker pull $(KRES_IMAGE) + @docker run --rm --net=host --user $(shell id -u):$(shell id -g) -v $(PWD):/src -w /src -e GITHUB_TOKEN $(KRES_IMAGE) -kernel-%: ## Updates the kernel configs: e.g. make kernel-olddefconfig; make kernel-menuconfig; etc. - for platform in $(subst $(,),$(space),$(PLATFORM)); do \ - arch=`basename $$platform` ; \ - $(MAKE) docker-kernel-prepare PLATFORM=$$platform TARGET_ARGS="--tag=$(REGISTRY)/$(USERNAME)/kernel:$(TAG)-$$arch --load"; \ - docker run --rm -it --entrypoint=/toolchain/bin/bash -e PATH=/toolchain/bin:/bin -w /src -v $$PWD/kernel/build/config-$$arch:/host/.hostconfig $(REGISTRY)/$(USERNAME)/kernel:$(TAG)-$$arch -c 'cp /host/.hostconfig .config && make $* && cp .config /host/.hostconfig'; \ - done +.PHONY: help +help: ## This help menu. + @echo "$$HELP_MENU_HEADER" + @grep -E '^[a-zA-Z%_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -# Utilities +.PHONY: release-notes +release-notes: + mkdir -p $(ARTIFACTS) + @ARTIFACTS=$(ARTIFACTS) ./hack/release.sh $@ $(ARTIFACTS)/RELEASE_NOTES.md $(TAG) .PHONY: conformance -conformance: ## Performs policy checks against the commit and source code. - docker run --rm -it -v $(PWD):/src -w /src ghcr.io/siderolabs/conform:latest enforce +conformance: + @docker pull $(CONFORMANCE_IMAGE) + @docker run --rm -it -v $(PWD):/src -w /src $(CONFORMANCE_IMAGE) enforce + diff --git a/deps.png b/deps.png index fa65563f2..b030848b9 100644 Binary files a/deps.png and b/deps.png differ diff --git a/hack/release.sh b/hack/release.sh new file mode 100755 index 000000000..a84db6277 --- /dev/null +++ b/hack/release.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. +# +# Generated on 2023-11-02T09:50:52Z by kres 3a2980e-dirty. + +set -e + +RELEASE_TOOL_IMAGE="ghcr.io/siderolabs/release-tool:latest" + +function release-tool { + docker pull "${RELEASE_TOOL_IMAGE}" >/dev/null + docker run --rm -w /src -v "${PWD}":/src:ro "${RELEASE_TOOL_IMAGE}" -l -d -n -t "${1}" ./hack/release.toml +} + +function changelog { + if [ "$#" -eq 1 ]; then + (release-tool ${1}; echo; cat CHANGELOG.md) > CHANGELOG.md- && mv CHANGELOG.md- CHANGELOG.md + else + echo 1>&2 "Usage: $0 changelog [tag]" + exit 1 + fi +} + +function release-notes { + release-tool "${2}" > "${1}" +} + +function cherry-pick { + if [ $# -ne 2 ]; then + echo 1>&2 "Usage: $0 cherry-pick " + exit 1 + fi + + git checkout $2 + git fetch + git rebase upstream/$2 + git cherry-pick -x $1 +} + +function commit { + if [ $# -ne 1 ]; then + echo 1>&2 "Usage: $0 commit " + exit 1 + fi + + git commit -s -m "release($1): prepare release" -m "This is the official $1 release." +} + +if declare -f "$1" > /dev/null +then + cmd="$1" + shift + $cmd "$@" +else + cat <