From 5191085576e4f9d043bbc2b0ca091d4bb81efaff Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 27 Jan 2023 16:05:03 -0800
Subject: [PATCH 01/90] initial commit

---
 Dockerfile               |  18 ++++
 Makefile                 | 179 +++++++++++++++++++++++++++++++++++++++
 README.md                |  59 +++++++++++++
 scripts/host-env         |  20 +++++
 scripts/packages-install |  26 ++++++
 scripts/packages-update  |  43 ++++++++++
 6 files changed, 345 insertions(+)
 create mode 100644 Dockerfile
 create mode 100644 Makefile
 create mode 100644 README.md
 create mode 100755 scripts/host-env
 create mode 100755 scripts/packages-install
 create mode 100755 scripts/packages-update

diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..37e384d
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,18 @@
+ARG DEBIAN_HASH
+FROM debian@sha256:${DEBIAN_HASH}
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    LANG=C.UTF-8 \
+    TZ=UTC \
+    PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+ARG CONFIG_DIR
+ADD ${CONFIG_DIR} /config
+
+ARG SCRIPTS_DIR
+ADD ${SCRIPTS_DIR} /usr/local/bin
+
+RUN packages-install
+
+RUN echo "/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1" \
+    > /etc/ld.so.preload
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..3d5921a
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,179 @@
+NAME := $(shell basename $(shell git rev-parse --show-toplevel))
+IMAGE := local/$(NAME):latest
+ARCH := x86_64
+TARGET := $(ARCH)
+USER := $(shell id -u):$(shell id -g)
+CPUS := $(shell docker run -it debian nproc)
+GIT_REF := $(shell git log -1 --format=%H config)
+GIT_AUTHOR := $(shell git log -1 --format=%an config)
+GIT_KEY := $(shell git log -1 --format=%GP config)
+GIT_EPOCH := $(shell git log -1 --format=%at config)
+GIT_DATETIME := \
+	$(shell git log -1 --format=%cd --date=format:'%Y-%m-%d %H:%M:%S' config)
+ifeq ($(strip $(shell git status --porcelain 2>/dev/null)),)
+	GIT_STATE=clean
+else
+	GIT_STATE=dirty
+endif
+VERSION := $(shell TZ=UTC0 git show --quiet --date='format-local:%Y%m%dT%H%M%SZ' --format="%cd")
+RELEASE_DIR := release/$(VERSION)
+CONFIG_DIR := config
+CACHE_DIR := cache
+SRC_DIR := src
+OUT_DIR := out
+docker = docker
+
+include $(CONFIG_DIR)/global.env
+export $(shell sed 's/=.*//' $(CONFIG_DIR)/global.env)
+
+## Use env vars from existing release if present
+ifneq (,$(wildcard $(RELEASE_DIR)/release.env))
+    include $(RELEASE_DIR)/release.env
+    export
+endif
+
+executables = $(docker) git patch
+
+.PHONY: toolchain
+toolchain: $(CACHE_DIR)/toolchain.tar $(CACHE_DIR)/toolchain.env
+
+# Launch a shell inside the toolchain container
+.PHONY: toolchain-shell
+toolchain-shell: toolchain
+	$(call toolchain,$(USER),"bash --norc")
+
+# Pin all packages in toolchain container to latest versions
+.PHONY: toolchain-update
+toolchain-update:
+	docker run \
+		--rm \
+		--env LOCAL_USER=$(USER) \
+		--platform=linux/$(ARCH) \
+		--volume $(PWD)/$(CONFIG_DIR):/config \
+		--volume $(PWD)/$(SRC_DIR)/toolchain/scripts:/usr/local/bin \
+		--env ARCH=$(ARCH) \
+		--interactive \
+		--tty \
+		debian@sha256:$(DEBIAN_HASH) \
+		bash -c /usr/local/bin/packages-update
+
+.PHONY: attest
+attest:
+	rm -rf $(CACHE_DIR) $(OUT_DIR)
+	$(MAKE)
+	diff -q $(OUT_DIR)/manifest.txt release/$(VERSION)/manifest.txt;
+
+$(RELEASE_DIR):
+	mkdir -p $@
+
+$(CACHE_DIR):
+	mkdir -p $@
+
+$(OUT_DIR):
+	mkdir -p $@
+
+.ONESHELL:
+$(CACHE_DIR)/toolchain.env: $(CACHE_DIR)
+	cat <<- EOF > $@
+		HOME=/home/build
+		PS1=$(NAME)-toolchain
+		GNUPGHOME=/cache/.gnupg
+		ARCH=$(ARCH)
+		TARGET=$(ARCH)
+		GIT_REF=$(GIT_REF)
+		GIT_AUTHOR=$(GIT_AUTHOR)
+		GIT_KEY=$(GIT_KEY)
+		GIT_DATETIME=$(GIT_DATETIME)
+		GIT_EPOCH=$(GIT_EPOCH)
+		FAKETIME_FMT="%s"
+		FAKETIME="1"
+		SOURCE_DATE_EPOCH=1
+		KBUILD_BUILD_TIMESTAMP="1970-01-01 00:00:00 UTC"
+		KCONFIG_NOTIMESTAMP=1
+		KBUILD_BUILD_USER=root
+		KBUILD_BUILD_HOST=$(NAME)
+		KBUILD_BUILD_VERSION=1
+		UID=$(shell id -u)
+		GID=$(shell id -g)
+		RELEASE_DIR=release/$(VERSION)
+		CONFIG_DIR=/home/build/$(CONFIG_DIR)
+		CACHE_DIR=/home/build/$(CACHE_DIR)
+		SRC_DIR=/home/build/$(SRC_DIR)
+		OUT_DIR=/home/build/$(OUT_DIR)
+	EOF
+
+$(CACHE_DIR)/toolchain.tar:
+	mkdir -p $(CACHE_DIR)
+	DOCKER_BUILDKIT=1 \
+	docker build \
+		--tag $(IMAGE) \
+		--build-arg DEBIAN_HASH=$(DEBIAN_HASH) \
+		--build-arg CONFIG_DIR=$(CONFIG_DIR) \
+		--build-arg SCRIPTS_DIR=$(SRC_DIR)/toolchain/scripts \
+		--platform=linux/$(ARCH) \
+		-f $(SRC_DIR)/toolchain/Dockerfile \
+		.
+	docker save "$(IMAGE)" -o "$@"
+
+$(RELEASE_DIR)/release.env: \
+	$(RELEASE_DIR) \
+	$(OUT_DIR)/release.env
+	cp $(OUT_DIR)/release.env $(RELEASE_DIR)/release.env
+
+$(RELEASE_DIR)/manifest.txt: \
+	$(RELEASE_DIR) \
+	$(OUT_DIR)/manifest.txt
+	cp $(OUT_DIR)/manifest.txt $(RELEASE_DIR)/manifest.txt
+
+$(OUT_DIR)/release.env: | $(OUT_DIR)
+	echo 'VERSION=$(VERSION)'            > $(OUT_DIR)/release.env
+	echo 'GIT_REF=$(GIT_REF)'           >> $(OUT_DIR)/release.env
+	echo 'GIT_AUTHOR=$(GIT_AUTHOR)'     >> $(OUT_DIR)/release.env
+	echo 'GIT_KEY=$(GIT_KEY)'           >> $(OUT_DIR)/release.env
+	echo 'GIT_DATETIME=$(GIT_DATETIME)' >> $(OUT_DIR)/release.env
+
+$(OUT_DIR)/manifest.txt: | $(OUT_DIR)
+	find $(OUT_DIR) \
+		-type f \
+		-not -path "$(OUT_DIR)/manifest.txt" \
+		-exec openssl sha256 -r {} \; \
+	| sed -e 's/ \*/ /g' -e 's/ \.\// /g' \
+	| LC_ALL=C sort -k2 \
+	> $@
+
+check_executables := $(foreach exec,$(executables),\$(if \
+	$(shell which $(exec)),some string,$(error "No $(exec) in PATH")))
+
+define git_clone
+	[ -d $(CACHE_DIR)/$(1) ] || git clone $(2) $(CACHE_DIR)/$(1)
+	git -C $(CACHE_DIR)/$(1) checkout $(3)
+	git -C $(CACHE_DIR)/$(1) rev-parse --verify HEAD | grep -q $(3) || { \
+		echo 'Error: Git ref/branch collision.'; exit 1; \
+	};
+endef
+
+define apply_patches
+	[ -d $(2) ] && $(call toolchain,$(USER)," \
+		cd $(1); \
+		git restore .; \
+		find /$(2) -type f -iname '*.patch' -print0 \
+		| xargs -t -0 -n 1 patch -p1 --no-backup-if-mismatch -i ; \
+	")
+endef
+
+define toolchain
+	docker load -i $(CACHE_DIR)/toolchain.tar
+	docker run \
+		--rm \
+		--tty \
+		--interactive \
+		--user=$(1) \
+		--platform=linux/$(ARCH) \
+		--cpus $(CPUS) \
+		--volume $(PWD):/home/build \
+		--workdir /home/build \
+		--env-file=$(CONFIG_DIR)/global.env \
+		--env-file=$(CACHE_DIR)/toolchain.env \
+		$(IMAGE) \
+		bash -c $(2)
+endef
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..2b501be
--- /dev/null
+++ b/README.md
@@ -0,0 +1,59 @@
+# Toolchain #
+
+<https://codeberg.org/distrust/toolchain>
+
+## About ##
+
+A library of opinionated make functions targeting projects that either need
+deterministic builds, or a shared deterministic toolchain shared across all
+who use a project.
+
+A dev of a Toolchain enabled project should never need to have anything
+on their host system installed but docker, and git. Everything else will be
+provided via a Docker container.
+
+Debian currently has the highest reproducibility score of any major Linux
+distribution, and as such it is the chosen base for Toolchain.
+
+This was built for Distrust projects, and some of our clients. It is unlikely
+to meet the needs of everyone. We suggest including this in your project as
+a git subtree, so you can make your own changes, but also pull in changes from
+us as desired.
+
+## Uses ##
+ * Ensure everyone on a team is using the exact same tools
+ * Ensure all releases and artifacts build hash-for-hash identical every time
+ * Control supply chain security with only signed/reproducible dependencies
+
+## Features ##
+ * Can run a shell with all toolchain tooling in the current directory
+ * Provide make functions for common tasks
+   * Git clone, apply patches, etc.
+ * Use a global env file as configuration
+ * Hash-locking of apt dependencies from a list of top-level required packages
+ * Provides release.env file with required vars to re-create old releases
+
+## Requirements ##
+
+* docker 18+
+* GNU Make 4+
+
+## Build ##
+
+### Build a new release
+
+    ```
+    make VERSION=1.0.0rc1 release
+    ```
+
+### Reproduce an existing release
+
+    ```
+    make VERSION=1.0.0rc1 attest
+    ```
+
+### Sign an existing release
+
+    ```
+    make VERSION=1.0.0rc1 sign
+    ```
diff --git a/scripts/host-env b/scripts/host-env
new file mode 100755
index 0000000..ec7ca98
--- /dev/null
+++ b/scripts/host-env
@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+uid=${UID?}
+gid=${GID?}
+user=${USER:-"build"}
+export HOME="/home/${user}"
+
+groupadd -g "$gid" "${user}"
+useradd \
+	-g "$gid" \
+	-u "$uid" \
+	-md "/home/${user}" \
+	-s /bin/bash \
+	"${user}"
+
+mkdir -p "$HOME"
+chown -R "$uid:$gid" "$HOME"
+cd "$HOME"
+setpriv --reuid="$uid" --regid="$gid" --init-groups "$@"
diff --git a/scripts/packages-install b/scripts/packages-install
new file mode 100755
index 0000000..13b6acf
--- /dev/null
+++ b/scripts/packages-install
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -e;
+
+ARCH=$(uname -m)
+
+cp /config/toolchain/* /etc/apt/
+
+apt-get update
+apt-get install debian-archive-keyring
+
+until apt-get install --download-only --reinstall --allow-downgrades -y $(cat /etc/apt/packages-${ARCH}.list); do
+    echo "apt install failed. Likely throttled. Retrying in 10 mins...";
+    sleep 600;
+done;
+
+(
+	cd /var/cache/apt/archives \
+		&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
+		| sed 's/.\///g' \
+		| LC_ALL=C sort
+) > /etc/apt/package-hashes-${ARCH}-compare.txt
+
+diff /etc/apt/package-hashes-${ARCH}{,-compare}.txt
+
+apt-get install --allow-downgrades -y $(cat /etc/apt/packages-${ARCH}.list)
+rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /var/tmp/*;
diff --git a/scripts/packages-update b/scripts/packages-update
new file mode 100755
index 0000000..d385f2b
--- /dev/null
+++ b/scripts/packages-update
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+[ -f /.dockerenv ] || { echo "please run in supplied container"; exit 1; }
+set -e
+
+snapshot_url="http://snapshot.debian.org/archive/debian"
+snapshot_date=$(date +"%Y%m%dT000000Z")
+cat <<-EOF > /etc/apt/sources.list
+deb http://deb.debian.org/debian bookworm main
+deb http://security.debian.org/debian-security bookworm-security main
+deb http://deb.debian.org/debian bookworm-updates main
+deb [check-valid-until=no] ${snapshot_url}/${snapshot_date} bookworm main
+deb [check-valid-until=no] ${snapshot_url}-security/${snapshot_date} bookworm-security main
+deb [check-valid-until=no] ${snapshot_url}/${snapshot_date} bookworm-updates main
+EOF
+cp /etc/apt/sources.list /config/toolchain/
+
+ARCH=$(uname -m)
+
+apt-get update
+apt-get install -y --download-only --reinstall $( \
+	dpkg-query \
+		-W \
+		-f='${db:Status-Abbrev}\t${binary:Package} - ${binary:Summary}\n' \
+	| awk -F'\t' '/^ii/ {print $2}' \
+	| awk '{print $1}' \
+)
+apt-get install -y --download-only $(cat /config/toolchain/packages-base.list)
+
+( cd /var/cache/apt/archives \
+	&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
+	| sed 's/.\///g' \
+	| LC_ALL=C sort
+) > /config/toolchain/package-hashes-${ARCH}.txt
+
+cp /dev/null /config/toolchain/packages-${ARCH}.list
+for deb in /var/cache/apt/archives/*.deb; do
+	package=$(dpkg-deb -f $deb Package);
+	version=$(dpkg --info ${deb} | grep "^ Version: " | sed 's/^ Version: //g');
+	echo "${package}=${version}" >> /config/toolchain/packages-${ARCH}.list;
+done
+
+chown -R $LOCAL_USER /config/toolchain

From 90d16ca3bab4ee3b7d56e29fa0d0d38b8e19843d Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 27 Jan 2023 16:52:14 -0800
Subject: [PATCH 02/90] more useful setup instructions

---
 README.md | 75 ++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 66 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 2b501be..72d9120 100644
--- a/README.md
+++ b/README.md
@@ -5,8 +5,8 @@
 ## About ##
 
 A library of opinionated make functions targeting projects that either need
-deterministic builds, or a shared deterministic toolchain shared across all
-who use a project.
+deterministic builds, or a deterministic toolchain shared across all who use a
+project.
 
 A dev of a Toolchain enabled project should never need to have anything
 on their host system installed but docker, and git. Everything else will be
@@ -38,22 +38,79 @@ us as desired.
 * docker 18+
 * GNU Make 4+
 
-## Build ##
+## Setup ##
 
-### Build a new release
+1. Clone toolchain as a git submodule somewhere in your project
 
     ```
-    make VERSION=1.0.0rc1 release
+    git submodule add https://codeburg.org/distrust/toolchain src/toolchain
     ```
 
-### Reproduce an existing release
+2. Include toolchain Makefile in your root Makefile
 
     ```
-    make VERSION=1.0.0rc1 attest
+    include src/toolchain/Makefile
     ```
 
-### Sign an existing release
+3. Define any build/dev dependencies for toolchain container
+
+    ```
+    echo "libfaketime" >> config/toolchain/packages-base.txt
+    echo "build-essential" >> config/toolchain/packages-base.txt
+    ```
+
+4. Lock a base Debian container image hash
+
+    ```
+    echo "DEBIAN_HASH=48b28b354484a7f0e683e340fa0e6e4c4bce3dc3aa0146fc2f78f443fde2c55d" >> config/global.env
+    ```
+
+5. Generate hashlocks files for all toolchain container dependencies
+    ```
+    make toolchain-update
+    ```
+
+6. Define your artifact targets
+
+    ```
+    $(OUT_DIR)/hello: toolchain \
+    	$(call toolchain,$(USER)," \
+        cd $(SRC_DIR)/; \
+        gcc hello.c -o $(OUT_DIR)/hello
+    	")
+    ```
+
+7. Define a release target for your project depending on manifest.txt
 
     ```
-    make VERSION=1.0.0rc1 sign
+    .PHONY: release
+    release: $(OUT_DIR)/hello $(OUT_DIR)/manifest.txt
+    	mkdir -p $(RELEASE_DIR)
+    	cp $(OUT_DIR)/my-binary $(RELEASE_DIR)/hello
+    	cp $(OUT_DIR)/release.env $(RELEASE_DIR)/release.env
+    	cp $(OUT_DIR)/manifest.txt $(RELEASE_DIR)/manifest.txt
     ```
+
+    Note that manifest.txt is optional, but it makes for an ideal single file
+    to sign if a release will contain more than one artifact.
+
+
+## Usage ##
+
+### Build a new release
+
+```
+make VERSION=1.0.0rc1 release
+```
+
+### Reproduce an existing release
+
+```
+make VERSION=1.0.0rc1 attest
+```
+
+### Sign an existing release
+
+```
+make VERSION=1.0.0rc1 sign
+```

From 75a8cec72ec4f2cf32e323984c05e4c70300a325 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 27 Jan 2023 16:54:51 -0800
Subject: [PATCH 03/90] fix example binary name

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 72d9120..2963640 100644
--- a/README.md
+++ b/README.md
@@ -86,7 +86,7 @@ us as desired.
     .PHONY: release
     release: $(OUT_DIR)/hello $(OUT_DIR)/manifest.txt
     	mkdir -p $(RELEASE_DIR)
-    	cp $(OUT_DIR)/my-binary $(RELEASE_DIR)/hello
+    	cp $(OUT_DIR)/hello $(RELEASE_DIR)/hello
     	cp $(OUT_DIR)/release.env $(RELEASE_DIR)/release.env
     	cp $(OUT_DIR)/manifest.txt $(RELEASE_DIR)/manifest.txt
     ```

From 76fab9946a9acb3e2ae59ce54b89bb195247227c Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 27 Jan 2023 16:56:08 -0800
Subject: [PATCH 04/90] whitespace

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 2963640..76183fb 100644
--- a/README.md
+++ b/README.md
@@ -74,10 +74,10 @@ us as desired.
 
     ```
     $(OUT_DIR)/hello: toolchain \
-    	$(call toolchain,$(USER)," \
+      $(call toolchain,$(USER)," \
         cd $(SRC_DIR)/; \
         gcc hello.c -o $(OUT_DIR)/hello
-    	")
+      ")
     ```
 
 7. Define a release target for your project depending on manifest.txt

From bf4dd6e0fa7c09f56ce7e525b78062e044648d31 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 27 Jan 2023 17:00:24 -0800
Subject: [PATCH 05/90] detail update and shell

---
 README.md | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 76183fb..cf6d7c9 100644
--- a/README.md
+++ b/README.md
@@ -109,8 +109,15 @@ make VERSION=1.0.0rc1 release
 make VERSION=1.0.0rc1 attest
 ```
 
-### Sign an existing release
+### Add and lock a new container dependency
 
 ```
-make VERSION=1.0.0rc1 sign
+echo "vim-nox" >> config/toolchain/packages-base.txt
+make toolchain-update
+```
+
+### Run a shell in the toolchain container
+
+```
+make toolchain-shell
 ```

From 9c45fa460ec2c0accf3f42ab35511feca417c91d Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 2 Feb 2023 21:30:11 -0800
Subject: [PATCH 06/90] several multi-arch improvements

---
 Dockerfile          |   5 ---
 Makefile            | 106 +++++++++++++++++++++++++++-----------------
 scripts/environment |  36 +++++++++++++++
 3 files changed, 102 insertions(+), 45 deletions(-)
 create mode 100755 scripts/environment

diff --git a/Dockerfile b/Dockerfile
index 37e384d..1efd80a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,11 +1,6 @@
 ARG DEBIAN_HASH
 FROM debian@sha256:${DEBIAN_HASH}
 
-ENV DEBIAN_FRONTEND=noninteractive \
-    LANG=C.UTF-8 \
-    TZ=UTC \
-    PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-
 ARG CONFIG_DIR
 ADD ${CONFIG_DIR} /config
 
diff --git a/Makefile b/Makefile
index 3d5921a..d3929f0 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,8 @@
+DEFAULT_GOAL := $(or $(DEFAULT_GOAL),toolchain)
+ARCH := $(or $(ARCH),x86_64)
+TARGET := $(or $(TARGET),$(ARCH))
 NAME := $(shell basename $(shell git rev-parse --show-toplevel))
 IMAGE := local/$(NAME):latest
-ARCH := x86_64
-TARGET := $(ARCH)
 USER := $(shell id -u):$(shell id -g)
 CPUS := $(shell docker run -it debian nproc)
 GIT_REF := $(shell git log -1 --format=%H config)
@@ -18,9 +19,20 @@ endif
 VERSION := $(shell TZ=UTC0 git show --quiet --date='format-local:%Y%m%dT%H%M%SZ' --format="%cd")
 RELEASE_DIR := release/$(VERSION)
 CONFIG_DIR := config
-CACHE_DIR := cache
+CACHE_DIR_ROOT := cache
+FETCH_DIR := $(CACHE_DIR_ROOT)/fetch
+ifeq ($(TARGET),$(ARCH))
+	CACHE_DIR := $(CACHE_DIR_ROOT)/$(TARGET)
+else
+	CACHE_DIR := $(CACHE_DIR_ROOT)/$(TARGET)/$(ARCH)
+endif
+BIN_DIR := $(CACHE_DIR_ROOT)/bin
 SRC_DIR := src
+KEY_DIR := keys
 OUT_DIR := out
+
+export
+
 docker = docker
 
 include $(CONFIG_DIR)/global.env
@@ -35,7 +47,14 @@ endif
 executables = $(docker) git patch
 
 .PHONY: toolchain
-toolchain: $(CACHE_DIR)/toolchain.tar $(CACHE_DIR)/toolchain.env
+toolchain: \
+	$(CACHE_DIR) \
+	$(FETCH_DIR) \
+	$(RELEASE_DIR) \
+	$(BIN_DIR) \
+	$(OUT_DIR) \
+	$(CACHE_DIR_ROOT)/toolchain.tar \
+	$(CACHE_DIR_ROOT)/toolchain.env
 
 # Launch a shell inside the toolchain container
 .PHONY: toolchain-shell
@@ -66,43 +85,29 @@ attest:
 $(RELEASE_DIR):
 	mkdir -p $@
 
+$(BIN_DIR):
+	mkdir -p $@
+
 $(CACHE_DIR):
 	mkdir -p $@
 
+$(FETCH_DIR):
+	mkdir -p $@
+
 $(OUT_DIR):
 	mkdir -p $@
 
-.ONESHELL:
-$(CACHE_DIR)/toolchain.env: $(CACHE_DIR)
-	cat <<- EOF > $@
-		HOME=/home/build
-		PS1=$(NAME)-toolchain
-		GNUPGHOME=/cache/.gnupg
-		ARCH=$(ARCH)
-		TARGET=$(ARCH)
-		GIT_REF=$(GIT_REF)
-		GIT_AUTHOR=$(GIT_AUTHOR)
-		GIT_KEY=$(GIT_KEY)
-		GIT_DATETIME=$(GIT_DATETIME)
-		GIT_EPOCH=$(GIT_EPOCH)
-		FAKETIME_FMT="%s"
-		FAKETIME="1"
-		SOURCE_DATE_EPOCH=1
-		KBUILD_BUILD_TIMESTAMP="1970-01-01 00:00:00 UTC"
-		KCONFIG_NOTIMESTAMP=1
-		KBUILD_BUILD_USER=root
-		KBUILD_BUILD_HOST=$(NAME)
-		KBUILD_BUILD_VERSION=1
-		UID=$(shell id -u)
-		GID=$(shell id -g)
-		RELEASE_DIR=release/$(VERSION)
-		CONFIG_DIR=/home/build/$(CONFIG_DIR)
-		CACHE_DIR=/home/build/$(CACHE_DIR)
-		SRC_DIR=/home/build/$(SRC_DIR)
-		OUT_DIR=/home/build/$(OUT_DIR)
-	EOF
-
-$(CACHE_DIR)/toolchain.tar:
+$(CACHE_DIR_ROOT)/toolchain.env: \
+	$(CACHE_DIR) \
+	$(SRC_DIR)/toolchain/scripts/environment
+	$(SRC_DIR)/toolchain/scripts/environment > $@
+
+$(CACHE_DIR_ROOT)/toolchain.tar: \
+	$(SRC_DIR)/toolchain/Dockerfile \
+	$(CONFIG_DIR)/toolchain/package-hashes-$(ARCH).txt \
+	$(CONFIG_DIR)/toolchain/packages-base.list \
+	$(CONFIG_DIR)/toolchain/packages-$(ARCH).list \
+	$(CONFIG_DIR)/toolchain/sources.list
 	mkdir -p $(CACHE_DIR)
 	DOCKER_BUILDKIT=1 \
 	docker build \
@@ -145,9 +150,9 @@ check_executables := $(foreach exec,$(executables),\$(if \
 	$(shell which $(exec)),some string,$(error "No $(exec) in PATH")))
 
 define git_clone
-	[ -d $(CACHE_DIR)/$(1) ] || git clone $(2) $(CACHE_DIR)/$(1)
-	git -C $(CACHE_DIR)/$(1) checkout $(3)
-	git -C $(CACHE_DIR)/$(1) rev-parse --verify HEAD | grep -q $(3) || { \
+	[ -d $(1) ] || git clone $(2) $(1)
+	git -C $(1) checkout $(3)
+	git -C $(1) rev-parse --verify HEAD | grep -q $(3) || { \
 		echo 'Error: Git ref/branch collision.'; exit 1; \
 	};
 endef
@@ -161,8 +166,29 @@ define apply_patches
 	")
 endef
 
+define fetch_pgp_key
+        mkdir -p $(KEY_DIR) && \
+        $(call toolchain,$(USER), " \
+                for server in \
+            ha.pool.sks-keyservers.net \
+            hkp://keyserver.ubuntu.com:80 \
+            hkp://p80.pool.sks-keyservers.net:80 \
+            pgp.mit.edu \
+        ; do \
+                        echo "Trying: $${server}"; \
+                gpg \
+                        --recv-key \
+                        --keyserver "$${server}" \
+                        --keyserver-options timeout=10 \
+                        --recv-keys "$(1)" \
+                && break; \
+        done; \
+                gpg --export -a $(1) > $(KEY_DIR)/$(1).asc; \
+        ")
+endef
+
 define toolchain
-	docker load -i $(CACHE_DIR)/toolchain.tar
+	docker load -i $(CACHE_DIR_ROOT)/toolchain.tar
 	docker run \
 		--rm \
 		--tty \
@@ -173,7 +199,7 @@ define toolchain
 		--volume $(PWD):/home/build \
 		--workdir /home/build \
 		--env-file=$(CONFIG_DIR)/global.env \
-		--env-file=$(CACHE_DIR)/toolchain.env \
+		--env-file=$(CACHE_DIR_ROOT)/toolchain.env \
 		$(IMAGE) \
 		bash -c $(2)
 endef
diff --git a/scripts/environment b/scripts/environment
new file mode 100755
index 0000000..6fb66bc
--- /dev/null
+++ b/scripts/environment
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+HOME=/home/build
+cat <<- EOF
+	HOME=${HOME}
+	PATH=${HOME}/${BIN_DIR}:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+	TZ=UTC
+	LANG=C.UTF-8
+	DEBIAN_FRONTEND=noninteractive
+	PS1=${NAME}-toolchain
+	GNUPGHOME=${HOME}/${CACHE_DIR}/.gnupg
+	ARCH=${ARCH}
+	TARGET=${ARCH}
+	GIT_REF=${GIT_REF}
+	GIT_AUTHOR=${GIT_AUTHOR}
+	GIT_KEY=${GIT_KEY}
+	GIT_DATETIME=${GIT_DATETIME}
+	GIT_EPOCH=${GIT_EPOCH}
+	FAKETIME_FMT=%s
+	FAKETIME=1
+	SOURCE_DATE_EPOCH=1
+	KBUILD_BUILD_TIMESTAMP=1970-01-01 00:00:00 UTC
+	KCONFIG_NOTIMESTAMP=1
+	KBUILD_BUILD_USER=root
+	KBUILD_BUILD_HOST=${NAME}
+	KBUILD_BUILD_VERSION=1
+	UID=$(id -u)
+	GID=$(id -g)
+	RELEASE_DIR=release/${VERSION}
+	CONFIG_DIR=${HOME}/${CONFIG_DIR}
+	CACHE_DIR=${HOME}/${CACHE_DIR}
+	SRC_DIR=${HOME}/${SRC_DIR}
+	OUT_DIR=${HOME}/${OUT_DIR}
+	BIN_DIR=${HOME}/${BIN_DIR}
+	FETCH_DIR=${HOME}/${FETCH_DIR}
+EOF

From db9ab961fd548b9f7427c519dbfa0e9fee8af033 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 3 Feb 2023 03:43:24 -0800
Subject: [PATCH 07/90] add toolchain-clean and default to FAKETIME off

---
 Makefile            | 5 +++++
 scripts/environment | 1 -
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index d3929f0..d9b89e2 100644
--- a/Makefile
+++ b/Makefile
@@ -76,6 +76,11 @@ toolchain-update:
 		debian@sha256:$(DEBIAN_HASH) \
 		bash -c /usr/local/bin/packages-update
 
+.PHONY: toolchain-clean
+toolchain-clean:
+	rm -rf cache out
+	docker image rm -f local/$(NAME)-build
+
 .PHONY: attest
 attest:
 	rm -rf $(CACHE_DIR) $(OUT_DIR)
diff --git a/scripts/environment b/scripts/environment
index 6fb66bc..7c2aae9 100755
--- a/scripts/environment
+++ b/scripts/environment
@@ -17,7 +17,6 @@ cat <<- EOF
 	GIT_DATETIME=${GIT_DATETIME}
 	GIT_EPOCH=${GIT_EPOCH}
 	FAKETIME_FMT=%s
-	FAKETIME=1
 	SOURCE_DATE_EPOCH=1
 	KBUILD_BUILD_TIMESTAMP=1970-01-01 00:00:00 UTC
 	KCONFIG_NOTIMESTAMP=1

From 68a9c216ab2e15dd79e3b708a6834145106e8e57 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Mon, 6 Feb 2023 12:54:17 -0800
Subject: [PATCH 08/90] add PLATFORM var

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index d9b89e2..5d19b44 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,7 @@
 DEFAULT_GOAL := $(or $(DEFAULT_GOAL),toolchain)
 ARCH := $(or $(ARCH),x86_64)
 TARGET := $(or $(TARGET),$(ARCH))
+PLATFORM := $(or $(PLATFORM),linux)
 NAME := $(shell basename $(shell git rev-parse --show-toplevel))
 IMAGE := local/$(NAME):latest
 USER := $(shell id -u):$(shell id -g)

From ab2e37ace2f068fa7244d8038db678e44e0b8d59 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Mon, 6 Feb 2023 12:54:56 -0800
Subject: [PATCH 09/90] Make all packages 'trusted' from debian archive to
 build when keys expire

---
 scripts/packages-update | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/packages-update b/scripts/packages-update
index d385f2b..c3e6ec4 100755
--- a/scripts/packages-update
+++ b/scripts/packages-update
@@ -9,9 +9,9 @@ cat <<-EOF > /etc/apt/sources.list
 deb http://deb.debian.org/debian bookworm main
 deb http://security.debian.org/debian-security bookworm-security main
 deb http://deb.debian.org/debian bookworm-updates main
-deb [check-valid-until=no] ${snapshot_url}/${snapshot_date} bookworm main
-deb [check-valid-until=no] ${snapshot_url}-security/${snapshot_date} bookworm-security main
-deb [check-valid-until=no] ${snapshot_url}/${snapshot_date} bookworm-updates main
+deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm main
+deb [trusted=yes] ${snapshot_url}-security/${snapshot_date} bookworm-security main
+deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm-updates main
 EOF
 cp /etc/apt/sources.list /config/toolchain/
 

From a4d04788af2b3dfa1b255f1db2dce844cc9499bf Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 7 Feb 2023 16:31:09 -0800
Subject: [PATCH 10/90] attest with needed args

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 5d19b44..98ce4a6 100644
--- a/Makefile
+++ b/Makefile
@@ -85,7 +85,7 @@ toolchain-clean:
 .PHONY: attest
 attest:
 	rm -rf $(CACHE_DIR) $(OUT_DIR)
-	$(MAKE)
+	$(MAKE) TARGET=$(TARGET) VERSION=$(VERSION)
 	diff -q $(OUT_DIR)/manifest.txt release/$(VERSION)/manifest.txt;
 
 $(RELEASE_DIR):

From 0fe125fa8f6281169d731862ca41cc085f6334ff Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Wed, 8 Feb 2023 17:07:23 -0800
Subject: [PATCH 11/90] switch to traditional naming: release -> dist

---
 Makefile | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/Makefile b/Makefile
index 98ce4a6..23c9727 100644
--- a/Makefile
+++ b/Makefile
@@ -18,7 +18,7 @@ else
 	GIT_STATE=dirty
 endif
 VERSION := $(shell TZ=UTC0 git show --quiet --date='format-local:%Y%m%dT%H%M%SZ' --format="%cd")
-RELEASE_DIR := release/$(VERSION)
+DIST_DIR := dist
 CONFIG_DIR := config
 CACHE_DIR_ROOT := cache
 FETCH_DIR := $(CACHE_DIR_ROOT)/fetch
@@ -40,8 +40,8 @@ include $(CONFIG_DIR)/global.env
 export $(shell sed 's/=.*//' $(CONFIG_DIR)/global.env)
 
 ## Use env vars from existing release if present
-ifneq (,$(wildcard $(RELEASE_DIR)/release.env))
-    include $(RELEASE_DIR)/release.env
+ifneq (,$(wildcard $(DIST_DIR)/release.env))
+    include $(DIST_DIR)/release.env
     export
 endif
 
@@ -51,7 +51,7 @@ executables = $(docker) git patch
 toolchain: \
 	$(CACHE_DIR) \
 	$(FETCH_DIR) \
-	$(RELEASE_DIR) \
+	$(DIST_DIR) \
 	$(BIN_DIR) \
 	$(OUT_DIR) \
 	$(CACHE_DIR_ROOT)/toolchain.tar \
@@ -86,9 +86,9 @@ toolchain-clean:
 attest:
 	rm -rf $(CACHE_DIR) $(OUT_DIR)
 	$(MAKE) TARGET=$(TARGET) VERSION=$(VERSION)
-	diff -q $(OUT_DIR)/manifest.txt release/$(VERSION)/manifest.txt;
+	diff -q $(OUT_DIR)/manifest.txt $(DIST_DIR)/manifest.txt;
 
-$(RELEASE_DIR):
+$(DIST_DIR):
 	mkdir -p $@
 
 $(BIN_DIR):
@@ -126,15 +126,15 @@ $(CACHE_DIR_ROOT)/toolchain.tar: \
 		.
 	docker save "$(IMAGE)" -o "$@"
 
-$(RELEASE_DIR)/release.env: \
-	$(RELEASE_DIR) \
+$(DIST_DIR)/release.env: \
+	$(DIST_DIR) \
 	$(OUT_DIR)/release.env
-	cp $(OUT_DIR)/release.env $(RELEASE_DIR)/release.env
+	cp $(OUT_DIR)/release.env $(DIST_DIR)/release.env
 
-$(RELEASE_DIR)/manifest.txt: \
-	$(RELEASE_DIR) \
+$(DIST_DIR)/manifest.txt: \
+	$(DIST_DIR) \
 	$(OUT_DIR)/manifest.txt
-	cp $(OUT_DIR)/manifest.txt $(RELEASE_DIR)/manifest.txt
+	cp $(OUT_DIR)/manifest.txt $(DIST_DIR)/manifest.txt
 
 $(OUT_DIR)/release.env: | $(OUT_DIR)
 	echo 'VERSION=$(VERSION)'            > $(OUT_DIR)/release.env

From 14c700d3be9ca2d673bae019966779fb5fd97acf Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 10 Feb 2023 12:57:10 -0800
Subject: [PATCH 12/90] build release.env against whole repo, not config folder

---
 Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 23c9727..e8e8fa7 100644
--- a/Makefile
+++ b/Makefile
@@ -6,10 +6,10 @@ NAME := $(shell basename $(shell git rev-parse --show-toplevel))
 IMAGE := local/$(NAME):latest
 USER := $(shell id -u):$(shell id -g)
 CPUS := $(shell docker run -it debian nproc)
-GIT_REF := $(shell git log -1 --format=%H config)
-GIT_AUTHOR := $(shell git log -1 --format=%an config)
-GIT_KEY := $(shell git log -1 --format=%GP config)
-GIT_EPOCH := $(shell git log -1 --format=%at config)
+GIT_REF := $(shell git log -1 --format=%H)
+GIT_AUTHOR := $(shell git log -1 --format=%an)
+GIT_KEY := $(shell git log -1 --format=%GP)
+GIT_EPOCH := $(shell git log -1 --format=%at)
 GIT_DATETIME := \
 	$(shell git log -1 --format=%cd --date=format:'%Y-%m-%d %H:%M:%S' config)
 ifeq ($(strip $(shell git status --porcelain 2>/dev/null)),)

From 99c9bd4f79956790b7bc08e86c325e738eee9e70 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 10 Feb 2023 13:07:12 -0800
Subject: [PATCH 13/90] use release.env from DIST_DIR when attesting

---
 Makefile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Makefile b/Makefile
index e8e8fa7..cf8350b 100644
--- a/Makefile
+++ b/Makefile
@@ -85,6 +85,8 @@ toolchain-clean:
 .PHONY: attest
 attest:
 	rm -rf $(CACHE_DIR) $(OUT_DIR)
+	mkdir -p $(OUT_DIR)
+	cp $(DIST_DIR)/release.env $(OUT_DIR)/release.env
 	$(MAKE) TARGET=$(TARGET) VERSION=$(VERSION)
 	diff -q $(OUT_DIR)/manifest.txt $(DIST_DIR)/manifest.txt;
 

From 4a4d61f1b7f126310e618b1fb5fe3ae9aad7f592 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 10 Feb 2023 16:37:22 -0800
Subject: [PATCH 14/90] use YY.MM.DD as default version string

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index cf8350b..6cfc0a7 100644
--- a/Makefile
+++ b/Makefile
@@ -17,7 +17,7 @@ ifeq ($(strip $(shell git status --porcelain 2>/dev/null)),)
 else
 	GIT_STATE=dirty
 endif
-VERSION := $(shell TZ=UTC0 git show --quiet --date='format-local:%Y%m%dT%H%M%SZ' --format="%cd")
+VERSION := $(shell TZ=UTC0 git show --quiet --date='format-local:%Y.%m.%d' --format="%cd")
 DIST_DIR := dist
 CONFIG_DIR := config
 CACHE_DIR_ROOT := cache

From 770c4add580c19ed9d9ead6a2ac765a397cb889c Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sun, 12 Feb 2023 16:54:54 -0800
Subject: [PATCH 15/90] saner toolchain-clean

---
 Makefile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 6cfc0a7..3773ac9 100644
--- a/Makefile
+++ b/Makefile
@@ -79,12 +79,11 @@ toolchain-update:
 
 .PHONY: toolchain-clean
 toolchain-clean:
-	rm -rf cache out
-	docker image rm -f local/$(NAME)-build
+	rm -rf $(CACHE_DIR_ROOT) $(OUT_DIR)
+	docker image rm -f $(IMAGE)
 
 .PHONY: attest
-attest:
-	rm -rf $(CACHE_DIR) $(OUT_DIR)
+attest: toolchain-clean
 	mkdir -p $(OUT_DIR)
 	cp $(DIST_DIR)/release.env $(OUT_DIR)/release.env
 	$(MAKE) TARGET=$(TARGET) VERSION=$(VERSION)

From 6961056dd3a3826174c11bd08c381d81a1cba56f Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sun, 12 Feb 2023 21:36:05 -0800
Subject: [PATCH 16/90] GIT EPOCH/DATETIME -> TIMESTAMP

---
 Makefile            | 14 ++++++--------
 scripts/environment |  3 +--
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/Makefile b/Makefile
index 3773ac9..0f31083 100644
--- a/Makefile
+++ b/Makefile
@@ -9,9 +9,7 @@ CPUS := $(shell docker run -it debian nproc)
 GIT_REF := $(shell git log -1 --format=%H)
 GIT_AUTHOR := $(shell git log -1 --format=%an)
 GIT_KEY := $(shell git log -1 --format=%GP)
-GIT_EPOCH := $(shell git log -1 --format=%at)
-GIT_DATETIME := \
-	$(shell git log -1 --format=%cd --date=format:'%Y-%m-%d %H:%M:%S' config)
+GIT_TIMESTAMP := $(shell git log -1 --format=%cd --date=iso)
 ifeq ($(strip $(shell git status --porcelain 2>/dev/null)),)
 	GIT_STATE=clean
 else
@@ -138,11 +136,11 @@ $(DIST_DIR)/manifest.txt: \
 	cp $(OUT_DIR)/manifest.txt $(DIST_DIR)/manifest.txt
 
 $(OUT_DIR)/release.env: | $(OUT_DIR)
-	echo 'VERSION=$(VERSION)'            > $(OUT_DIR)/release.env
-	echo 'GIT_REF=$(GIT_REF)'           >> $(OUT_DIR)/release.env
-	echo 'GIT_AUTHOR=$(GIT_AUTHOR)'     >> $(OUT_DIR)/release.env
-	echo 'GIT_KEY=$(GIT_KEY)'           >> $(OUT_DIR)/release.env
-	echo 'GIT_DATETIME=$(GIT_DATETIME)' >> $(OUT_DIR)/release.env
+	echo 'VERSION=$(VERSION)'              > $(OUT_DIR)/release.env
+	echo 'GIT_REF=$(GIT_REF)'             >> $(OUT_DIR)/release.env
+	echo 'GIT_AUTHOR=$(GIT_AUTHOR)'       >> $(OUT_DIR)/release.env
+	echo 'GIT_KEY=$(GIT_KEY)'             >> $(OUT_DIR)/release.env
+	echo 'GIT_TIMESTAMP=$(GIT_TIMESTAMP)' >> $(OUT_DIR)/release.env
 
 $(OUT_DIR)/manifest.txt: | $(OUT_DIR)
 	find $(OUT_DIR) \
diff --git a/scripts/environment b/scripts/environment
index 7c2aae9..883f0d4 100755
--- a/scripts/environment
+++ b/scripts/environment
@@ -14,8 +14,7 @@ cat <<- EOF
 	GIT_REF=${GIT_REF}
 	GIT_AUTHOR=${GIT_AUTHOR}
 	GIT_KEY=${GIT_KEY}
-	GIT_DATETIME=${GIT_DATETIME}
-	GIT_EPOCH=${GIT_EPOCH}
+	GIT_TIMESTAMP=${GIT_DATETIME}
 	FAKETIME_FMT=%s
 	SOURCE_DATE_EPOCH=1
 	KBUILD_BUILD_TIMESTAMP=1970-01-01 00:00:00 UTC

From be41f7cfe1ea6331e17b8b3f4fffe2f509326e76 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sun, 12 Feb 2023 21:38:56 -0800
Subject: [PATCH 17/90] fix mis-named variable

---
 scripts/environment | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/environment b/scripts/environment
index 883f0d4..05a129c 100755
--- a/scripts/environment
+++ b/scripts/environment
@@ -14,7 +14,7 @@ cat <<- EOF
 	GIT_REF=${GIT_REF}
 	GIT_AUTHOR=${GIT_AUTHOR}
 	GIT_KEY=${GIT_KEY}
-	GIT_TIMESTAMP=${GIT_DATETIME}
+	GIT_TIMESTAMP=${GIT_TIMESTAMP}
 	FAKETIME_FMT=%s
 	SOURCE_DATE_EPOCH=1
 	KBUILD_BUILD_TIMESTAMP=1970-01-01 00:00:00 UTC

From d0d66c2d57be4ed0bb8c43fe2f2120d75d9ce2d3 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Mon, 13 Feb 2023 12:36:17 -0800
Subject: [PATCH 18/90] ensure NAME is always lowercase

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 0f31083..2696825 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@ DEFAULT_GOAL := $(or $(DEFAULT_GOAL),toolchain)
 ARCH := $(or $(ARCH),x86_64)
 TARGET := $(or $(TARGET),$(ARCH))
 PLATFORM := $(or $(PLATFORM),linux)
-NAME := $(shell basename $(shell git rev-parse --show-toplevel))
+NAME := $(shell basename $(shell git rev-parse --show-toplevel | tr A-Z a-z ))
 IMAGE := local/$(NAME):latest
 USER := $(shell id -u):$(shell id -g)
 CPUS := $(shell docker run -it debian nproc)

From 7a2917d37a4a9d70bb3bff5ce5cd8ad17e5934ad Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Wed, 15 Feb 2023 04:09:04 -0800
Subject: [PATCH 19/90] add VERSION to environment

---
 scripts/environment | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/environment b/scripts/environment
index 05a129c..bef6152 100755
--- a/scripts/environment
+++ b/scripts/environment
@@ -15,6 +15,7 @@ cat <<- EOF
 	GIT_AUTHOR=${GIT_AUTHOR}
 	GIT_KEY=${GIT_KEY}
 	GIT_TIMESTAMP=${GIT_TIMESTAMP}
+	VERSION=${VERSION}
 	FAKETIME_FMT=%s
 	SOURCE_DATE_EPOCH=1
 	KBUILD_BUILD_TIMESTAMP=1970-01-01 00:00:00 UTC

From 8c2e4b6ad12ac413a5ea9d82c878db632244ae36 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Wed, 15 Feb 2023 13:57:45 -0800
Subject: [PATCH 20/90] cache loading of toolchain.tar

---
 Makefile | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 2696825..86b59d7 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@ ARCH := $(or $(ARCH),x86_64)
 TARGET := $(or $(TARGET),$(ARCH))
 PLATFORM := $(or $(PLATFORM),linux)
 NAME := $(shell basename $(shell git rev-parse --show-toplevel | tr A-Z a-z ))
-IMAGE := local/$(NAME):latest
+IMAGE := local/$(NAME)
 USER := $(shell id -u):$(shell id -g)
 CPUS := $(shell docker run -it debian nproc)
 GIT_REF := $(shell git log -1 --format=%H)
@@ -53,6 +53,7 @@ toolchain: \
 	$(BIN_DIR) \
 	$(OUT_DIR) \
 	$(CACHE_DIR_ROOT)/toolchain.tar \
+	$(CACHE_DIR_ROOT)/toolchain.state \
 	$(CACHE_DIR_ROOT)/toolchain.env
 
 # Launch a shell inside the toolchain container
@@ -125,6 +126,11 @@ $(CACHE_DIR_ROOT)/toolchain.tar: \
 		.
 	docker save "$(IMAGE)" -o "$@"
 
+$(CACHE_DIR_ROOT)/toolchain.state: \
+	$(CACHE_DIR_ROOT)/toolchain.tar
+	docker load -i $(CACHE_DIR_ROOT)/toolchain.tar
+	docker images --no-trunc --quiet $(IMAGE) > $@
+
 $(DIST_DIR)/release.env: \
 	$(DIST_DIR) \
 	$(OUT_DIR)/release.env
@@ -193,7 +199,6 @@ define fetch_pgp_key
 endef
 
 define toolchain
-	docker load -i $(CACHE_DIR_ROOT)/toolchain.tar
 	docker run \
 		--rm \
 		--tty \
@@ -205,6 +210,6 @@ define toolchain
 		--workdir /home/build \
 		--env-file=$(CONFIG_DIR)/global.env \
 		--env-file=$(CACHE_DIR_ROOT)/toolchain.env \
-		$(IMAGE) \
+		$(shell cat cache/toolchain.state) \
 		bash -c $(2)
 endef

From 8b98574565d17b9261e7696ee8e83a6c0740a29c Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 16 Feb 2023 21:15:10 -0800
Subject: [PATCH 21/90] clone only required ref in git_clone

---
 Makefile | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 86b59d7..2569eba 100644
--- a/Makefile
+++ b/Makefile
@@ -161,11 +161,17 @@ check_executables := $(foreach exec,$(executables),\$(if \
 	$(shell which $(exec)),some string,$(error "No $(exec) in PATH")))
 
 define git_clone
-	[ -d $(1) ] || git clone $(2) $(1)
-	git -C $(1) checkout $(3)
-	git -C $(1) rev-parse --verify HEAD | grep -q $(3) || { \
-		echo 'Error: Git ref/branch collision.'; exit 1; \
-	};
+	[ -d $(1) ] || \
+		mkdir -p $(FETCH_DIR) && \
+		mkdir $(1).tmp && \
+		git -C $(1).tmp init && \
+		git -C $(1).tmp remote add origin $(2) && \
+		git -C $(1).tmp fetch origin $(3) && \
+		git -C $(1).tmp -c advice.detachedHead=false checkout $(3) && \
+		git -C $(1).tmp rev-parse --verify HEAD | grep -q $(3) || { \
+			echo 'Error: Git ref/branch collision.'; exit 1; \
+		} && \
+		mv $(1).tmp $(1);
 endef
 
 define apply_patches

From ca3e7960ea2abb9e448610c633dc92d7786ce8ab Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 24 Feb 2023 13:30:28 -0800
Subject: [PATCH 22/90] remove 'out/' prefix in manifest.txt

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2569eba..049e317 100644
--- a/Makefile
+++ b/Makefile
@@ -153,7 +153,7 @@ $(OUT_DIR)/manifest.txt: | $(OUT_DIR)
 		-type f \
 		-not -path "$(OUT_DIR)/manifest.txt" \
 		-exec openssl sha256 -r {} \; \
-	| sed -e 's/ \*/ /g' -e 's/ \.\// /g' \
+	| sed -e 's/ \*out\// /g' -e 's/ \.\// /g' \
 	| LC_ALL=C sort -k2 \
 	> $@
 

From 2e67bce822f6a072e3444758ebdc8b31788e3fd5 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 24 Feb 2023 13:42:59 -0800
Subject: [PATCH 23/90] only depend on toolchain.state in global 'toolchain'
 phony

---
 Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Makefile b/Makefile
index 049e317..9501da2 100644
--- a/Makefile
+++ b/Makefile
@@ -52,7 +52,6 @@ toolchain: \
 	$(DIST_DIR) \
 	$(BIN_DIR) \
 	$(OUT_DIR) \
-	$(CACHE_DIR_ROOT)/toolchain.tar \
 	$(CACHE_DIR_ROOT)/toolchain.state \
 	$(CACHE_DIR_ROOT)/toolchain.env
 

From 64b677d2353b0313f5f0b1561686905ceb90747e Mon Sep 17 00:00:00 2001
From: Jack Kearney <jack@turnkey.io>
Date: Fri, 24 Feb 2023 16:56:39 -0500
Subject: [PATCH 24/90] Follow symlinks while building manifest.txt

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 9501da2..5fb6bc7 100644
--- a/Makefile
+++ b/Makefile
@@ -148,7 +148,7 @@ $(OUT_DIR)/release.env: | $(OUT_DIR)
 	echo 'GIT_TIMESTAMP=$(GIT_TIMESTAMP)' >> $(OUT_DIR)/release.env
 
 $(OUT_DIR)/manifest.txt: | $(OUT_DIR)
-	find $(OUT_DIR) \
+	find -L $(OUT_DIR) \
 		-type f \
 		-not -path "$(OUT_DIR)/manifest.txt" \
 		-exec openssl sha256 -r {} \; \

From 6883a7dced5b3e466819e3abd703764b7864da81 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Mon, 27 Feb 2023 14:26:04 -0800
Subject: [PATCH 25/90] add DIST_DIR target

---
 Makefile | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/Makefile b/Makefile
index 2569eba..3fc54e4 100644
--- a/Makefile
+++ b/Makefile
@@ -49,7 +49,6 @@ executables = $(docker) git patch
 toolchain: \
 	$(CACHE_DIR) \
 	$(FETCH_DIR) \
-	$(DIST_DIR) \
 	$(BIN_DIR) \
 	$(OUT_DIR) \
 	$(CACHE_DIR_ROOT)/toolchain.tar \
@@ -88,8 +87,10 @@ attest: toolchain-clean
 	$(MAKE) TARGET=$(TARGET) VERSION=$(VERSION)
 	diff -q $(OUT_DIR)/manifest.txt $(DIST_DIR)/manifest.txt;
 
-$(DIST_DIR):
+$(DIST_DIR): default
 	mkdir -p $@
+	rm -rf $@/*
+	cp -R $(OUT_DIR)/* $@/
 
 $(BIN_DIR):
 	mkdir -p $@
@@ -131,16 +132,6 @@ $(CACHE_DIR_ROOT)/toolchain.state: \
 	docker load -i $(CACHE_DIR_ROOT)/toolchain.tar
 	docker images --no-trunc --quiet $(IMAGE) > $@
 
-$(DIST_DIR)/release.env: \
-	$(DIST_DIR) \
-	$(OUT_DIR)/release.env
-	cp $(OUT_DIR)/release.env $(DIST_DIR)/release.env
-
-$(DIST_DIR)/manifest.txt: \
-	$(DIST_DIR) \
-	$(OUT_DIR)/manifest.txt
-	cp $(OUT_DIR)/manifest.txt $(DIST_DIR)/manifest.txt
-
 $(OUT_DIR)/release.env: | $(OUT_DIR)
 	echo 'VERSION=$(VERSION)'              > $(OUT_DIR)/release.env
 	echo 'GIT_REF=$(GIT_REF)'             >> $(OUT_DIR)/release.env

From 27e0da87730fb3bb8845c68ea014b1f791149268 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 28 Feb 2023 10:33:59 -0800
Subject: [PATCH 26/90] ignore expired releases for apt installs

---
 scripts/packages-install | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/scripts/packages-install b/scripts/packages-install
index 13b6acf..756ea05 100755
--- a/scripts/packages-install
+++ b/scripts/packages-install
@@ -5,10 +5,16 @@ ARCH=$(uname -m)
 
 cp /config/toolchain/* /etc/apt/
 
-apt-get update
+apt-get update -o Acquire::Check-Valid-Until=false
 apt-get install debian-archive-keyring
 
-until apt-get install --download-only --reinstall --allow-downgrades -y $(cat /etc/apt/packages-${ARCH}.list); do
+until apt-get install \
+    --download-only \
+    --reinstall \
+    --allow-downgrades \
+    -o Acquire::Check-Valid-Until=false \
+    -y $(cat /etc/apt/packages-${ARCH}.list);
+do
     echo "apt install failed. Likely throttled. Retrying in 10 mins...";
     sleep 600;
 done;
@@ -22,5 +28,7 @@ done;
 
 diff /etc/apt/package-hashes-${ARCH}{,-compare}.txt
 
-apt-get install --allow-downgrades -y $(cat /etc/apt/packages-${ARCH}.list)
+apt-get install \
+    --allow-downgrades \
+    -y $(cat /etc/apt/packages-${ARCH}.list)
 rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /var/tmp/*;

From 71f3e33623da960c06274104883f68d71aab298f Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 2 Mar 2023 09:27:27 -0800
Subject: [PATCH 27/90] smarter dist/manifest management

---
 Makefile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index a840eab..39811f3 100644
--- a/Makefile
+++ b/Makefile
@@ -86,9 +86,10 @@ attest: toolchain-clean
 	$(MAKE) TARGET=$(TARGET) VERSION=$(VERSION)
 	diff -q $(OUT_DIR)/manifest.txt $(DIST_DIR)/manifest.txt;
 
-$(DIST_DIR): default
-	mkdir -p $@
+.PHONY: $(DIST_DIR)
+$(DIST_DIR):
 	rm -rf $@/*
+	$(MAKE) toolchain-clean default
 	cp -R $(OUT_DIR)/* $@/
 
 $(BIN_DIR):
@@ -138,7 +139,7 @@ $(OUT_DIR)/release.env: | $(OUT_DIR)
 	echo 'GIT_KEY=$(GIT_KEY)'             >> $(OUT_DIR)/release.env
 	echo 'GIT_TIMESTAMP=$(GIT_TIMESTAMP)' >> $(OUT_DIR)/release.env
 
-$(OUT_DIR)/manifest.txt: | $(OUT_DIR)
+$(OUT_DIR)/manifest.txt: $(wildcard $(OUT_DIR)/*)
 	find -L $(OUT_DIR) \
 		-type f \
 		-not -path "$(OUT_DIR)/manifest.txt" \

From d6189f0f2856d309685bfd40d9999e2b7a08ce54 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 7 Mar 2023 18:09:52 -0800
Subject: [PATCH 28/90] drop USER and use host-env for all toolchain
 invocations

---
 Dockerfile       |  3 ---
 Makefile         | 29 ++++++++++++++++++-----------
 scripts/host-env |  4 +---
 3 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 1efd80a..021ea14 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -8,6 +8,3 @@ ARG SCRIPTS_DIR
 ADD ${SCRIPTS_DIR} /usr/local/bin
 
 RUN packages-install
-
-RUN echo "/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1" \
-    > /etc/ld.so.preload
diff --git a/Makefile b/Makefile
index 39811f3..317020c 100644
--- a/Makefile
+++ b/Makefile
@@ -4,12 +4,15 @@ TARGET := $(or $(TARGET),$(ARCH))
 PLATFORM := $(or $(PLATFORM),linux)
 NAME := $(shell basename $(shell git rev-parse --show-toplevel | tr A-Z a-z ))
 IMAGE := local/$(NAME)
-USER := $(shell id -u):$(shell id -g)
+UID := $(shell id -u)
+GID := $(shell id -g)
+USER := $(UID):$(GID)
 CPUS := $(shell docker run -it debian nproc)
 GIT_REF := $(shell git log -1 --format=%H)
 GIT_AUTHOR := $(shell git log -1 --format=%an)
 GIT_KEY := $(shell git log -1 --format=%GP)
 GIT_TIMESTAMP := $(shell git log -1 --format=%cd --date=iso)
+, := ,
 ifeq ($(strip $(shell git status --porcelain 2>/dev/null)),)
 	GIT_STATE=clean
 else
@@ -57,22 +60,24 @@ toolchain: \
 # Launch a shell inside the toolchain container
 .PHONY: toolchain-shell
 toolchain-shell: toolchain
-	$(call toolchain,$(USER),"bash --norc")
+	$(call toolchain,bash --norc)
 
 # Pin all packages in toolchain container to latest versions
 .PHONY: toolchain-update
 toolchain-update:
 	docker run \
 		--rm \
-		--env LOCAL_USER=$(USER) \
+		--tty \
+		--interactive \
 		--platform=linux/$(ARCH) \
+		--env LOCAL_USER=$(UID):$(GID) \
 		--volume $(PWD)/$(CONFIG_DIR):/config \
 		--volume $(PWD)/$(SRC_DIR)/toolchain/scripts:/usr/local/bin \
-		--env ARCH=$(ARCH) \
-		--interactive \
-		--tty \
+		--cpus $(CPUS) \
+		--volume $(PWD):/home/build \
+		--workdir /home/build \
 		debian@sha256:$(DEBIAN_HASH) \
-		bash -c /usr/local/bin/packages-update
+		/usr/local/bin/packages-update
 
 .PHONY: toolchain-clean
 toolchain-clean:
@@ -166,7 +171,7 @@ define git_clone
 endef
 
 define apply_patches
-	[ -d $(2) ] && $(call toolchain,$(USER)," \
+	[ -d $(2) ] && $(call toolchain," \
 		cd $(1); \
 		git restore .; \
 		find /$(2) -type f -iname '*.patch' -print0 \
@@ -176,7 +181,7 @@ endef
 
 define fetch_pgp_key
         mkdir -p $(KEY_DIR) && \
-        $(call toolchain,$(USER), " \
+        $(call toolchain," \
                 for server in \
             ha.pool.sks-keyservers.net \
             hkp://keyserver.ubuntu.com:80 \
@@ -200,13 +205,15 @@ define toolchain
 		--rm \
 		--tty \
 		--interactive \
-		--user=$(1) \
+		--env UID=$(UID) \
+		--env GID=$(GID) \
 		--platform=linux/$(ARCH) \
+		--privileged \
 		--cpus $(CPUS) \
 		--volume $(PWD):/home/build \
 		--workdir /home/build \
 		--env-file=$(CONFIG_DIR)/global.env \
 		--env-file=$(CACHE_DIR_ROOT)/toolchain.env \
 		$(shell cat cache/toolchain.state) \
-		bash -c $(2)
+		$(SRC_DIR)/toolchain/scripts/host-env bash -c $(1)
 endef
diff --git a/scripts/host-env b/scripts/host-env
index ec7ca98..499cf3d 100755
--- a/scripts/host-env
+++ b/scripts/host-env
@@ -10,11 +10,9 @@ groupadd -g "$gid" "${user}"
 useradd \
 	-g "$gid" \
 	-u "$uid" \
-	-md "/home/${user}" \
+	-d "/home/${user}" \
 	-s /bin/bash \
 	"${user}"
 
-mkdir -p "$HOME"
-chown -R "$uid:$gid" "$HOME"
 cd "$HOME"
 setpriv --reuid="$uid" --regid="$gid" --init-groups "$@"

From 47e883a3482a718e1b752d382d5eed4ee9ad05e1 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 16 Mar 2023 20:11:43 -0700
Subject: [PATCH 29/90] fix pgp key fetching

---
 Makefile | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/Makefile b/Makefile
index 317020c..77c43fd 100644
--- a/Makefile
+++ b/Makefile
@@ -181,23 +181,23 @@ endef
 
 define fetch_pgp_key
         mkdir -p $(KEY_DIR) && \
-        $(call toolchain," \
-                for server in \
-            ha.pool.sks-keyservers.net \
-            hkp://keyserver.ubuntu.com:80 \
-            hkp://p80.pool.sks-keyservers.net:80 \
-            pgp.mit.edu \
-        ; do \
-                        echo "Trying: $${server}"; \
-                gpg \
-                        --recv-key \
-                        --keyserver "$${server}" \
-                        --keyserver-options timeout=10 \
-                        --recv-keys "$(1)" \
-                && break; \
-        done; \
-                gpg --export -a $(1) > $(KEY_DIR)/$(1).asc; \
-        ")
+        $(call toolchain,' \
+			for server in \
+        	    keys.openpgp.org \
+        	    hkp://keyserver.ubuntu.com:80 \
+        	    hkp://p80.pool.sks-keyservers.net:80 \
+        	    ha.pool.sks-keyservers.net \
+        	    pgp.mit.edu \
+        	; do \
+        	                echo "Trying: $${server}"; \
+        	        gpg \
+        	                --keyserver "$${server}" \
+        	                --keyserver-options timeout=10 \
+        	                --recv-keys "$(1)" \
+        	        && break; \
+        	done; \
+            gpg --export -a $(1) > $(KEY_DIR)/$(1).asc; \
+        ')
 endef
 
 define toolchain

From fbf48b33a005d1c71b55fb3eefeab0ba900e38ae Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sun, 2 Apr 2023 21:09:46 -0700
Subject: [PATCH 30/90] delete dialout group that conflicts with mac users

---
 scripts/host-env | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/host-env b/scripts/host-env
index 499cf3d..760e90f 100755
--- a/scripts/host-env
+++ b/scripts/host-env
@@ -6,6 +6,7 @@ gid=${GID?}
 user=${USER:-"build"}
 export HOME="/home/${user}"
 
+groupdel dialout
 groupadd -g "$gid" "${user}"
 useradd \
 	-g "$gid" \

From d5fcfe288666f202f4ef71e0536c81fa1e8dcbc1 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 4 Apr 2023 09:42:46 -0700
Subject: [PATCH 31/90] Host os/arch vars

---
 Makefile | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Makefile b/Makefile
index 77c43fd..315de26 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,12 @@
+lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
+altarch = $(subst x86_64,amd64,$(subst aarch64,arm64,$1))
+
 DEFAULT_GOAL := $(or $(DEFAULT_GOAL),toolchain)
 ARCH := $(or $(ARCH),x86_64)
 TARGET := $(or $(TARGET),$(ARCH))
+HOST_ARCH := $(call lc,$(shell uname -m))
+HOST_ARCH_ALT := $(call altarch,$(HOST_ARCH))
+HOST_OS := $(call lc,$(shell uname -s))
 PLATFORM := $(or $(PLATFORM),linux)
 NAME := $(shell basename $(shell git rev-parse --show-toplevel | tr A-Z a-z ))
 IMAGE := local/$(NAME)

From bcea9f50b8a33dbf61e1ad7e94be02315ef9ae24 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 4 Apr 2023 14:16:53 -0700
Subject: [PATCH 32/90] attest -> reproduce

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 315de26..18f5d06 100644
--- a/Makefile
+++ b/Makefile
@@ -90,8 +90,8 @@ toolchain-clean:
 	rm -rf $(CACHE_DIR_ROOT) $(OUT_DIR)
 	docker image rm -f $(IMAGE)
 
-.PHONY: attest
-attest: toolchain-clean
+.PHONY: reproduce
+reproduce: toolchain-clean
 	mkdir -p $(OUT_DIR)
 	cp $(DIST_DIR)/release.env $(OUT_DIR)/release.env
 	$(MAKE) TARGET=$(TARGET) VERSION=$(VERSION)

From 7445dc5da882f65b8c8b3318b5ca715cc49a007f Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 4 Apr 2023 14:36:28 -0700
Subject: [PATCH 33/90] remove read only cache files

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 18f5d06..c71aac3 100644
--- a/Makefile
+++ b/Makefile
@@ -87,6 +87,7 @@ toolchain-update:
 
 .PHONY: toolchain-clean
 toolchain-clean:
+	chmod -R u+w $(CACHE_DIR_ROOT)
 	rm -rf $(CACHE_DIR_ROOT) $(OUT_DIR)
 	docker image rm -f $(IMAGE)
 

From 8027bcc0b5d78e462c9f1ceb7bf6a127509fd91c Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 4 Apr 2023 14:41:02 -0700
Subject: [PATCH 34/90] suppress toolchain state missing errors

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index c71aac3..c31f116 100644
--- a/Makefile
+++ b/Makefile
@@ -221,6 +221,6 @@ define toolchain
 		--workdir /home/build \
 		--env-file=$(CONFIG_DIR)/global.env \
 		--env-file=$(CACHE_DIR_ROOT)/toolchain.env \
-		$(shell cat cache/toolchain.state) \
+		$(shell cat cache/toolchain.state 2> /dev/null) \
 		$(SRC_DIR)/toolchain/scripts/host-env bash -c $(1)
 endef

From 26ad161faf9da97cec3692599abb3da4f2f97581 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 4 Apr 2023 14:52:04 -0700
Subject: [PATCH 35/90] Inform user when reproduce works

---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index c31f116..55e158c 100644
--- a/Makefile
+++ b/Makefile
@@ -96,7 +96,8 @@ reproduce: toolchain-clean
 	mkdir -p $(OUT_DIR)
 	cp $(DIST_DIR)/release.env $(OUT_DIR)/release.env
 	$(MAKE) TARGET=$(TARGET) VERSION=$(VERSION)
-	diff -q $(OUT_DIR)/manifest.txt $(DIST_DIR)/manifest.txt;
+	diff -q $(OUT_DIR)/manifest.txt $(DIST_DIR)/manifest.txt \
+	&& echo "Success: $(OUT_DIR) and $(DIST_DIR) are identical"
 
 .PHONY: $(DIST_DIR)
 $(DIST_DIR):

From beac29f3d937b7737f5ce2dbbdd54b72f4257d0c Mon Sep 17 00:00:00 2001
From: shane <shane@distrust.co>
Date: Wed, 5 Apr 2023 11:08:28 -0500
Subject: [PATCH 36/90] Fix typo in README- 'codeburg' => 'codeberg'

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index cf6d7c9..255d6dc 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ us as desired.
 1. Clone toolchain as a git submodule somewhere in your project
 
     ```
-    git submodule add https://codeburg.org/distrust/toolchain src/toolchain
+    git submodule add https://codeberg.org/distrust/toolchain src/toolchain
     ```
 
 2. Include toolchain Makefile in your root Makefile

From b499727bbbfb2d10a2ff66c767c61192ddf98261 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Wed, 5 Apr 2023 14:16:34 -0700
Subject: [PATCH 37/90] use sh for environment file

---
 scripts/environment | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/environment b/scripts/environment
index bef6152..2bb6cd2 100755
--- a/scripts/environment
+++ b/scripts/environment
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 
 HOME=/home/build
 cat <<- EOF

From a65be367d953846718ad37901bb08bcc78f1a6c8 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 13 Apr 2023 14:20:55 -0700
Subject: [PATCH 38/90] sudo support

---
 scripts/host-env         | 1 +
 scripts/packages-install | 2 ++
 scripts/packages-update  | 6 +++++-
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/scripts/host-env b/scripts/host-env
index 760e90f..efa7e99 100755
--- a/scripts/host-env
+++ b/scripts/host-env
@@ -10,6 +10,7 @@ groupdel dialout
 groupadd -g "$gid" "${user}"
 useradd \
 	-g "$gid" \
+	-G sudo \
 	-u "$uid" \
 	-d "/home/${user}" \
 	-s /bin/bash \
diff --git a/scripts/packages-install b/scripts/packages-install
index 756ea05..f09a1cd 100755
--- a/scripts/packages-install
+++ b/scripts/packages-install
@@ -32,3 +32,5 @@ apt-get install \
     --allow-downgrades \
     -y $(cat /etc/apt/packages-${ARCH}.list)
 rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /var/tmp/*;
+
+echo "%sudo ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers
diff --git a/scripts/packages-update b/scripts/packages-update
index c3e6ec4..fa0b4f4 100755
--- a/scripts/packages-update
+++ b/scripts/packages-update
@@ -25,7 +25,11 @@ apt-get install -y --download-only --reinstall $( \
 	| awk -F'\t' '/^ii/ {print $2}' \
 	| awk '{print $1}' \
 )
-apt-get install -y --download-only $(cat /config/toolchain/packages-base.list)
+apt-get install \
+	-y \
+	--download-only \
+	sudo \
+	$(cat /config/toolchain/packages-base.list)
 
 ( cd /var/cache/apt/archives \
 	&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \

From dafafd5d6c7070958f4eea1198a52099b4f2ac09 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 27 Apr 2023 13:40:31 -0700
Subject: [PATCH 39/90] make only toolchain-shell interactive, only delete dirs
 if they exist

---
 Makefile | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 55e158c..e724643 100644
--- a/Makefile
+++ b/Makefile
@@ -66,7 +66,7 @@ toolchain: \
 # Launch a shell inside the toolchain container
 .PHONY: toolchain-shell
 toolchain-shell: toolchain
-	$(call toolchain,bash --norc)
+	$(call toolchain,bash --norc,--interactive)
 
 # Pin all packages in toolchain container to latest versions
 .PHONY: toolchain-update
@@ -74,7 +74,6 @@ toolchain-update:
 	docker run \
 		--rm \
 		--tty \
-		--interactive \
 		--platform=linux/$(ARCH) \
 		--env LOCAL_USER=$(UID):$(GID) \
 		--volume $(PWD)/$(CONFIG_DIR):/config \
@@ -87,9 +86,14 @@ toolchain-update:
 
 .PHONY: toolchain-clean
 toolchain-clean:
-	chmod -R u+w $(CACHE_DIR_ROOT)
-	rm -rf $(CACHE_DIR_ROOT) $(OUT_DIR)
-	docker image rm -f $(IMAGE)
+	if [ -d "$(CACHE_DIR_ROOT)" ]; then \
+		chmod -R u+w $(CACHE_DIR_ROOT); \
+		rm -rf $(CACHE_DIR_ROOT); \
+	fi
+	if [ -d "$(OUT_DIR)" ]; then \
+		rm -rf $(OUT_DIR); \
+	fi
+	docker image rm -f $(IMAGE) || :
 
 .PHONY: reproduce
 reproduce: toolchain-clean
@@ -212,7 +216,7 @@ define toolchain
 	docker run \
 		--rm \
 		--tty \
-		--interactive \
+		$(2) \
 		--env UID=$(UID) \
 		--env GID=$(GID) \
 		--platform=linux/$(ARCH) \

From 4eff8b258b1e9bc64f3d831ae160ac9256bfdd79 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 27 Apr 2023 13:54:27 -0700
Subject: [PATCH 40/90] optional PRESERVE_CACHE argument

---
 Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index e724643..6a1f0cb 100644
--- a/Makefile
+++ b/Makefile
@@ -14,6 +14,7 @@ UID := $(shell id -u)
 GID := $(shell id -g)
 USER := $(UID):$(GID)
 CPUS := $(shell docker run -it debian nproc)
+PRESERVE_CACHE := "false"
 GIT_REF := $(shell git log -1 --format=%H)
 GIT_AUTHOR := $(shell git log -1 --format=%an)
 GIT_KEY := $(shell git log -1 --format=%GP)
@@ -106,7 +107,8 @@ reproduce: toolchain-clean
 .PHONY: $(DIST_DIR)
 $(DIST_DIR):
 	rm -rf $@/*
-	$(MAKE) toolchain-clean default
+	[ "$(PRESERVE_CACHE)" = "true" ] || $(MAKE) toolchain-clean
+	$(MAKE) default
 	cp -R $(OUT_DIR)/* $@/
 
 $(BIN_DIR):

From 70c9e403ac30d02871923dc6af2f035de0d1cc3c Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Mon, 1 May 2023 14:10:33 -0700
Subject: [PATCH 41/90] add git-lfs to required binary checks

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 6a1f0cb..33e1752 100644
--- a/Makefile
+++ b/Makefile
@@ -53,7 +53,7 @@ ifneq (,$(wildcard $(DIST_DIR)/release.env))
     export
 endif
 
-executables = $(docker) git patch
+executables = $(docker) git git-lfs patch
 
 .PHONY: toolchain
 toolchain: \

From 108089eac206def41dd86b7312190e90b0c0c038 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Mon, 1 May 2023 15:53:32 -0700
Subject: [PATCH 42/90] handle host user being root or a default debian
 user/group

---
 scripts/host-env | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/scripts/host-env b/scripts/host-env
index efa7e99..17b978e 100755
--- a/scripts/host-env
+++ b/scripts/host-env
@@ -6,15 +6,23 @@ gid=${GID?}
 user=${USER:-"build"}
 export HOME="/home/${user}"
 
-groupdel dialout
-groupadd -g "$gid" "${user}"
-useradd \
-	-g "$gid" \
-	-G sudo \
-	-u "$uid" \
-	-d "/home/${user}" \
-	-s /bin/bash \
-	"${user}"
+# If running user is not root, make a custom user/group
+[ "$uid" != "0" ] && {
+    getent group "$gid" && groupdel "$gid"
+    getent passwd "$uid" && userdel "$uid"
+    groupadd -g "$gid" "${user}"
+    useradd \
+    	-g "$gid" \
+    	-G sudo \
+    	-u "$uid" \
+    	-d "/home/${user}" \
+    	-s /bin/bash \
+    	"${user}"
+}
 
 cd "$HOME"
-setpriv --reuid="$uid" --regid="$gid" --init-groups "$@"
+
+# If running user is not root, pivot to running user
+[ "$uid" != "0" ] && {
+    setpriv --reuid="$uid" --regid="$gid" --init-groups "$@"
+}

From a4086420ca122e7e6355c9aa726fa5db67080c1d Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 2 May 2023 14:55:45 -0700
Subject: [PATCH 43/90] allow override for toolchain docker volume and workdir
 for niche environments

---
 Makefile         | 35 +++++++++++++++++++----------------
 scripts/host-env | 26 +++++++++++++++-----------
 2 files changed, 34 insertions(+), 27 deletions(-)

diff --git a/Makefile b/Makefile
index 33e1752..18ff378 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,7 @@ IMAGE := local/$(NAME)
 UID := $(shell id -u)
 GID := $(shell id -g)
 USER := $(UID):$(GID)
-CPUS := $(shell docker run -it debian nproc)
+CPUS := $(shell docker run debian nproc)
 PRESERVE_CACHE := "false"
 GIT_REF := $(shell git log -1 --format=%H)
 GIT_AUTHOR := $(shell git log -1 --format=%an)
@@ -214,20 +214,23 @@ define fetch_pgp_key
         ')
 endef
 
+TOOLCHAIN_VOLUME := $(PWD):/home/build
+TOOLCHAIN_WORKDIR := /home/build
 define toolchain
-	docker run \
-		--rm \
-		--tty \
-		$(2) \
-		--env UID=$(UID) \
-		--env GID=$(GID) \
-		--platform=linux/$(ARCH) \
-		--privileged \
-		--cpus $(CPUS) \
-		--volume $(PWD):/home/build \
-		--workdir /home/build \
-		--env-file=$(CONFIG_DIR)/global.env \
-		--env-file=$(CACHE_DIR_ROOT)/toolchain.env \
-		$(shell cat cache/toolchain.state 2> /dev/null) \
-		$(SRC_DIR)/toolchain/scripts/host-env bash -c $(1)
+        docker run \
+                --rm \
+				--tty \
+                $(2) \
+                --env UID=$(UID) \
+                --env GID=$(GID) \
+                --platform=linux/$(ARCH) \
+                --privileged \
+                --cpus $(CPUS) \
+                --volume $(TOOLCHAIN_VOLUME) \
+                --workdir $(TOOLCHAIN_WORKDIR) \
+                --env-file=$(CONFIG_DIR)/global.env \
+                --env-file=$(CACHE_DIR_ROOT)/toolchain.env \
+                $(shell cat cache/toolchain.state 2> /dev/null) \
+                $(SRC_DIR)/toolchain/scripts/host-env bash -c $(1)
 endef
+
diff --git a/scripts/host-env b/scripts/host-env
index efa7e99..8c71cb7 100755
--- a/scripts/host-env
+++ b/scripts/host-env
@@ -6,15 +6,19 @@ gid=${GID?}
 user=${USER:-"build"}
 export HOME="/home/${user}"
 
-groupdel dialout
-groupadd -g "$gid" "${user}"
-useradd \
-	-g "$gid" \
-	-G sudo \
-	-u "$uid" \
-	-d "/home/${user}" \
-	-s /bin/bash \
-	"${user}"
+# If running user is not root, pivot to custom user/group
+[ "$uid" != "0" ] && {
+    getent group "$gid" && groupdel "$gid"
+    getent passwd "$uid" && userdel "$uid"
+    groupadd -g "$gid" "${user}"
+    useradd \
+        -g "$gid" \
+        -G sudo \
+        -u "$uid" \
+        -d "/home/${user}" \
+        -s /bin/bash \
+        "${user}"
+    setpriv --reuid="$uid" --regid="$gid" --init-groups "$@"
+}
 
-cd "$HOME"
-setpriv --reuid="$uid" --regid="$gid" --init-groups "$@"
+exec "$@"

From 2027f56e1ad65e9966df7c219f33fba634fd9cf7 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 2 May 2023 16:02:33 -0700
Subject: [PATCH 44/90] use variable for /home/build everywhere

---
 Makefile | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 4e2c375..4a854f5 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,9 @@
 lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
 altarch = $(subst x86_64,amd64,$(subst aarch64,arm64,$1))
 
+TOOLCHAIN_VOLUME := $(PWD):/home/build
+TOOLCHAIN_WORKDIR := /home/build
+
 DEFAULT_GOAL := $(or $(DEFAULT_GOAL),toolchain)
 ARCH := $(or $(ARCH),x86_64)
 TARGET := $(or $(TARGET),$(ARCH))
@@ -80,8 +83,8 @@ toolchain-update:
 		--volume $(PWD)/$(CONFIG_DIR):/config \
 		--volume $(PWD)/$(SRC_DIR)/toolchain/scripts:/usr/local/bin \
 		--cpus $(CPUS) \
-		--volume $(PWD):/home/build \
-		--workdir /home/build \
+		--volume $(TOOLCHAIN_VOLUME) \
+		--workdir $(TOOLCHAIN_WORKDIR) \
 		debian@sha256:$(DEBIAN_HASH) \
 		/usr/local/bin/packages-update
 
@@ -214,8 +217,6 @@ define fetch_pgp_key
         ')
 endef
 
-TOOLCHAIN_VOLUME := $(PWD):/home/build
-TOOLCHAIN_WORKDIR := /home/build
 define toolchain
         docker run \
                 --rm \

From 2aac13f095f20fefdc83c79ce2ce3354fb5b816a Mon Sep 17 00:00:00 2001
From: Arnaud Brousseau <rno@turnkey.io>
Date: Wed, 3 May 2023 15:04:10 -0500
Subject: [PATCH 45/90] Normalize HOST_ARCH

---
 Makefile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 6a1f0cb..8ea4c95 100644
--- a/Makefile
+++ b/Makefile
@@ -4,8 +4,11 @@ altarch = $(subst x86_64,amd64,$(subst aarch64,arm64,$1))
 DEFAULT_GOAL := $(or $(DEFAULT_GOAL),toolchain)
 ARCH := $(or $(ARCH),x86_64)
 TARGET := $(or $(TARGET),$(ARCH))
-HOST_ARCH := $(call lc,$(shell uname -m))
+
+normarch = $(subst arm64,aarch64,$(subst amd64,x86_64,$1))
+HOST_ARCH := $(call normarch,$(call lc,$(shell uname -m)))
 HOST_ARCH_ALT := $(call altarch,$(HOST_ARCH))
+
 HOST_OS := $(call lc,$(shell uname -s))
 PLATFORM := $(or $(PLATFORM),linux)
 NAME := $(shell basename $(shell git rev-parse --show-toplevel | tr A-Z a-z ))

From c80a7dc0182fdc5d3e737296eed20286a81fabeb Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Wed, 3 May 2023 13:22:51 -0700
Subject: [PATCH 46/90] delete existing user/groups by name vs id

---
 scripts/host-env | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/host-env b/scripts/host-env
index 2c6d91b..463977a 100755
--- a/scripts/host-env
+++ b/scripts/host-env
@@ -8,8 +8,10 @@ export HOME="/home/${user}"
 
 # If running user is not root, pivot to custom user/group
 [ "$uid" != "0" ] && {
-    getent group "$gid" && groupdel "$gid"
-    getent passwd "$uid" && userdel "$uid"
+    getent group "$gid" \
+        && groupdel "$(awk -v i="$gid" -F: '$3 == i' /etc/group | cut -d: -f1)"
+    getent passwd "$uid" \
+        && userdel "$(awk -v i="$uid" -F: '$3 == i' /etc/passwd | cut -d: -f1)"
     groupadd -g "$gid" "${user}"
     useradd \
         -g "$gid" \

From 7b178a53ef99504f94c184b1630468cfd6b2b010 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Wed, 3 May 2023 18:30:07 -0700
Subject: [PATCH 47/90] only exec if calling user is root

---
 scripts/host-env | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/scripts/host-env b/scripts/host-env
index 463977a..159136d 100755
--- a/scripts/host-env
+++ b/scripts/host-env
@@ -7,7 +7,7 @@ user=${USER:-"build"}
 export HOME="/home/${user}"
 
 # If running user is not root, pivot to custom user/group
-[ "$uid" != "0" ] && {
+if [ "$uid" != "0" ]; then
     getent group "$gid" \
         && groupdel "$(awk -v i="$gid" -F: '$3 == i' /etc/group | cut -d: -f1)"
     getent passwd "$uid" \
@@ -21,8 +21,6 @@ export HOME="/home/${user}"
         -s /bin/bash \
     "${user}"
     setpriv --reuid="$uid" --regid="$gid" --init-groups "$@"
-}
-
-exec "$@"
-
-
+else
+    exec "$@"
+fi

From e2f68de8da5dbc6936478f53fb9497817c2ce7ef Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 4 May 2023 12:10:38 -0700
Subject: [PATCH 48/90] drop manifest.txt support favoring diff and git-sig

---
 Makefile  | 11 +----------
 README.md | 15 ---------------
 2 files changed, 1 insertion(+), 25 deletions(-)

diff --git a/Makefile b/Makefile
index 96e1487..eb6bae4 100644
--- a/Makefile
+++ b/Makefile
@@ -107,7 +107,7 @@ reproduce: toolchain-clean
 	mkdir -p $(OUT_DIR)
 	cp $(DIST_DIR)/release.env $(OUT_DIR)/release.env
 	$(MAKE) TARGET=$(TARGET) VERSION=$(VERSION)
-	diff -q $(OUT_DIR)/manifest.txt $(DIST_DIR)/manifest.txt \
+	diff -q $(OUT_DIR) $(DIST_DIR) \
 	&& echo "Success: $(OUT_DIR) and $(DIST_DIR) are identical"
 
 .PHONY: $(DIST_DIR)
@@ -164,15 +164,6 @@ $(OUT_DIR)/release.env: | $(OUT_DIR)
 	echo 'GIT_KEY=$(GIT_KEY)'             >> $(OUT_DIR)/release.env
 	echo 'GIT_TIMESTAMP=$(GIT_TIMESTAMP)' >> $(OUT_DIR)/release.env
 
-$(OUT_DIR)/manifest.txt: $(wildcard $(OUT_DIR)/*)
-	find -L $(OUT_DIR) \
-		-type f \
-		-not -path "$(OUT_DIR)/manifest.txt" \
-		-exec openssl sha256 -r {} \; \
-	| sed -e 's/ \*out\// /g' -e 's/ \.\// /g' \
-	| LC_ALL=C sort -k2 \
-	> $@
-
 check_executables := $(foreach exec,$(executables),\$(if \
 	$(shell which $(exec)),some string,$(error "No $(exec) in PATH")))
 
diff --git a/README.md b/README.md
index 255d6dc..4902cb5 100644
--- a/README.md
+++ b/README.md
@@ -80,21 +80,6 @@ us as desired.
       ")
     ```
 
-7. Define a release target for your project depending on manifest.txt
-
-    ```
-    .PHONY: release
-    release: $(OUT_DIR)/hello $(OUT_DIR)/manifest.txt
-    	mkdir -p $(RELEASE_DIR)
-    	cp $(OUT_DIR)/hello $(RELEASE_DIR)/hello
-    	cp $(OUT_DIR)/release.env $(RELEASE_DIR)/release.env
-    	cp $(OUT_DIR)/manifest.txt $(RELEASE_DIR)/manifest.txt
-    ```
-
-    Note that manifest.txt is optional, but it makes for an ideal single file
-    to sign if a release will contain more than one artifact.
-
-
 ## Usage ##
 
 ### Build a new release

From 1e1d4ae3a9ca8bc9515214edecef06596e2ee926 Mon Sep 17 00:00:00 2001
From: RyanSquared <ryan@hashbang.sh>
Date: Fri, 5 May 2023 11:11:17 -0400
Subject: [PATCH 49/90] scripts/pcakages-update: only use packages from debian
 snapshot

---
 scripts/packages-update | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scripts/packages-update b/scripts/packages-update
index fa0b4f4..c9c90e6 100755
--- a/scripts/packages-update
+++ b/scripts/packages-update
@@ -6,14 +6,12 @@ set -e
 snapshot_url="http://snapshot.debian.org/archive/debian"
 snapshot_date=$(date +"%Y%m%dT000000Z")
 cat <<-EOF > /etc/apt/sources.list
-deb http://deb.debian.org/debian bookworm main
-deb http://security.debian.org/debian-security bookworm-security main
-deb http://deb.debian.org/debian bookworm-updates main
 deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm main
 deb [trusted=yes] ${snapshot_url}-security/${snapshot_date} bookworm-security main
 deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm-updates main
 EOF
 cp /etc/apt/sources.list /config/toolchain/
+rm /etc/apt/sources.list.d/*
 
 ARCH=$(uname -m)
 

From ee915227d502daccc4824d92623525973cec90d9 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 1 Jun 2023 18:04:41 -0700
Subject: [PATCH 50/90] separate toolchain.env and global.env. Allow env vars
 in global.env

---
 Makefile                | 24 +++++++++++++++++-------
 scripts/environment     |  6 ++++++
 scripts/packages-update |  2 +-
 3 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/Makefile b/Makefile
index 96e1487..777c64a 100644
--- a/Makefile
+++ b/Makefile
@@ -50,8 +50,8 @@ export
 
 docker = docker
 
-include $(CONFIG_DIR)/global.env
-export $(shell sed 's/=.*//' $(CONFIG_DIR)/global.env)
+include $(CONFIG_DIR)/toolchain.env
+export $(shell sed 's/=.*//' $(CONFIG_DIR)/toolchain.env)
 
 ## Use env vars from existing release if present
 ifneq (,$(wildcard $(DIST_DIR)/release.env))
@@ -130,11 +130,22 @@ $(OUT_DIR):
 	mkdir -p $@
 
 $(CACHE_DIR_ROOT)/toolchain.env: \
-	$(CACHE_DIR) \
-	$(SRC_DIR)/toolchain/scripts/environment
-	$(SRC_DIR)/toolchain/scripts/environment > $@
+	$(CACHE_DIR_ROOT)/toolchain.state
+	env > $(CACHE_DIR)/bootstrap.env
+	docker run \
+        --rm \
+        --env UID=$(UID) \
+        --env GID=$(GID) \
+        --env-file $(CACHE_DIR)/bootstrap.env \
+        --platform=linux/$(ARCH) \
+        --volume $(TOOLCHAIN_VOLUME) \
+        --workdir $(TOOLCHAIN_WORKDIR) \
+        $(shell cat cache/toolchain.state 2> /dev/null) \
+        $(SRC_DIR)/toolchain/scripts/environment > $@
+	rm $(CACHE_DIR)/bootstrap.env
 
 $(CACHE_DIR_ROOT)/toolchain.tar: \
+	$(CONFIG_DIR)/toolchain.env \
 	$(SRC_DIR)/toolchain/Dockerfile \
 	$(CONFIG_DIR)/toolchain/package-hashes-$(ARCH).txt \
 	$(CONFIG_DIR)/toolchain/packages-base.list \
@@ -222,7 +233,7 @@ endef
 
 define toolchain
         docker run \
-                --rm \
+        --rm \
 		--tty \
                 $(2) \
                 --env UID=$(UID) \
@@ -232,7 +243,6 @@ define toolchain
                 --cpus $(CPUS) \
                 --volume $(TOOLCHAIN_VOLUME) \
                 --workdir $(TOOLCHAIN_WORKDIR) \
-                --env-file=$(CONFIG_DIR)/global.env \
                 --env-file=$(CACHE_DIR_ROOT)/toolchain.env \
                 $(shell cat cache/toolchain.state 2> /dev/null) \
                 $(SRC_DIR)/toolchain/scripts/host-env bash -c $(1)
diff --git a/scripts/environment b/scripts/environment
index 2bb6cd2..ed939ce 100755
--- a/scripts/environment
+++ b/scripts/environment
@@ -1,6 +1,10 @@
 #!/bin/sh
 
 HOME=/home/build
+CONFIG_DIR=/home/build/config
+
+cat ${CONFIG_DIR}/toolchain.env
+
 cat <<- EOF
 	HOME=${HOME}
 	PATH=${HOME}/${BIN_DIR}:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
@@ -33,3 +37,5 @@ cat <<- EOF
 	BIN_DIR=${HOME}/${BIN_DIR}
 	FETCH_DIR=${HOME}/${FETCH_DIR}
 EOF
+
+envsubst < ${CONFIG_DIR}/global.env
diff --git a/scripts/packages-update b/scripts/packages-update
index fa0b4f4..8e7bac8 100755
--- a/scripts/packages-update
+++ b/scripts/packages-update
@@ -28,7 +28,7 @@ apt-get install -y --download-only --reinstall $( \
 apt-get install \
 	-y \
 	--download-only \
-	sudo \
+	sudo gettext \
 	$(cat /config/toolchain/packages-base.list)
 
 ( cd /var/cache/apt/archives \

From a8c0099576720fcf3c17a56238d87b21fa0cf089 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 8 Jun 2023 05:37:19 -0700
Subject: [PATCH 51/90] separate generated env files between container and make
 uses

---
 Makefile            | 29 ++++++++++++++++++++---------
 scripts/environment |  9 ++++++---
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/Makefile b/Makefile
index 5869bab..cc5e180 100644
--- a/Makefile
+++ b/Makefile
@@ -45,18 +45,28 @@ BIN_DIR := $(CACHE_DIR_ROOT)/bin
 SRC_DIR := src
 KEY_DIR := keys
 OUT_DIR := out
-
-export
-
 docker = docker
 
 include $(CONFIG_DIR)/toolchain.env
 export $(shell sed 's/=.*//' $(CONFIG_DIR)/toolchain.env)
+export
+
+AUTOBUILD_TOOLCHAIN := true
+ifeq ($(AUTOBUILD_TOOLCHAIN),true)
+ifeq ("$(wildcard $(CACHE_DIR_ROOT)/make.env)","")
+	echo := $(info $(shell echo "Initializing toolchain."))
+	build_env := $(shell $(MAKE) AUTOBUILD_TOOLCHAIN=false toolchain )
+endif
+endif
+ifneq (,$(wildcard $(CACHE_DIR_ROOT)/make.env))
+	include $(CACHE_DIR_ROOT)/make.env
+	export $(shell sed 's/=.*//' $(CACHE_DIR_ROOT)/make.env)
+endif
 
 ## Use env vars from existing release if present
 ifneq (,$(wildcard $(DIST_DIR)/release.env))
-    include $(DIST_DIR)/release.env
-    export
+	include $(DIST_DIR)/release.env
+	export
 endif
 
 executables = $(docker) git git-lfs patch
@@ -68,7 +78,8 @@ toolchain: \
 	$(BIN_DIR) \
 	$(OUT_DIR) \
 	$(CACHE_DIR_ROOT)/toolchain.state \
-	$(CACHE_DIR_ROOT)/toolchain.env
+	$(CACHE_DIR_ROOT)/container.env \
+	$(CACHE_DIR_ROOT)/make.env
 
 # Launch a shell inside the toolchain container
 .PHONY: toolchain-shell
@@ -129,7 +140,7 @@ $(FETCH_DIR):
 $(OUT_DIR):
 	mkdir -p $@
 
-$(CACHE_DIR_ROOT)/toolchain.env: \
+$(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \
 	$(CACHE_DIR_ROOT)/toolchain.state
 	env > $(CACHE_DIR)/bootstrap.env
 	docker run \
@@ -141,7 +152,7 @@ $(CACHE_DIR_ROOT)/toolchain.env: \
         --volume $(TOOLCHAIN_VOLUME) \
         --workdir $(TOOLCHAIN_WORKDIR) \
         $(shell cat cache/toolchain.state 2> /dev/null) \
-        $(SRC_DIR)/toolchain/scripts/environment > $@
+        $(SRC_DIR)/toolchain/scripts/environment $(CACHE_DIR_ROOT)
 	rm $(CACHE_DIR)/bootstrap.env
 
 $(CACHE_DIR_ROOT)/toolchain.tar: \
@@ -234,7 +245,7 @@ define toolchain
                 --cpus $(CPUS) \
                 --volume $(TOOLCHAIN_VOLUME) \
                 --workdir $(TOOLCHAIN_WORKDIR) \
-                --env-file=$(CACHE_DIR_ROOT)/toolchain.env \
+                --env-file=$(CACHE_DIR_ROOT)/container.env \
                 $(shell cat cache/toolchain.state 2> /dev/null) \
                 $(SRC_DIR)/toolchain/scripts/host-env bash -c $(1)
 endef
diff --git a/scripts/environment b/scripts/environment
index ed939ce..5f41ccc 100755
--- a/scripts/environment
+++ b/scripts/environment
@@ -1,11 +1,12 @@
 #!/bin/sh
 
+CACHE_DIR_ROOT=${1?}
 HOME=/home/build
 CONFIG_DIR=/home/build/config
 
-cat ${CONFIG_DIR}/toolchain.env
+cat ${CONFIG_DIR}/toolchain.env > ${CACHE_DIR_ROOT}/container.env
 
-cat <<- EOF
+cat <<- EOF >> ${CACHE_DIR_ROOT}/container.env
 	HOME=${HOME}
 	PATH=${HOME}/${BIN_DIR}:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 	TZ=UTC
@@ -38,4 +39,6 @@ cat <<- EOF
 	FETCH_DIR=${HOME}/${FETCH_DIR}
 EOF
 
-envsubst < ${CONFIG_DIR}/global.env
+envsubst < ${CONFIG_DIR}/global.env > ${CACHE_DIR_ROOT}/make.env
+
+cat ${CACHE_DIR_ROOT}/make.env >> ${CACHE_DIR_ROOT}/container.env

From 42f1d5850ce40fb6a86477f913a0fd03e555d00f Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 9 Jun 2023 16:08:03 -0700
Subject: [PATCH 52/90] fetch file support

---
 Makefile | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index cc5e180..8f9333b 100644
--- a/Makefile
+++ b/Makefile
@@ -35,7 +35,7 @@ VERSION := $(shell TZ=UTC0 git show --quiet --date='format-local:%Y.%m.%d' --for
 DIST_DIR := dist
 CONFIG_DIR := config
 CACHE_DIR_ROOT := cache
-FETCH_DIR := $(CACHE_DIR_ROOT)/fetch
+FETCH_DIR := fetch
 ifeq ($(TARGET),$(ARCH))
 	CACHE_DIR := $(CACHE_DIR_ROOT)/$(TARGET)
 else
@@ -141,6 +141,8 @@ $(OUT_DIR):
 	mkdir -p $@
 
 $(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \
+	$(CONFIG_DIR)/global.env \
+	$(CONFIG_DIR)/toolchain.env \
 	$(CACHE_DIR_ROOT)/toolchain.state
 	env > $(CACHE_DIR)/bootstrap.env
 	docker run \
@@ -189,6 +191,22 @@ $(OUT_DIR)/release.env: | $(OUT_DIR)
 check_executables := $(foreach exec,$(executables),\$(if \
 	$(shell which $(exec)),some string,$(error "No $(exec) in PATH")))
 
+define sha256_file
+$$(openssl sha256 $(1) | awk '{ print $$2}')
+endef
+
+define fetch_file
+	bash -c " \
+		echo \"Fetching $(1)\" \
+		&& curl \
+			--location $(1) \
+			--output $(CACHE_DIR)/$(notdir $@) \
+		&& [[ "\""$(call sha256_file,$(CACHE_DIR)/$(notdir $@))"\"" == "\""$(2)"\"" ]] \
+		|| { echo 'Error: Hash check failed'; exit 1; } \
+		&& mv $(CACHE_DIR)/$(notdir $@) $@; \
+	"
+endef
+
 define git_clone
 	[ -d $(1) ] || \
 		mkdir -p $(FETCH_DIR) && \

From c710181e040c5c7a621c63d8707dfffd9ccab693 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 15 Jun 2023 04:36:43 -0700
Subject: [PATCH 53/90] readme corrections

---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 4902cb5..1698432 100644
--- a/README.md
+++ b/README.md
@@ -62,10 +62,10 @@ us as desired.
 4. Lock a base Debian container image hash
 
     ```
-    echo "DEBIAN_HASH=48b28b354484a7f0e683e340fa0e6e4c4bce3dc3aa0146fc2f78f443fde2c55d" >> config/global.env
+    echo "DEBIAN_HASH=48b28b354484a7f0e683e340fa0e6e4c4bce3dc3aa0146fc2f78f443fde2c55d" >> config/toolchain.env
     ```
 
-5. Generate hashlocks files for all toolchain container dependencies
+5. Generate pinned hashes for all toolchain container dependencies
     ```
     make toolchain-update
     ```
@@ -82,16 +82,16 @@ us as desired.
 
 ## Usage ##
 
-### Build a new release
+### Build a new release with named version
 
 ```
-make VERSION=1.0.0rc1 release
+make VERSION=1.0.0rc1 dist
 ```
 
 ### Reproduce an existing release
 
 ```
-make VERSION=1.0.0rc1 attest
+make reproduce
 ```
 
 ### Add and lock a new container dependency

From 388334ef6afaa3a0df1bd6ecd14bb8a05d021ee0 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 15 Jun 2023 12:42:16 -0700
Subject: [PATCH 54/90] be explicit about env vars copied into containers

---
 Makefile | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 8f9333b..f5a9db5 100644
--- a/Makefile
+++ b/Makefile
@@ -144,18 +144,40 @@ $(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \
 	$(CONFIG_DIR)/global.env \
 	$(CONFIG_DIR)/toolchain.env \
 	$(CACHE_DIR_ROOT)/toolchain.state
-	env > $(CACHE_DIR)/bootstrap.env
 	docker run \
         --rm \
         --env UID=$(UID) \
         --env GID=$(GID) \
-        --env-file $(CACHE_DIR)/bootstrap.env \
+		--env NAME="$(NAME)" \
+		--env IMAGE="$(IMAGE)" \
+		--env USER="$(USER)" \
+		--env ARCH="$(ARCH)" \
+		--env HOST_ARCH="$(HOST_ARCH)" \
+		--env HOST_ARCH_ALT="$(HOST_ARCH_ALT)" \
+		--env HOST_OS="$(HOST_OS)" \
+		--env PLATFORM="$(PLATFORM)" \
+		--env CPUS="$(CPUS)" \
+		--env TARGET="$(TARGET)" \
+		--env GIT_REF="$(GIT_REF)" \
+		--env GIT_AUTHOR="$(GIT_AUTHOR)" \
+		--env GIT_KEY="$(GIT_KEY)" \
+		--env GIT_TIMESTAMP="$(GIT_TIMESTAMP)" \
+		--env VERSION="$(VERSION)" \
+		--env DIST_DIR="$(DIST_DIR)" \
+		--env FETCH_DIR="$(FETCH_DIR)" \
+		--env KEY_DIR="$(KEY_DIR)" \
+		--env OUT_DIR="$(OUT_DIR)" \
+		--env SRC_DIR="$(SRC_DIR)" \
+		--env CACHE_DIR="$(CACHE_DIR)" \
+		--env CACHE_DIR_ROOT="$(CACHE_DIR_ROOT)" \
+		--env CONFIG_DIR="$(CONFIG_DIR)" \
+		--env TOOLCHAIN_VOLUME="$(TOOLCHAIN_VOLUME)" \
+		--env TOOLCHAIN_WORKDIR="$(TOOLCHAIN_WORKDIR)" \
         --platform=linux/$(ARCH) \
         --volume $(TOOLCHAIN_VOLUME) \
         --workdir $(TOOLCHAIN_WORKDIR) \
         $(shell cat cache/toolchain.state 2> /dev/null) \
         $(SRC_DIR)/toolchain/scripts/environment $(CACHE_DIR_ROOT)
-	rm $(CACHE_DIR)/bootstrap.env
 
 $(CACHE_DIR_ROOT)/toolchain.tar: \
 	$(CONFIG_DIR)/toolchain.env \

From 5bbf26be78a05fe504c5aeb989d16bfa44383253 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 15 Jun 2023 23:23:10 -0700
Subject: [PATCH 55/90] cache debian artifacts in fetch directory

---
 Dockerfile               |  6 ++++--
 Makefile                 | 43 ++++++++++++++++++++++++++++++++++------
 scripts/packages-fetch   | 34 +++++++++++++++++++++++++++++++
 scripts/packages-install | 31 ++++++++---------------------
 scripts/packages-update  | 29 +++++++++++++++------------
 5 files changed, 99 insertions(+), 44 deletions(-)
 create mode 100755 scripts/packages-fetch

diff --git a/Dockerfile b/Dockerfile
index 021ea14..bbe3e4f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
 ARG DEBIAN_HASH
-FROM debian@sha256:${DEBIAN_HASH}
+FROM debian@sha256:${DEBIAN_HASH} as build-base
 
 ARG CONFIG_DIR
 ADD ${CONFIG_DIR} /config
@@ -7,4 +7,6 @@ ADD ${CONFIG_DIR} /config
 ARG SCRIPTS_DIR
 ADD ${SCRIPTS_DIR} /usr/local/bin
 
-RUN packages-install
+ARG FETCH_DIR
+RUN --mount=type=bind,source=fetch,target=/fetch,rw \
+    packages-install
diff --git a/Makefile b/Makefile
index f5a9db5..6c8740d 100644
--- a/Makefile
+++ b/Makefile
@@ -86,10 +86,21 @@ toolchain: \
 toolchain-shell: toolchain
 	$(call toolchain,bash --norc,--interactive)
 
-# Pin all packages in toolchain container to latest versions
 .PHONY: toolchain-update
 toolchain-update:
-	docker run \
+	rm \
+		$(CONFIG_DIR)/apt-pins-x86_64.list \
+		$(CONFIG_DIR)/apt-sources-x86_64.list \
+		$(CONFIG_DIR)/apt-hashes-x86_64.list
+	$(MAKE) $(CONFIG_DIR)/apt-hashes-x86_64.list \
+
+# Regenerate toolchain dependency packages to latest versions
+$(CONFIG_DIR)/apt-base.list \
+$(CONFIG_DIR)/apt-pins-x86_64.list \
+$(CONFIG_DIR)/apt-sources-x86_64.list \
+$(CONFIG_DIR)/apt-hashes-x86_64.list:
+	mkdir -p $(FETCH_DIR)/apt \
+	&& docker run \
 		--rm \
 		--tty \
 		--platform=linux/$(ARCH) \
@@ -102,6 +113,24 @@ toolchain-update:
 		debian@sha256:$(DEBIAN_HASH) \
 		/usr/local/bin/packages-update
 
+# Pin all packages in toolchain container to latest versions
+$(FETCH_DIR)/apt/Packages.gz:
+	docker run \
+		--rm \
+		--tty \
+		--platform=linux/$(ARCH) \
+		--env LOCAL_USER=$(UID):$(GID) \
+		--env FETCH_DIR="$(FETCH_DIR)" \
+		--env PACKAGES_LATEST=$(PACKAGES_LATEST) \
+		--volume $(PWD)/$(CONFIG_DIR):/config \
+		--volume $(PWD)/$(SRC_DIR)/toolchain/scripts:/usr/local/bin \
+		--volume $(PWD)/$(FETCH_DIR):/fetch \
+		--cpus $(CPUS) \
+		--volume $(TOOLCHAIN_VOLUME) \
+		--workdir $(TOOLCHAIN_WORKDIR) \
+		debian@sha256:$(DEBIAN_HASH) \
+		/usr/local/bin/packages-fetch
+
 .PHONY: toolchain-clean
 toolchain-clean:
 	if [ -d "$(CACHE_DIR_ROOT)" ]; then \
@@ -182,16 +211,18 @@ $(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \
 $(CACHE_DIR_ROOT)/toolchain.tar: \
 	$(CONFIG_DIR)/toolchain.env \
 	$(SRC_DIR)/toolchain/Dockerfile \
-	$(CONFIG_DIR)/toolchain/package-hashes-$(ARCH).txt \
-	$(CONFIG_DIR)/toolchain/packages-base.list \
-	$(CONFIG_DIR)/toolchain/packages-$(ARCH).list \
-	$(CONFIG_DIR)/toolchain/sources.list
+	$(CONFIG_DIR)/apt-base.list \
+	$(CONFIG_DIR)/apt-sources-$(ARCH).list \
+	$(CONFIG_DIR)/apt-pins-$(ARCH).list \
+	$(CONFIG_DIR)/apt-hashes-$(ARCH).list \
+	$(FETCH_DIR)/apt/Packages.gz
 	mkdir -p $(CACHE_DIR)
 	DOCKER_BUILDKIT=1 \
 	docker build \
 		--tag $(IMAGE) \
 		--build-arg DEBIAN_HASH=$(DEBIAN_HASH) \
 		--build-arg CONFIG_DIR=$(CONFIG_DIR) \
+		--build-arg FETCH_DIR=$(PWD)/$(FETCH_DIR) \
 		--build-arg SCRIPTS_DIR=$(SRC_DIR)/toolchain/scripts \
 		--platform=linux/$(ARCH) \
 		-f $(SRC_DIR)/toolchain/Dockerfile \
diff --git a/scripts/packages-fetch b/scripts/packages-fetch
new file mode 100755
index 0000000..9e15744
--- /dev/null
+++ b/scripts/packages-fetch
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+[ -f /.dockerenv ] || { echo "please run in supplied container"; exit 1; }
+set -e
+
+ARCH=$(uname -m)
+
+cp /config/* /etc/apt/
+apt update -o Acquire::Check-Valid-Until=false
+
+until apt-get install \
+    --download-only \
+    --allow-downgrades \
+    -o Acquire::Check-Valid-Until=false \
+    -y $(cat /etc/apt/apt-pins-${ARCH}.list);
+do
+    echo "apt install failed. Likely throttled. Retrying in 10 mins...";
+    sleep 600;
+done;
+
+(
+	cd /var/cache/apt/archives \
+		&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
+		| sed 's/.\///g' \
+		| LC_ALL=C sort
+) > /etc/apt/apt-hashes-${ARCH}-compare.list
+
+diff /etc/apt/apt-hashes-${ARCH}{,-compare}.list
+
+mkdir -p /fetch/apt
+
+mv /var/cache/apt/archives/*.deb /fetch/apt/
+apt-get install -y dpkg-dev
+env -C /fetch dpkg-scanpackages apt | gzip > /fetch/apt/Packages.gz
diff --git a/scripts/packages-install b/scripts/packages-install
index f09a1cd..bc1365f 100755
--- a/scripts/packages-install
+++ b/scripts/packages-install
@@ -3,34 +3,19 @@ set -e;
 
 ARCH=$(uname -m)
 
-cp /config/toolchain/* /etc/apt/
+cp /config/* /etc/apt/
 
-apt-get update -o Acquire::Check-Valid-Until=false
-apt-get install debian-archive-keyring
+cat <<-EOF > /etc/apt/sources.list
+deb [trusted=yes] file:///fetch apt/
+EOF
+rm /etc/apt/sources.list.d/*
 
-until apt-get install \
-    --download-only \
-    --reinstall \
-    --allow-downgrades \
-    -o Acquire::Check-Valid-Until=false \
-    -y $(cat /etc/apt/packages-${ARCH}.list);
-do
-    echo "apt install failed. Likely throttled. Retrying in 10 mins...";
-    sleep 600;
-done;
-
-(
-	cd /var/cache/apt/archives \
-		&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
-		| sed 's/.\///g' \
-		| LC_ALL=C sort
-) > /etc/apt/package-hashes-${ARCH}-compare.txt
-
-diff /etc/apt/package-hashes-${ARCH}{,-compare}.txt
+apt update -o Acquire::Check-Valid-Until=false
 
 apt-get install \
     --allow-downgrades \
-    -y $(cat /etc/apt/packages-${ARCH}.list)
+    -y $(cat /etc/apt/apt-pins-${ARCH}.list)
+
 rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* /var/tmp/*;
 
 echo "%sudo ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers
diff --git a/scripts/packages-update b/scripts/packages-update
index 6516a25..5d85b20 100755
--- a/scripts/packages-update
+++ b/scripts/packages-update
@@ -3,20 +3,17 @@
 [ -f /.dockerenv ] || { echo "please run in supplied container"; exit 1; }
 set -e
 
-snapshot_url="http://snapshot.debian.org/archive/debian"
-snapshot_date=$(date +"%Y%m%dT000000Z")
 cat <<-EOF > /etc/apt/sources.list
-deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm main
-deb [trusted=yes] ${snapshot_url}-security/${snapshot_date} bookworm-security main
-deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm-updates main
+deb http://deb.debian.org/debian bookworm main
+deb http://security.debian.org/debian-security bookworm-security main
+deb http://deb.debian.org/debian bookworm-updates main
 EOF
-cp /etc/apt/sources.list /config/toolchain/
 rm /etc/apt/sources.list.d/*
 
 ARCH=$(uname -m)
 
 apt-get update
-apt-get install -y --download-only --reinstall $( \
+apt-get install -y --download-only $( \
 	dpkg-query \
 		-W \
 		-f='${db:Status-Abbrev}\t${binary:Package} - ${binary:Summary}\n' \
@@ -26,20 +23,26 @@ apt-get install -y --download-only --reinstall $( \
 apt-get install \
 	-y \
 	--download-only \
-	sudo gettext \
-	$(cat /config/toolchain/packages-base.list)
+	sudo gettext dpkg-dev \
+	$(cat /config/apt-base.list)
 
 ( cd /var/cache/apt/archives \
 	&& find . -type f \( -iname \*.deb \) -exec sha256sum {} \; \
 	| sed 's/.\///g' \
 	| LC_ALL=C sort
-) > /config/toolchain/package-hashes-${ARCH}.txt
+) > /config/apt-hashes-${ARCH}.list
 
-cp /dev/null /config/toolchain/packages-${ARCH}.list
 for deb in /var/cache/apt/archives/*.deb; do
 	package=$(dpkg-deb -f $deb Package);
 	version=$(dpkg --info ${deb} | grep "^ Version: " | sed 's/^ Version: //g');
-	echo "${package}=${version}" >> /config/toolchain/packages-${ARCH}.list;
+	echo "${package}=${version}" >> /config/apt-pins-${ARCH}.list;
 done
 
-chown -R $LOCAL_USER /config/toolchain
+snapshot_url="http://snapshot.debian.org/archive/debian"
+snapshot_date=$(date +"%Y%m%dT000000Z")
+cat <<-EOF > /config/apt-sources-x86_64.list
+deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm main
+deb [trusted=yes] ${snapshot_url}-security/${snapshot_date} bookworm-security main
+deb [trusted=yes] ${snapshot_url}/${snapshot_date} bookworm-updates main
+EOF
+chown -R $LOCAL_USER /config/

From e1e679256912f718d7bbb104e89b114ec0fea05b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Se=C3=A1n=20C=20McCord?= <ulexus@gmail.com>
Date: Fri, 16 Jun 2023 11:56:43 -0700
Subject: [PATCH 56/90] Correct unsupported deb Packages compression, and copy
 all config files in scripts

---
 scripts/packages-fetch   | 4 ++--
 scripts/packages-install | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/packages-fetch b/scripts/packages-fetch
index 9e15744..362f1be 100755
--- a/scripts/packages-fetch
+++ b/scripts/packages-fetch
@@ -5,7 +5,7 @@ set -e
 
 ARCH=$(uname -m)
 
-cp /config/* /etc/apt/
+cp -R /config/* /etc/apt/
 apt update -o Acquire::Check-Valid-Until=false
 
 until apt-get install \
@@ -31,4 +31,4 @@ mkdir -p /fetch/apt
 
 mv /var/cache/apt/archives/*.deb /fetch/apt/
 apt-get install -y dpkg-dev
-env -C /fetch dpkg-scanpackages apt | gzip > /fetch/apt/Packages.gz
+env -C /fetch dpkg-scanpackages apt | bzip2 > /fetch/apt/Packages.bz2
diff --git a/scripts/packages-install b/scripts/packages-install
index bc1365f..02fec26 100755
--- a/scripts/packages-install
+++ b/scripts/packages-install
@@ -3,7 +3,7 @@ set -e;
 
 ARCH=$(uname -m)
 
-cp /config/* /etc/apt/
+cp -R /config/* /etc/apt/
 
 cat <<-EOF > /etc/apt/sources.list
 deb [trusted=yes] file:///fetch apt/

From c55daf1cdf7648e965a02374d37669c034e65b75 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 16 Jun 2023 12:11:11 -0700
Subject: [PATCH 57/90] Packages.gz -> Packages.bz2

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 6c8740d..1ed8374 100644
--- a/Makefile
+++ b/Makefile
@@ -114,7 +114,7 @@ $(CONFIG_DIR)/apt-hashes-x86_64.list:
 		/usr/local/bin/packages-update
 
 # Pin all packages in toolchain container to latest versions
-$(FETCH_DIR)/apt/Packages.gz:
+$(FETCH_DIR)/apt/Packages.bz2:
 	docker run \
 		--rm \
 		--tty \

From 489fcbabf34663563ba8b70d372680713059758d Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 16 Jun 2023 12:58:05 -0700
Subject: [PATCH 58/90] make dependecy and doc fixes

---
 Makefile  | 15 ++++++++++-----
 README.md |  6 +++---
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/Makefile b/Makefile
index 1ed8374..f574100 100644
--- a/Makefile
+++ b/Makefile
@@ -88,17 +88,22 @@ toolchain-shell: toolchain
 
 .PHONY: toolchain-update
 toolchain-update:
-	rm \
+	rm -rf \
 		$(CONFIG_DIR)/apt-pins-x86_64.list \
 		$(CONFIG_DIR)/apt-sources-x86_64.list \
-		$(CONFIG_DIR)/apt-hashes-x86_64.list
+		$(CONFIG_DIR)/apt-hashes-x86_64.list \
+		$(FETCH_DIR)/apt
 	$(MAKE) $(CONFIG_DIR)/apt-hashes-x86_64.list \
 
+
+$(CONFIG_DIR)/apt-base.list:
+	touch $(CONFIG_DIR)/apt-base.list
+
 # Regenerate toolchain dependency packages to latest versions
-$(CONFIG_DIR)/apt-base.list \
 $(CONFIG_DIR)/apt-pins-x86_64.list \
 $(CONFIG_DIR)/apt-sources-x86_64.list \
-$(CONFIG_DIR)/apt-hashes-x86_64.list:
+$(CONFIG_DIR)/apt-hashes-x86_64.list: \
+$(CONFIG_DIR)/apt-base.list
 	mkdir -p $(FETCH_DIR)/apt \
 	&& docker run \
 		--rm \
@@ -215,7 +220,7 @@ $(CACHE_DIR_ROOT)/toolchain.tar: \
 	$(CONFIG_DIR)/apt-sources-$(ARCH).list \
 	$(CONFIG_DIR)/apt-pins-$(ARCH).list \
 	$(CONFIG_DIR)/apt-hashes-$(ARCH).list \
-	$(FETCH_DIR)/apt/Packages.gz
+	$(FETCH_DIR)/apt/Packages.bz2
 	mkdir -p $(CACHE_DIR)
 	DOCKER_BUILDKIT=1 \
 	docker build \
diff --git a/README.md b/README.md
index 1698432..fd36b69 100644
--- a/README.md
+++ b/README.md
@@ -55,8 +55,8 @@ us as desired.
 3. Define any build/dev dependencies for toolchain container
 
     ```
-    echo "libfaketime" >> config/toolchain/packages-base.txt
-    echo "build-essential" >> config/toolchain/packages-base.txt
+    echo "libfaketime" >> config/apt-base.list
+    echo "build-essential" >> config/apt-base.list
     ```
 
 4. Lock a base Debian container image hash
@@ -97,7 +97,7 @@ make reproduce
 ### Add and lock a new container dependency
 
 ```
-echo "vim-nox" >> config/toolchain/packages-base.txt
+echo "vim-nox" >> config/apt-base.list
 make toolchain-update
 ```
 

From e75a319f87652567423c417a2cfc99d473e5637d Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sun, 18 Jun 2023 02:41:10 -0700
Subject: [PATCH 59/90] fix packages-update after autobuild

---
 Makefile               | 3 +--
 scripts/packages-fetch | 5 ++++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index f574100..509ba5c 100644
--- a/Makefile
+++ b/Makefile
@@ -93,8 +93,7 @@ toolchain-update:
 		$(CONFIG_DIR)/apt-sources-x86_64.list \
 		$(CONFIG_DIR)/apt-hashes-x86_64.list \
 		$(FETCH_DIR)/apt
-	$(MAKE) $(CONFIG_DIR)/apt-hashes-x86_64.list \
-
+	$(MAKE) AUTOBUILD_TOOLCHAIN=false $(CONFIG_DIR)/apt-hashes-x86_64.list
 
 $(CONFIG_DIR)/apt-base.list:
 	touch $(CONFIG_DIR)/apt-base.list
diff --git a/scripts/packages-fetch b/scripts/packages-fetch
index 362f1be..70b0560 100755
--- a/scripts/packages-fetch
+++ b/scripts/packages-fetch
@@ -5,7 +5,10 @@ set -e
 
 ARCH=$(uname -m)
 
-cp -R /config/* /etc/apt/
+cp /config/apt-sources-x86_64.list /etc/apt/sources.list
+cp /config/apt-hashes-x86_64.list /etc/apt/
+cp /config/apt-pins-x86_64.list /etc/apt/
+rm /etc/apt/sources.list.d/*
 apt update -o Acquire::Check-Valid-Until=false
 
 until apt-get install \

From 78376e209439cbb1b73994a9790cd2f84d9fddc3 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sun, 18 Jun 2023 03:21:57 -0700
Subject: [PATCH 60/90] include BIN_DIR in environment files

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 509ba5c..cca6e9f 100644
--- a/Makefile
+++ b/Makefile
@@ -199,6 +199,7 @@ $(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \
 		--env DIST_DIR="$(DIST_DIR)" \
 		--env FETCH_DIR="$(FETCH_DIR)" \
 		--env KEY_DIR="$(KEY_DIR)" \
+		--env BIN_DIR="$(BIN_DIR)" \
 		--env OUT_DIR="$(OUT_DIR)" \
 		--env SRC_DIR="$(SRC_DIR)" \
 		--env CACHE_DIR="$(CACHE_DIR)" \

From cb72be053fa02a5873fc57cf151178f043e3eeb4 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sun, 18 Jun 2023 15:04:53 -0700
Subject: [PATCH 61/90] remove cache/make.env recursion insanity. It breaks too
 many things

---
 Makefile            | 29 ++++++++---------------------
 scripts/environment | 11 +++++------
 2 files changed, 13 insertions(+), 27 deletions(-)

diff --git a/Makefile b/Makefile
index cca6e9f..d91fb04 100644
--- a/Makefile
+++ b/Makefile
@@ -47,21 +47,10 @@ KEY_DIR := keys
 OUT_DIR := out
 docker = docker
 
-include $(CONFIG_DIR)/toolchain.env
-export $(shell sed 's/=.*//' $(CONFIG_DIR)/toolchain.env)
 export
 
-AUTOBUILD_TOOLCHAIN := true
-ifeq ($(AUTOBUILD_TOOLCHAIN),true)
-ifeq ("$(wildcard $(CACHE_DIR_ROOT)/make.env)","")
-	echo := $(info $(shell echo "Initializing toolchain."))
-	build_env := $(shell $(MAKE) AUTOBUILD_TOOLCHAIN=false toolchain )
-endif
-endif
-ifneq (,$(wildcard $(CACHE_DIR_ROOT)/make.env))
-	include $(CACHE_DIR_ROOT)/make.env
-	export $(shell sed 's/=.*//' $(CACHE_DIR_ROOT)/make.env)
-endif
+include $(CONFIG_DIR)/make.env
+export $(shell sed 's/=.*//' $(CONFIG_DIR)/make.env)
 
 ## Use env vars from existing release if present
 ifneq (,$(wildcard $(DIST_DIR)/release.env))
@@ -78,8 +67,7 @@ toolchain: \
 	$(BIN_DIR) \
 	$(OUT_DIR) \
 	$(CACHE_DIR_ROOT)/toolchain.state \
-	$(CACHE_DIR_ROOT)/container.env \
-	$(CACHE_DIR_ROOT)/make.env
+	$(CACHE_DIR_ROOT)/container.env
 
 # Launch a shell inside the toolchain container
 .PHONY: toolchain-shell
@@ -93,7 +81,7 @@ toolchain-update:
 		$(CONFIG_DIR)/apt-sources-x86_64.list \
 		$(CONFIG_DIR)/apt-hashes-x86_64.list \
 		$(FETCH_DIR)/apt
-	$(MAKE) AUTOBUILD_TOOLCHAIN=false $(CONFIG_DIR)/apt-hashes-x86_64.list
+	$(MAKE) $(CONFIG_DIR)/apt-hashes-x86_64.list
 
 $(CONFIG_DIR)/apt-base.list:
 	touch $(CONFIG_DIR)/apt-base.list
@@ -173,9 +161,8 @@ $(FETCH_DIR):
 $(OUT_DIR):
 	mkdir -p $@
 
-$(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \
-	$(CONFIG_DIR)/global.env \
-	$(CONFIG_DIR)/toolchain.env \
+$(CACHE_DIR_ROOT)/container.env: \
+	$(CONFIG_DIR)/make.env \
 	$(CACHE_DIR_ROOT)/toolchain.state
 	docker run \
         --rm \
@@ -211,10 +198,10 @@ $(CACHE_DIR_ROOT)/make.env $(CACHE_DIR_ROOT)/container.env: \
         --volume $(TOOLCHAIN_VOLUME) \
         --workdir $(TOOLCHAIN_WORKDIR) \
         $(shell cat cache/toolchain.state 2> /dev/null) \
-        $(SRC_DIR)/toolchain/scripts/environment $(CACHE_DIR_ROOT)
+        $(SRC_DIR)/toolchain/scripts/environment > $@
 
 $(CACHE_DIR_ROOT)/toolchain.tar: \
-	$(CONFIG_DIR)/toolchain.env \
+	$(CONFIG_DIR)/make.env \
 	$(SRC_DIR)/toolchain/Dockerfile \
 	$(CONFIG_DIR)/apt-base.list \
 	$(CONFIG_DIR)/apt-sources-$(ARCH).list \
diff --git a/scripts/environment b/scripts/environment
index 5f41ccc..d4a8e23 100755
--- a/scripts/environment
+++ b/scripts/environment
@@ -1,12 +1,11 @@
 #!/bin/sh
 
-CACHE_DIR_ROOT=${1?}
 HOME=/home/build
 CONFIG_DIR=/home/build/config
 
-cat ${CONFIG_DIR}/toolchain.env > ${CACHE_DIR_ROOT}/container.env
+cat ${CONFIG_DIR}/make.env
 
-cat <<- EOF >> ${CACHE_DIR_ROOT}/container.env
+cat <<- EOF
 	HOME=${HOME}
 	PATH=${HOME}/${BIN_DIR}:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 	TZ=UTC
@@ -39,6 +38,6 @@ cat <<- EOF >> ${CACHE_DIR_ROOT}/container.env
 	FETCH_DIR=${HOME}/${FETCH_DIR}
 EOF
 
-envsubst < ${CONFIG_DIR}/global.env > ${CACHE_DIR_ROOT}/make.env
-
-cat ${CACHE_DIR_ROOT}/make.env >> ${CACHE_DIR_ROOT}/container.env
+if [ -f "${CONFIG_DIR}/container.env" ]; then
+    envsubst < ${CONFIG_DIR}/container.env
+fi

From 88527fc42793467b60b9a09eee0b1792d696cb6f Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Wed, 21 Jun 2023 18:28:27 -0700
Subject: [PATCH 62/90] new KEY_DIR under fetch

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index d91fb04..9305ec8 100644
--- a/Makefile
+++ b/Makefile
@@ -43,7 +43,7 @@ else
 endif
 BIN_DIR := $(CACHE_DIR_ROOT)/bin
 SRC_DIR := src
-KEY_DIR := keys
+KEY_DIR := fetch/keys
 OUT_DIR := out
 docker = docker
 
@@ -292,7 +292,7 @@ define fetch_pgp_key
         	                --recv-keys "$(1)" \
         	        && break; \
         	done; \
-            gpg --export -a $(1) > $(KEY_DIR)/$(1).asc; \
+            gpg --export -a $(1) > $@; \
         ')
 endef
 

From 09c3a79d8c7f1cbe42858266db400571bab5defc Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 22 Jun 2023 02:50:56 -0700
Subject: [PATCH 63/90] add git archive function

---
 Makefile | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 9305ec8..72f1af7 100644
--- a/Makefile
+++ b/Makefile
@@ -252,18 +252,33 @@ define fetch_file
 	"
 endef
 
+define git_archive
+		$(call git_clone,$(CACHE_DIR)/$(notdir $@),$(1),$(2)) \
+		&& tar \
+				-C $(CACHE_DIR)/$(notdir $@) \
+				--sort=name \
+				--mtime='@0' \
+				--owner=0 \
+				--group=0 \
+				--numeric-owner \
+				-cvf - \
+				. \
+			| gzip -n > $@ \
+		&& rm -rf $(CACHE_DIR)/$(notdir $@)
+endef
+
 define git_clone
 	[ -d $(1) ] || \
-		mkdir -p $(FETCH_DIR) && \
-		mkdir $(1).tmp && \
+		mkdir -p $(1).tmp && \
 		git -C $(1).tmp init && \
 		git -C $(1).tmp remote add origin $(2) && \
 		git -C $(1).tmp fetch origin $(3) && \
 		git -C $(1).tmp -c advice.detachedHead=false checkout $(3) && \
+		git -C $(1).tmp submodule update --init && \
 		git -C $(1).tmp rev-parse --verify HEAD | grep -q $(3) || { \
 			echo 'Error: Git ref/branch collision.'; exit 1; \
 		} && \
-		mv $(1).tmp $(1);
+		mv $(1).tmp $(1)
 endef
 
 define apply_patches

From 352783aa6563a8c104f02905e38be8e0e6da30c8 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 30 Jun 2023 12:55:08 -0700
Subject: [PATCH 64/90] regen Packages.gz when apt hashes change

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 72f1af7..3244be3 100644
--- a/Makefile
+++ b/Makefile
@@ -106,7 +106,7 @@ $(CONFIG_DIR)/apt-base.list
 		/usr/local/bin/packages-update
 
 # Pin all packages in toolchain container to latest versions
-$(FETCH_DIR)/apt/Packages.bz2:
+$(FETCH_DIR)/apt/Packages.bz2: $(CONFIG_DIR)/apt-hashes-x86_64.list
 	docker run \
 		--rm \
 		--tty \

From 2560e543a1871b86cb1c29b31ce9141eaec94e8c Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 20 Jul 2023 18:24:42 -0700
Subject: [PATCH 65/90] support path prefix

---
 Makefile         | 1 +
 scripts/host-env | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/Makefile b/Makefile
index 3244be3..398175a 100644
--- a/Makefile
+++ b/Makefile
@@ -318,6 +318,7 @@ define toolchain
                 $(2) \
                 --env UID=$(UID) \
                 --env GID=$(GID) \
+				--env PATH_PREFIX=$(PATH_PREFIX) \
                 --platform=linux/$(ARCH) \
                 --privileged \
                 --cpus $(CPUS) \
diff --git a/scripts/host-env b/scripts/host-env
index 159136d..0464734 100755
--- a/scripts/host-env
+++ b/scripts/host-env
@@ -5,6 +5,8 @@ uid=${UID?}
 gid=${GID?}
 user=${USER:-"build"}
 export HOME="/home/${user}"
+[ ! -z "$PATH_PREFIX" ] && \
+    export PATH="${PATH_PREFIX}:${PATH}"
 
 # If running user is not root, pivot to custom user/group
 if [ "$uid" != "0" ]; then

From d3f66385c237d9e6edf4cf4c0106e6d7a0b3f0a6 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Mon, 7 Aug 2023 14:33:01 -0700
Subject: [PATCH 66/90] make Packages.bz2 an order-only to allow deletion

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 398175a..5ecab20 100644
--- a/Makefile
+++ b/Makefile
@@ -207,7 +207,7 @@ $(CACHE_DIR_ROOT)/toolchain.tar: \
 	$(CONFIG_DIR)/apt-sources-$(ARCH).list \
 	$(CONFIG_DIR)/apt-pins-$(ARCH).list \
 	$(CONFIG_DIR)/apt-hashes-$(ARCH).list \
-	$(FETCH_DIR)/apt/Packages.bz2
+	| $(FETCH_DIR)/apt/Packages.bz2
 	mkdir -p $(CACHE_DIR)
 	DOCKER_BUILDKIT=1 \
 	docker build \

From 9ddcbe18e5224d9593c1f840ec2d647ef3953ff2 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sun, 3 Sep 2023 02:46:55 -0700
Subject: [PATCH 67/90] more default vars

---
 Makefile | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 5ecab20..73db0b7 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,6 @@ altarch = $(subst x86_64,amd64,$(subst aarch64,arm64,$1))
 
 TOOLCHAIN_VOLUME := $(PWD):/home/build
 TOOLCHAIN_WORKDIR := /home/build
-
 DEFAULT_GOAL := $(or $(DEFAULT_GOAL),toolchain)
 ARCH := $(or $(ARCH),x86_64)
 TARGET := $(or $(TARGET),$(ARCH))
@@ -20,6 +19,7 @@ UID := $(shell id -u)
 GID := $(shell id -g)
 USER := $(UID):$(GID)
 CPUS := $(shell docker run debian nproc)
+ARCHIVE_SOURCES := true
 PRESERVE_CACHE := "false"
 GIT_REF := $(shell git log -1 --format=%H)
 GIT_AUTHOR := $(shell git log -1 --format=%an)
@@ -47,6 +47,10 @@ KEY_DIR := fetch/keys
 OUT_DIR := out
 docker = docker
 
+PATH_PREFIX := /home/build/$(CACHE_DIR)/bin:/home/build/$(OUT_DIR)/linux/x86_64
+PREFIX := $(HOME)/.local/bin
+XDG_CONFIG_HOME := $(HOME)/.config
+
 export
 
 include $(CONFIG_DIR)/make.env
@@ -113,7 +117,7 @@ $(FETCH_DIR)/apt/Packages.bz2: $(CONFIG_DIR)/apt-hashes-x86_64.list
 		--platform=linux/$(ARCH) \
 		--env LOCAL_USER=$(UID):$(GID) \
 		--env FETCH_DIR="$(FETCH_DIR)" \
-		--env PACKAGES_LATEST=$(PACKAGES_LATEST) \
+		--env ARCHIVE_SOURCES=$(ARCHIVE_SOURCES) \
 		--volume $(PWD)/$(CONFIG_DIR):/config \
 		--volume $(PWD)/$(SRC_DIR)/toolchain/scripts:/usr/local/bin \
 		--volume $(PWD)/$(FETCH_DIR):/fetch \

From fda01343a53bd7f481b8e995cb5bc931d8364abb Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sun, 3 Sep 2023 02:47:17 -0700
Subject: [PATCH 68/90] allow bypassing use of slow/unreliable archive apt
 sources. Useful when using own LFS cache or similar

---
 scripts/packages-fetch | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scripts/packages-fetch b/scripts/packages-fetch
index 70b0560..69bbe56 100755
--- a/scripts/packages-fetch
+++ b/scripts/packages-fetch
@@ -5,10 +5,14 @@ set -e
 
 ARCH=$(uname -m)
 
-cp /config/apt-sources-x86_64.list /etc/apt/sources.list
+echo ARCHIVE_SOURCES -> $ARCHIVE_SOURCES
+if [[ "${ARCHIVE_SOURCES}" == "true" ]]; then
+	rm /etc/apt/sources.list.d/*
+	cp /config/apt-sources-x86_64.list /etc/apt/sources.list
+fi
+
 cp /config/apt-hashes-x86_64.list /etc/apt/
 cp /config/apt-pins-x86_64.list /etc/apt/
-rm /etc/apt/sources.list.d/*
 apt update -o Acquire::Check-Valid-Until=false
 
 until apt-get install \

From 23fc267a9dfdda30ba4287f8234879961722bafb Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sun, 3 Sep 2023 03:29:28 -0700
Subject: [PATCH 69/90] fix default paths

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 73db0b7..7d2d718 100644
--- a/Makefile
+++ b/Makefile
@@ -47,8 +47,8 @@ KEY_DIR := fetch/keys
 OUT_DIR := out
 docker = docker
 
-PATH_PREFIX := /home/build/$(CACHE_DIR)/bin:/home/build/$(OUT_DIR)/linux/x86_64
-PREFIX := $(HOME)/.local/bin
+PATH_PREFIX := /home/build/.local/bin:/home/build/$(CACHE_DIR)/bin:/home/build/$(OUT_DIR)/linux/x86_64
+PREFIX := $(HOME)/.local
 XDG_CONFIG_HOME := $(HOME)/.config
 
 export

From b9894265d06618e5c94e17ab03885402db372258 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 8 Sep 2023 18:35:33 -0700
Subject: [PATCH 70/90] build toolchain on demand, require clean tree for dist

---
 Makefile | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 7d2d718..f1d32fa 100644
--- a/Makefile
+++ b/Makefile
@@ -148,6 +148,10 @@ reproduce: toolchain-clean
 
 .PHONY: $(DIST_DIR)
 $(DIST_DIR):
+	git ls-files -o --exclude-standard | grep . \
+		&& { echo "Error: Git has untracked files present"; exit 1; } || :
+	git diff --name-only | grep . \
+		&& { echo "Error: Git has unstaged changes present"; exit 1; } || :
 	rm -rf $@/*
 	[ "$(PRESERVE_CACHE)" = "true" ] || $(MAKE) toolchain-clean
 	$(MAKE) default
@@ -316,9 +320,10 @@ define fetch_pgp_key
 endef
 
 define toolchain
-        docker run \
-        --rm \
-		--tty \
+        $(MAKE) toolchain \
+        && docker run \
+                --rm \
+                --tty \
                 $(2) \
                 --env UID=$(UID) \
                 --env GID=$(GID) \
@@ -329,7 +334,7 @@ define toolchain
                 --volume $(TOOLCHAIN_VOLUME) \
                 --workdir $(TOOLCHAIN_WORKDIR) \
                 --env-file=$(CACHE_DIR_ROOT)/container.env \
-                $(shell cat cache/toolchain.state 2> /dev/null) \
+                $$(cat $(CACHE_DIR_ROOT)/toolchain.state 2> /dev/null) \
                 $(SRC_DIR)/toolchain/scripts/host-env bash -c $(1)
 endef
 

From f966d23708dd56bdaf1608641ef9bdf2443abc82 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sat, 9 Sep 2023 00:56:31 -0700
Subject: [PATCH 71/90] allow re-use of old unchanged dist artifacts

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index f1d32fa..aacfaa4 100644
--- a/Makefile
+++ b/Makefile
@@ -152,10 +152,10 @@ $(DIST_DIR):
 		&& { echo "Error: Git has untracked files present"; exit 1; } || :
 	git diff --name-only | grep . \
 		&& { echo "Error: Git has unstaged changes present"; exit 1; } || :
-	rm -rf $@/*
+	cp -Rp $@/* $(OUT_DIR)/
 	[ "$(PRESERVE_CACHE)" = "true" ] || $(MAKE) toolchain-clean
 	$(MAKE) default
-	cp -R $(OUT_DIR)/* $@/
+	cp -Rp $(OUT_DIR)/* $@/
 
 $(BIN_DIR):
 	mkdir -p $@

From a5ab6f8dcd3d25aba8973664467cd4fa5a2c7d98 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sat, 9 Sep 2023 01:01:36 -0700
Subject: [PATCH 72/90] allow each project to choose if cache is reused

---
 Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Makefile b/Makefile
index aacfaa4..e82feef 100644
--- a/Makefile
+++ b/Makefile
@@ -152,7 +152,6 @@ $(DIST_DIR):
 		&& { echo "Error: Git has untracked files present"; exit 1; } || :
 	git diff --name-only | grep . \
 		&& { echo "Error: Git has unstaged changes present"; exit 1; } || :
-	cp -Rp $@/* $(OUT_DIR)/
 	[ "$(PRESERVE_CACHE)" = "true" ] || $(MAKE) toolchain-clean
 	$(MAKE) default
 	cp -Rp $(OUT_DIR)/* $@/

From ec9175bf4546263c83c5e24be9acddcac148e62c Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Mon, 18 Sep 2023 17:38:31 -0700
Subject: [PATCH 73/90] always regen release.env unless REPRODUCE=true

---
 Makefile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index e82feef..aa12f75 100644
--- a/Makefile
+++ b/Makefile
@@ -57,7 +57,7 @@ include $(CONFIG_DIR)/make.env
 export $(shell sed 's/=.*//' $(CONFIG_DIR)/make.env)
 
 ## Use env vars from existing release if present
-ifneq (,$(wildcard $(DIST_DIR)/release.env))
+ifeq ($(REPRODUCE),"true")
 	include $(DIST_DIR)/release.env
 	export
 endif
@@ -141,8 +141,7 @@ toolchain-clean:
 .PHONY: reproduce
 reproduce: toolchain-clean
 	mkdir -p $(OUT_DIR)
-	cp $(DIST_DIR)/release.env $(OUT_DIR)/release.env
-	$(MAKE) TARGET=$(TARGET) VERSION=$(VERSION)
+	$(MAKE) REPRODUCE="true"
 	diff -q $(OUT_DIR) $(DIST_DIR) \
 	&& echo "Success: $(OUT_DIR) and $(DIST_DIR) are identical"
 
@@ -233,7 +232,7 @@ $(CACHE_DIR_ROOT)/toolchain.state: \
 	docker load -i $(CACHE_DIR_ROOT)/toolchain.tar
 	docker images --no-trunc --quiet $(IMAGE) > $@
 
-$(OUT_DIR)/release.env: | $(OUT_DIR)
+$(OUT_DIR)/release.env: $(shell git ls-files)
 	echo 'VERSION=$(VERSION)'              > $(OUT_DIR)/release.env
 	echo 'GIT_REF=$(GIT_REF)'             >> $(OUT_DIR)/release.env
 	echo 'GIT_AUTHOR=$(GIT_AUTHOR)'       >> $(OUT_DIR)/release.env

From d8ce33f628b80e2099fa8586ac68841974d06f1a Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 21 Sep 2023 10:55:57 -0700
Subject: [PATCH 74/90] namespace toolchain commands, add dist caching and
 git-restore-mtime support

---
 Makefile                | 23 +++++++++++++++++------
 scripts/packages-update |  2 +-
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index aa12f75..9c3b7cb 100644
--- a/Makefile
+++ b/Makefile
@@ -57,7 +57,7 @@ include $(CONFIG_DIR)/make.env
 export $(shell sed 's/=.*//' $(CONFIG_DIR)/make.env)
 
 ## Use env vars from existing release if present
-ifeq ($(REPRODUCE),"true")
+ifeq ($(TOOLCHAIN_REPRODUCE),"true")
 	include $(DIST_DIR)/release.env
 	export
 endif
@@ -87,6 +87,17 @@ toolchain-update:
 		$(FETCH_DIR)/apt
 	$(MAKE) $(CONFIG_DIR)/apt-hashes-x86_64.list
 
+.PHONY: toolchain-restore-mtime
+toolchain-restore-mtime:
+	$(call toolchain," \
+		git restore-mtime \
+		&& echo "Git mtime restored" \
+	")
+
+.PHONY: toolchain-dist-cache
+toolchain-dist-cache:
+	cp -Rp $(DIST_DIR)/* $(OUT_DIR)/
+
 $(CONFIG_DIR)/apt-base.list:
 	touch $(CONFIG_DIR)/apt-base.list
 
@@ -138,15 +149,15 @@ toolchain-clean:
 	fi
 	docker image rm -f $(IMAGE) || :
 
-.PHONY: reproduce
-reproduce: toolchain-clean
+.PHONY: toolchain-reproduce
+toolchain-reproduce: toolchain-clean
 	mkdir -p $(OUT_DIR)
-	$(MAKE) REPRODUCE="true"
+	$(MAKE) TOOLCHAIN_REPRODUCE="true"
 	diff -q $(OUT_DIR) $(DIST_DIR) \
 	&& echo "Success: $(OUT_DIR) and $(DIST_DIR) are identical"
 
-.PHONY: $(DIST_DIR)
-$(DIST_DIR):
+.PHONY: toolchain-dist
+toolchain-dist: toolchain-restore-mtime toolchain-dist-cache
 	git ls-files -o --exclude-standard | grep . \
 		&& { echo "Error: Git has untracked files present"; exit 1; } || :
 	git diff --name-only | grep . \
diff --git a/scripts/packages-update b/scripts/packages-update
index 5d85b20..3d0c3c2 100755
--- a/scripts/packages-update
+++ b/scripts/packages-update
@@ -23,7 +23,7 @@ apt-get install -y --download-only $( \
 apt-get install \
 	-y \
 	--download-only \
-	sudo gettext dpkg-dev \
+	sudo gettext dpkg-dev git-restore-mtime \
 	$(cat /config/apt-base.list)
 
 ( cd /var/cache/apt/archives \

From 985107c4d2dd2d2ca18446a17c8d762212279466 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 21 Sep 2023 15:50:23 -0700
Subject: [PATCH 75/90] fix dist dir name

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 9c3b7cb..0cd6f7d 100644
--- a/Makefile
+++ b/Makefile
@@ -164,7 +164,7 @@ toolchain-dist: toolchain-restore-mtime toolchain-dist-cache
 		&& { echo "Error: Git has unstaged changes present"; exit 1; } || :
 	[ "$(PRESERVE_CACHE)" = "true" ] || $(MAKE) toolchain-clean
 	$(MAKE) default
-	cp -Rp $(OUT_DIR)/* $@/
+	cp -Rp $(OUT_DIR)/* $(DIST_DIR)/
 
 $(BIN_DIR):
 	mkdir -p $@

From 52811cee5fa2cf467d498e13cf2f2372e2962e66 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 21 Sep 2023 16:00:21 -0700
Subject: [PATCH 76/90] fix TOOLCHAIN_REPRODUCE match

---
 Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 0cd6f7d..7f312ac 100644
--- a/Makefile
+++ b/Makefile
@@ -57,9 +57,9 @@ include $(CONFIG_DIR)/make.env
 export $(shell sed 's/=.*//' $(CONFIG_DIR)/make.env)
 
 ## Use env vars from existing release if present
-ifeq ($(TOOLCHAIN_REPRODUCE),"true")
-	include $(DIST_DIR)/release.env
-	export
+ifeq ($(TOOLCHAIN_REPRODUCE),true)
+include $(DIST_DIR)/release.env
+export
 endif
 
 executables = $(docker) git git-lfs patch

From 206148838d7382da2e5c7b7684b6ea6ef21f2c70 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sun, 24 Sep 2023 05:17:39 -0700
Subject: [PATCH 77/90] create out if it does not exist

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 7f312ac..aa153b3 100644
--- a/Makefile
+++ b/Makefile
@@ -96,6 +96,7 @@ toolchain-restore-mtime:
 
 .PHONY: toolchain-dist-cache
 toolchain-dist-cache:
+	mkdir -p $(OUT_DIR)
 	cp -Rp $(DIST_DIR)/* $(OUT_DIR)/
 
 $(CONFIG_DIR)/apt-base.list:

From 7d2bc3d6f8308e7606b0da9e655c4f1451b1c21c Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Sun, 24 Sep 2023 22:47:26 -0700
Subject: [PATCH 78/90] drop undocumented/unused PRESERVE_CACHE

---
 Makefile | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index aa153b3..3356d62 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,6 @@ GID := $(shell id -g)
 USER := $(UID):$(GID)
 CPUS := $(shell docker run debian nproc)
 ARCHIVE_SOURCES := true
-PRESERVE_CACHE := "false"
 GIT_REF := $(shell git log -1 --format=%H)
 GIT_AUTHOR := $(shell git log -1 --format=%an)
 GIT_KEY := $(shell git log -1 --format=%GP)
@@ -158,12 +157,11 @@ toolchain-reproduce: toolchain-clean
 	&& echo "Success: $(OUT_DIR) and $(DIST_DIR) are identical"
 
 .PHONY: toolchain-dist
-toolchain-dist: toolchain-restore-mtime toolchain-dist-cache
+toolchain-dist: toolchain-clean toolchain-restore-mtime toolchain-dist-cache
 	git ls-files -o --exclude-standard | grep . \
 		&& { echo "Error: Git has untracked files present"; exit 1; } || :
 	git diff --name-only | grep . \
 		&& { echo "Error: Git has unstaged changes present"; exit 1; } || :
-	[ "$(PRESERVE_CACHE)" = "true" ] || $(MAKE) toolchain-clean
 	$(MAKE) default
 	cp -Rp $(OUT_DIR)/* $(DIST_DIR)/
 

From bba9553c2f4f3d6bd461637a414aeccdc458bc5b Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Wed, 27 Sep 2023 14:49:17 -0700
Subject: [PATCH 79/90] save/restore toolchain image as tgz for up to 3x disk
 savings

---
 Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 3356d62..2dca072 100644
--- a/Makefile
+++ b/Makefile
@@ -216,7 +216,7 @@ $(CACHE_DIR_ROOT)/container.env: \
         $(shell cat cache/toolchain.state 2> /dev/null) \
         $(SRC_DIR)/toolchain/scripts/environment > $@
 
-$(CACHE_DIR_ROOT)/toolchain.tar: \
+$(CACHE_DIR_ROOT)/toolchain.tgz: \
 	$(CONFIG_DIR)/make.env \
 	$(SRC_DIR)/toolchain/Dockerfile \
 	$(CONFIG_DIR)/apt-base.list \
@@ -235,11 +235,11 @@ $(CACHE_DIR_ROOT)/toolchain.tar: \
 		--platform=linux/$(ARCH) \
 		-f $(SRC_DIR)/toolchain/Dockerfile \
 		.
-	docker save "$(IMAGE)" -o "$@"
+	docker save "$(IMAGE)" | gzip > "$@"
 
 $(CACHE_DIR_ROOT)/toolchain.state: \
-	$(CACHE_DIR_ROOT)/toolchain.tar
-	docker load -i $(CACHE_DIR_ROOT)/toolchain.tar
+	$(CACHE_DIR_ROOT)/toolchain.tgz
+	docker load -i $(CACHE_DIR_ROOT)/toolchain.tgz
 	docker images --no-trunc --quiet $(IMAGE) > $@
 
 $(OUT_DIR)/release.env: $(shell git ls-files)

From f4a93830c536fb6d7586746ab88215e405a50e9d Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Wed, 27 Sep 2023 17:22:01 -0700
Subject: [PATCH 80/90] use hash of cache directory as toolchain tag

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2dca072..663ce7c 100644
--- a/Makefile
+++ b/Makefile
@@ -14,7 +14,6 @@ HOST_ARCH_ALT := $(call altarch,$(HOST_ARCH))
 HOST_OS := $(call lc,$(shell uname -s))
 PLATFORM := $(or $(PLATFORM),linux)
 NAME := $(shell basename $(shell git rev-parse --show-toplevel | tr A-Z a-z ))
-IMAGE := local/$(NAME)
 UID := $(shell id -u)
 GID := $(shell id -g)
 USER := $(UID):$(GID)
@@ -44,6 +43,7 @@ BIN_DIR := $(CACHE_DIR_ROOT)/bin
 SRC_DIR := src
 KEY_DIR := fetch/keys
 OUT_DIR := out
+IMAGE := toolchain/$(shell git ls-files -s $(CONFIG_DIR) | git hash-object --stdin)
 docker = docker
 
 PATH_PREFIX := /home/build/.local/bin:/home/build/$(CACHE_DIR)/bin:/home/build/$(OUT_DIR)/linux/x86_64

From 86e5600952d1249b3a8d61912d1a3d8735892bd5 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Wed, 27 Sep 2023 19:22:08 -0700
Subject: [PATCH 81/90] ensure default is run before any toolchain commands in
 dist, to allow user to optionally restore toolchain.tgz cache

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 663ce7c..6c52807 100644
--- a/Makefile
+++ b/Makefile
@@ -157,12 +157,12 @@ toolchain-reproduce: toolchain-clean
 	&& echo "Success: $(OUT_DIR) and $(DIST_DIR) are identical"
 
 .PHONY: toolchain-dist
-toolchain-dist: toolchain-clean toolchain-restore-mtime toolchain-dist-cache
+toolchain-dist:
 	git ls-files -o --exclude-standard | grep . \
 		&& { echo "Error: Git has untracked files present"; exit 1; } || :
 	git diff --name-only | grep . \
 		&& { echo "Error: Git has unstaged changes present"; exit 1; } || :
-	$(MAKE) default
+	$(MAKE) toolchain-clean default toolchain-restore-mtime toolchain-dist-cache
 	cp -Rp $(OUT_DIR)/* $(DIST_DIR)/
 
 $(BIN_DIR):

From 550178ad98e47da842fdb047d4d451fd57146eb7 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 3 Oct 2023 10:52:51 -0700
Subject: [PATCH 82/90] fix build order for dist

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 6c52807..a00579f 100644
--- a/Makefile
+++ b/Makefile
@@ -162,7 +162,7 @@ toolchain-dist:
 		&& { echo "Error: Git has untracked files present"; exit 1; } || :
 	git diff --name-only | grep . \
 		&& { echo "Error: Git has unstaged changes present"; exit 1; } || :
-	$(MAKE) toolchain-clean default toolchain-restore-mtime toolchain-dist-cache
+	$(MAKE) toolchain-clean toolchain-restore-mtime toolchain-dist-cache default
 	cp -Rp $(OUT_DIR)/* $(DIST_DIR)/
 
 $(BIN_DIR):

From 832b9640c541bc5de7964dfb8428cab76324eb24 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 3 Oct 2023 12:10:22 -0700
Subject: [PATCH 83/90] add space variable

---
 Makefile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Makefile b/Makefile
index a00579f..21f789d 100644
--- a/Makefile
+++ b/Makefile
@@ -24,6 +24,8 @@ GIT_AUTHOR := $(shell git log -1 --format=%an)
 GIT_KEY := $(shell git log -1 --format=%GP)
 GIT_TIMESTAMP := $(shell git log -1 --format=%cd --date=iso)
 , := ,
+empty :=
+space := $(empty) $(empty)
 ifeq ($(strip $(shell git status --porcelain 2>/dev/null)),)
 	GIT_STATE=clean
 else

From f0d39c33bcc9b5dc7546eb35c315f1fb1b7646d4 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Mon, 9 Oct 2023 14:55:37 -0700
Subject: [PATCH 84/90] add basic profiling support

---
 Makefile | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/Makefile b/Makefile
index 21f789d..3d2f7d5 100644
--- a/Makefile
+++ b/Makefile
@@ -52,6 +52,31 @@ PATH_PREFIX := /home/build/.local/bin:/home/build/$(CACHE_DIR)/bin:/home/build/$
 PREFIX := $(HOME)/.local
 XDG_CONFIG_HOME := $(HOME)/.config
 
+ifneq ($(TOOLCHAIN_PROFILE),false)
+mkc := $(shell mkdir -p $(CACHE_DIR_ROOT))
+ifndef TOOLCHAIN_PROFILE_RUNNING
+rmp := $(shell rm -f $(CACHE_DIR_ROOT)/toolchain-profile.csv)
+TOOLCHAIN_PROFILE_START := 0
+TOOLCHAIN_PROFILE_RUNNING := true
+export TOOLCHAIN_PROFILE_RUNNING TOOLCHAIN_PROFILE_START
+endif
+
+.PHONY: toolchain-profile
+toolchain-profile:
+	@echo Target build times:
+	@column -s, -t < $(CACHE_DIR_ROOT)/toolchain-profile.csv
+endif
+
+define toolchain_profile_start
+	$(eval TOOLCHAIN_PROFILE_START=$(shell date +%s))
+	echo START=$(TOOLCHAIN_PROFILE_START)
+	@printf "%s," "$@" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
+endef
+
+define toolchain_profile_end
+printf "%s\n" "$$(date -d@$$(($$(date +%s)-$(TOOLCHAIN_PROFILE_START))) -u +%H:%M:%S)" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
+endef
+
 export
 
 include $(CONFIG_DIR)/make.env

From c61a119e3c9b4bd3ecd6b5925900920df0e2dfa7 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Mon, 9 Oct 2023 17:03:44 -0700
Subject: [PATCH 85/90] improved profiling, and profile self

---
 Makefile | 41 +++++++++++++++++++++++++++++++++--------
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/Makefile b/Makefile
index 3d2f7d5..78346f8 100644
--- a/Makefile
+++ b/Makefile
@@ -56,25 +56,45 @@ ifneq ($(TOOLCHAIN_PROFILE),false)
 mkc := $(shell mkdir -p $(CACHE_DIR_ROOT))
 ifndef TOOLCHAIN_PROFILE_RUNNING
 rmp := $(shell rm -f $(CACHE_DIR_ROOT)/toolchain-profile.csv)
+TOOLCHAIN_PROFILE_INIT := $(shell date +%s)
 TOOLCHAIN_PROFILE_START := 0
+TOOLCHAIN_PROFILE_TOTAL := 0
+TOOLCHAIN_PROFILE_TRACKED := 0
+TOOLCHAIN_PROFILE_UNTRACKED := 0
 TOOLCHAIN_PROFILE_RUNNING := true
-export TOOLCHAIN_PROFILE_RUNNING TOOLCHAIN_PROFILE_START
+export TOOLCHAIN_PROFILE_RUNNING TOOLCHAIN_PROFILE_START TOOLCHAIN_PROFILE_TOTAL TOOLCHAIN_PROFILE_UNTRACKED TOOLCHAIN_PROFILE_TRACKED
 endif
 
 .PHONY: toolchain-profile
 toolchain-profile:
-	@echo Target build times:
-	@column -s, -t < $(CACHE_DIR_ROOT)/toolchain-profile.csv
+	$(call toolchain-profile-total)
+	$(call toolchain-profile-tracked)
+	$(call toolchain-profile-untracked)
+	@printf "unprofiled,%s\n" "$(TOOLCHAIN_PROFILE_UNTRACKED)" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
+	@printf "total,%s" "$(TOOLCHAIN_PROFILE_TOTAL)" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
+	@echo Build times:
+	@column -c 80 -s, -t < $(CACHE_DIR_ROOT)/toolchain-profile.csv
 endif
 
-define toolchain_profile_start
+define toolchain-profile-total
+	$(eval TOOLCHAIN_PROFILE_TOTAL=$(shell date -d@$$(($(shell date +%s)-$(TOOLCHAIN_PROFILE_INIT))) -u +%s))
+endef
+
+define toolchain-profile-tracked
+	$(eval TOOLCHAIN_PROFILE_TRACKED=$(shell cat $(CACHE_DIR_ROOT)/toolchain-profile.csv | cut -d ',' -f2 | awk '{ sum += $$1 } END { print sum }'))
+endef
+
+define toolchain-profile-untracked
+	$(eval TOOLCHAIN_PROFILE_UNTRACKED=$(shell printf $$(($(TOOLCHAIN_PROFILE_TOTAL)-$(TOOLCHAIN_PROFILE_TRACKED))) -u +%s))
+endef
+
+define toolchain-profile-start
 	$(eval TOOLCHAIN_PROFILE_START=$(shell date +%s))
-	echo START=$(TOOLCHAIN_PROFILE_START)
 	@printf "%s," "$@" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
 endef
 
-define toolchain_profile_end
-printf "%s\n" "$$(date -d@$$(($$(date +%s)-$(TOOLCHAIN_PROFILE_START))) -u +%H:%M:%S)" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
+define toolchain-profile-end
+printf "%s\n" "$$(($$(date +%s)-$(TOOLCHAIN_PROFILE_START)))" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
 endef
 
 export
@@ -133,6 +153,7 @@ $(CONFIG_DIR)/apt-pins-x86_64.list \
 $(CONFIG_DIR)/apt-sources-x86_64.list \
 $(CONFIG_DIR)/apt-hashes-x86_64.list: \
 $(CONFIG_DIR)/apt-base.list
+	$(call toolchain-profile-start)
 	mkdir -p $(FETCH_DIR)/apt \
 	&& docker run \
 		--rm \
@@ -146,9 +167,11 @@ $(CONFIG_DIR)/apt-base.list
 		--workdir $(TOOLCHAIN_WORKDIR) \
 		debian@sha256:$(DEBIAN_HASH) \
 		/usr/local/bin/packages-update
+	$(call toolchain-profile-end)
 
 # Pin all packages in toolchain container to latest versions
 $(FETCH_DIR)/apt/Packages.bz2: $(CONFIG_DIR)/apt-hashes-x86_64.list
+	$(call toolchain-profile-start)
 	docker run \
 		--rm \
 		--tty \
@@ -164,6 +187,7 @@ $(FETCH_DIR)/apt/Packages.bz2: $(CONFIG_DIR)/apt-hashes-x86_64.list
 		--workdir $(TOOLCHAIN_WORKDIR) \
 		debian@sha256:$(DEBIAN_HASH) \
 		/usr/local/bin/packages-fetch
+	$(call toolchain-profile-end)
 
 .PHONY: toolchain-clean
 toolchain-clean:
@@ -251,6 +275,7 @@ $(CACHE_DIR_ROOT)/toolchain.tgz: \
 	$(CONFIG_DIR)/apt-pins-$(ARCH).list \
 	$(CONFIG_DIR)/apt-hashes-$(ARCH).list \
 	| $(FETCH_DIR)/apt/Packages.bz2
+	$(call toolchain-profile-start)
 	mkdir -p $(CACHE_DIR)
 	DOCKER_BUILDKIT=1 \
 	docker build \
@@ -263,6 +288,7 @@ $(CACHE_DIR_ROOT)/toolchain.tgz: \
 		-f $(SRC_DIR)/toolchain/Dockerfile \
 		.
 	docker save "$(IMAGE)" | gzip > "$@"
+	$(call toolchain-profile-end)
 
 $(CACHE_DIR_ROOT)/toolchain.state: \
 	$(CACHE_DIR_ROOT)/toolchain.tgz
@@ -372,4 +398,3 @@ define toolchain
                 $$(cat $(CACHE_DIR_ROOT)/toolchain.state 2> /dev/null) \
                 $(SRC_DIR)/toolchain/scripts/host-env bash -c $(1)
 endef
-

From 9a968c87c874a796d17706d5afd2c0386ee5abb2 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 10 Oct 2023 14:59:00 -0700
Subject: [PATCH 86/90] Use native git-restore mtime and use custom profile
 files

---
 Makefile | 65 +++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 45 insertions(+), 20 deletions(-)

diff --git a/Makefile b/Makefile
index 78346f8..0280254 100644
--- a/Makefile
+++ b/Makefile
@@ -17,6 +17,8 @@ NAME := $(shell basename $(shell git rev-parse --show-toplevel | tr A-Z a-z ))
 UID := $(shell id -u)
 GID := $(shell id -g)
 USER := $(UID):$(GID)
+USERNAME := $(shell whoami)
+HOSTNAME := $(shell uname -n)
 CPUS := $(shell docker run debian nproc)
 ARCHIVE_SOURCES := true
 GIT_REF := $(shell git log -1 --format=%H)
@@ -53,16 +55,17 @@ PREFIX := $(HOME)/.local
 XDG_CONFIG_HOME := $(HOME)/.config
 
 ifneq ($(TOOLCHAIN_PROFILE),false)
-mkc := $(shell mkdir -p $(CACHE_DIR_ROOT))
+TOOLCHAIN_PROFILE_DIR := .toolchain/profiles
+mkc := $(shell mkdir -p $(TOOLCHAIN_PROFILE_DIR))
 ifndef TOOLCHAIN_PROFILE_RUNNING
-rmp := $(shell rm -f $(CACHE_DIR_ROOT)/toolchain-profile.csv)
 TOOLCHAIN_PROFILE_INIT := $(shell date +%s)
 TOOLCHAIN_PROFILE_START := 0
 TOOLCHAIN_PROFILE_TOTAL := 0
 TOOLCHAIN_PROFILE_TRACKED := 0
 TOOLCHAIN_PROFILE_UNTRACKED := 0
 TOOLCHAIN_PROFILE_RUNNING := true
-export TOOLCHAIN_PROFILE_RUNNING TOOLCHAIN_PROFILE_START TOOLCHAIN_PROFILE_TOTAL TOOLCHAIN_PROFILE_UNTRACKED TOOLCHAIN_PROFILE_TRACKED
+TOOLCHAIN_PROFILE_FILE := \
+	$(TOOLCHAIN_PROFILE_DIR)/$(HOSTNAME)-$(USERNAME)-$(HOST_OS)-$(HOST_ARCH).$(shell date -u -d @$(TOOLCHAIN_PROFILE_INIT) +%Y%m%dT%H%M%S).csv
 endif
 
 .PHONY: toolchain-profile
@@ -70,31 +73,38 @@ toolchain-profile:
 	$(call toolchain-profile-total)
 	$(call toolchain-profile-tracked)
 	$(call toolchain-profile-untracked)
-	@printf "unprofiled,%s\n" "$(TOOLCHAIN_PROFILE_UNTRACKED)" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
-	@printf "total,%s" "$(TOOLCHAIN_PROFILE_TOTAL)" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
+	@printf "unprofiled,%s\n" "$(TOOLCHAIN_PROFILE_UNTRACKED)" \
+		>> $(TOOLCHAIN_PROFILE_FILE)
+	@printf "total,%s\n" "$(TOOLCHAIN_PROFILE_TOTAL)" \
+		>> $(TOOLCHAIN_PROFILE_FILE)
 	@echo Build times:
-	@column -c 80 -s, -t < $(CACHE_DIR_ROOT)/toolchain-profile.csv
+	@bash -c ' \
+		while IFS=, read -r target seconds; do \
+			echo $$target,$$(date -u -d @$$seconds +%T); \
+		done < $(TOOLCHAIN_PROFILE_FILE)' \
+		| column -c 80 -s, -t
 endif
 
 define toolchain-profile-total
-	$(eval TOOLCHAIN_PROFILE_TOTAL=$(shell date -d@$$(($(shell date +%s)-$(TOOLCHAIN_PROFILE_INIT))) -u +%s))
+	$(eval TOOLCHAIN_PROFILE_TOTAL=$(shell expr $(shell date +%s) - $(TOOLCHAIN_PROFILE_INIT)) )
 endef
 
 define toolchain-profile-tracked
-	$(eval TOOLCHAIN_PROFILE_TRACKED=$(shell cat $(CACHE_DIR_ROOT)/toolchain-profile.csv | cut -d ',' -f2 | awk '{ sum += $$1 } END { print sum }'))
+	$(eval TOOLCHAIN_PROFILE_TRACKED=$(shell cat $(TOOLCHAIN_PROFILE_FILE) | cut -d ',' -f2 | awk '{ sum += $$1 } END { print sum }'))
 endef
 
 define toolchain-profile-untracked
-	$(eval TOOLCHAIN_PROFILE_UNTRACKED=$(shell printf $$(($(TOOLCHAIN_PROFILE_TOTAL)-$(TOOLCHAIN_PROFILE_TRACKED))) -u +%s))
+	$(eval TOOLCHAIN_PROFILE_UNTRACKED=$(shell expr $(TOOLCHAIN_PROFILE_TOTAL) - $(TOOLCHAIN_PROFILE_TRACKED)) )
 endef
 
 define toolchain-profile-start
 	$(eval TOOLCHAIN_PROFILE_START=$(shell date +%s))
-	@printf "%s," "$@" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
+	printf "%s," "$@" >> $(TOOLCHAIN_PROFILE_FILE)
 endef
 
-define toolchain-profile-end
-printf "%s\n" "$$(($$(date +%s)-$(TOOLCHAIN_PROFILE_START)))" >> $(CACHE_DIR_ROOT)/toolchain-profile.csv
+define toolchain-profile-stop
+printf "%s\n" "$$(($$(date +%s)-$(TOOLCHAIN_PROFILE_START)))" \
+	>> $(TOOLCHAIN_PROFILE_FILE)
 endef
 
 export
@@ -135,10 +145,21 @@ toolchain-update:
 
 .PHONY: toolchain-restore-mtime
 toolchain-restore-mtime:
-	$(call toolchain," \
-		git restore-mtime \
-		&& echo "Git mtime restored" \
-	")
+	$(call toolchain-profile-start)
+	bash -c '\
+		for d in $$(git ls-files | xargs -n 1 dirname | uniq); do \
+			mkdir -p "$$d"; \
+		done; \
+		for f in $$(git ls-tree -r -t --full-name --name-only "HEAD"); do \
+			touch -t \
+				$$(git log \
+					--pretty=format:%cd \
+					--date=format:%Y%m%d%H%M.%S \
+					-1 "HEAD" -- "$$f"\
+				) "$$f"; \
+		done; \
+	'
+	$(call toolchain-profile-stop)
 
 .PHONY: toolchain-dist-cache
 toolchain-dist-cache:
@@ -167,7 +188,7 @@ $(CONFIG_DIR)/apt-base.list
 		--workdir $(TOOLCHAIN_WORKDIR) \
 		debian@sha256:$(DEBIAN_HASH) \
 		/usr/local/bin/packages-update
-	$(call toolchain-profile-end)
+	$(call toolchain-profile-stop)
 
 # Pin all packages in toolchain container to latest versions
 $(FETCH_DIR)/apt/Packages.bz2: $(CONFIG_DIR)/apt-hashes-x86_64.list
@@ -187,10 +208,11 @@ $(FETCH_DIR)/apt/Packages.bz2: $(CONFIG_DIR)/apt-hashes-x86_64.list
 		--workdir $(TOOLCHAIN_WORKDIR) \
 		debian@sha256:$(DEBIAN_HASH) \
 		/usr/local/bin/packages-fetch
-	$(call toolchain-profile-end)
+	$(call toolchain-profile-stop)
 
 .PHONY: toolchain-clean
 toolchain-clean:
+	$(call toolchain-profile-start)
 	if [ -d "$(CACHE_DIR_ROOT)" ]; then \
 		chmod -R u+w $(CACHE_DIR_ROOT); \
 		rm -rf $(CACHE_DIR_ROOT); \
@@ -199,6 +221,7 @@ toolchain-clean:
 		rm -rf $(OUT_DIR); \
 	fi
 	docker image rm -f $(IMAGE) || :
+	$(call toolchain-profile-stop)
 
 .PHONY: toolchain-reproduce
 toolchain-reproduce: toolchain-clean
@@ -213,7 +236,7 @@ toolchain-dist:
 		&& { echo "Error: Git has untracked files present"; exit 1; } || :
 	git diff --name-only | grep . \
 		&& { echo "Error: Git has unstaged changes present"; exit 1; } || :
-	$(MAKE) toolchain-clean toolchain-restore-mtime toolchain-dist-cache default
+	$(MAKE) toolchain-restore-mtime toolchain-clean toolchain-dist-cache default
 	cp -Rp $(OUT_DIR)/* $(DIST_DIR)/
 
 $(BIN_DIR):
@@ -288,12 +311,14 @@ $(CACHE_DIR_ROOT)/toolchain.tgz: \
 		-f $(SRC_DIR)/toolchain/Dockerfile \
 		.
 	docker save "$(IMAGE)" | gzip > "$@"
-	$(call toolchain-profile-end)
+	$(call toolchain-profile-stop)
 
 $(CACHE_DIR_ROOT)/toolchain.state: \
 	$(CACHE_DIR_ROOT)/toolchain.tgz
+	$(call toolchain-profile-start)
 	docker load -i $(CACHE_DIR_ROOT)/toolchain.tgz
 	docker images --no-trunc --quiet $(IMAGE) > $@
+	$(call toolchain-profile-stop)
 
 $(OUT_DIR)/release.env: $(shell git ls-files)
 	echo 'VERSION=$(VERSION)'              > $(OUT_DIR)/release.env

From b33b2a98aca87f1bdd53e4cf703c1f9ae533ea02 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Tue, 10 Oct 2023 15:18:24 -0700
Subject: [PATCH 87/90] git mtime updates skip non-existant paths

---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 0280254..fc0aea0 100644
--- a/Makefile
+++ b/Makefile
@@ -151,7 +151,8 @@ toolchain-restore-mtime:
 			mkdir -p "$$d"; \
 		done; \
 		for f in $$(git ls-tree -r -t --full-name --name-only "HEAD"); do \
-			touch -t \
+			( test -f "$$f" || test -d "$$f" ) \
+			&& touch -t \
 				$$(git log \
 					--pretty=format:%cd \
 					--date=format:%Y%m%d%H%M.%S \

From 91aa2147501675097e965c172f02e8eaf7f2224b Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 12 Oct 2023 17:29:27 -0700
Subject: [PATCH 88/90] eval based start/stop refactor for profiling

---
 Makefile | 41 +++++++++++++++++++++--------------------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/Makefile b/Makefile
index fc0aea0..1229abf 100644
--- a/Makefile
+++ b/Makefile
@@ -54,39 +54,38 @@ PATH_PREFIX := /home/build/.local/bin:/home/build/$(CACHE_DIR)/bin:/home/build/$
 PREFIX := $(HOME)/.local
 XDG_CONFIG_HOME := $(HOME)/.config
 
+# MacOS users do not have a 'date' command that supports milliseconds
+# This is what we are forced to do. Other ideas welcome
+define epochms
+$$(python3 -c 'from time import time; print(int(round(time() * 1000)))')
+endef
+
 ifneq ($(TOOLCHAIN_PROFILE),false)
 TOOLCHAIN_PROFILE_DIR := .toolchain/profiles
 mkc := $(shell mkdir -p $(TOOLCHAIN_PROFILE_DIR))
 ifndef TOOLCHAIN_PROFILE_RUNNING
-TOOLCHAIN_PROFILE_INIT := $(shell date +%s)
-TOOLCHAIN_PROFILE_START := 0
-TOOLCHAIN_PROFILE_TOTAL := 0
-TOOLCHAIN_PROFILE_TRACKED := 0
-TOOLCHAIN_PROFILE_UNTRACKED := 0
+TOOLCHAIN_PROFILE_INIT := $(shell printf $(call epochms))
 TOOLCHAIN_PROFILE_RUNNING := true
 TOOLCHAIN_PROFILE_FILE := \
-	$(TOOLCHAIN_PROFILE_DIR)/$(HOSTNAME)-$(USERNAME)-$(HOST_OS)-$(HOST_ARCH).$(shell date -u -d @$(TOOLCHAIN_PROFILE_INIT) +%Y%m%dT%H%M%S).csv
+	$(TOOLCHAIN_PROFILE_DIR)/$(HOSTNAME)-$(USERNAME)-$(HOST_OS)-$(HOST_ARCH).$(shell date -u -d @$$(($(TOOLCHAIN_PROFILE_INIT) / 1000)) +%Y%m%dT%H%M%S).csv
 endif
 
 .PHONY: toolchain-profile
 toolchain-profile:
 	$(call toolchain-profile-total)
 	$(call toolchain-profile-tracked)
-	$(call toolchain-profile-untracked)
-	@printf "unprofiled,%s\n" "$(TOOLCHAIN_PROFILE_UNTRACKED)" \
-		>> $(TOOLCHAIN_PROFILE_FILE)
-	@printf "total,%s\n" "$(TOOLCHAIN_PROFILE_TOTAL)" \
-		>> $(TOOLCHAIN_PROFILE_FILE)
 	@echo Build times:
 	@bash -c ' \
-		while IFS=, read -r target seconds; do \
-			echo $$target,$$(date -u -d @$$seconds +%T); \
+		while IFS=, read -r target milliseconds; do \
+			echo $$target,$$(date -u -d @$$(( $$milliseconds / 1000 )) +%T); \
 		done < $(TOOLCHAIN_PROFILE_FILE)' \
 		| column -c 80 -s, -t
+	@echo "Real Total: $$(($(TOOLCHAIN_PROFILE_TOTAL)/1000))"
+	@echo "Tracked Total: $$(($(TOOLCHAIN_PROFILE_TRACKED)/1000))"
 endif
 
 define toolchain-profile-total
-	$(eval TOOLCHAIN_PROFILE_TOTAL=$(shell expr $(shell date +%s) - $(TOOLCHAIN_PROFILE_INIT)) )
+	$(eval TOOLCHAIN_PROFILE_TOTAL=$(shell expr $(call epochms) - $(TOOLCHAIN_PROFILE_INIT)) )
 endef
 
 define toolchain-profile-tracked
@@ -98,13 +97,12 @@ define toolchain-profile-untracked
 endef
 
 define toolchain-profile-start
-	$(eval TOOLCHAIN_PROFILE_START=$(shell date +%s))
-	printf "%s," "$@" >> $(TOOLCHAIN_PROFILE_FILE)
+	$(eval TOOLCHAIN_PROFILE_START_$(shell printf "$@"| openssl sha256 | awk '{ print $$2}')=$$($(shell printf $(call epochms))))
 endef
 
 define toolchain-profile-stop
-printf "%s\n" "$$(($$(date +%s)-$(TOOLCHAIN_PROFILE_START)))" \
-	>> $(TOOLCHAIN_PROFILE_FILE)
+printf "%s,%s\n" "$@" "$$(($(call epochms)-$(TOOLCHAIN_PROFILE_START_$(shell printf "$@" | openssl sha256 | awk '{ print $$2}'))))" \
+	>> $(TOOLCHAIN_PROFILE_FILE);
 endef
 
 export
@@ -150,7 +148,7 @@ toolchain-restore-mtime:
 		for d in $$(git ls-files | xargs -n 1 dirname | uniq); do \
 			mkdir -p "$$d"; \
 		done; \
-		for f in $$(git ls-tree -r -t --full-name --name-only "HEAD"); do \
+		for f in $$((git ls-files --modified; git ls-files) | sort | uniq -u); do \
 			( test -f "$$f" || test -d "$$f" ) \
 			&& touch -t \
 				$$(git log \
@@ -407,7 +405,10 @@ define fetch_pgp_key
 endef
 
 define toolchain
-        $(MAKE) toolchain \
+		( test -f $(CACHE_DIR_ROOT)/toolchain.state || { \
+			echo "Error: toolchain.state not found. Check dependencies!"; \
+			exit 1; \
+		};) \
         && docker run \
                 --rm \
                 --tty \

From 10767638ca795513ef5a63c98cedba060958687a Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Thu, 12 Oct 2023 19:09:16 -0700
Subject: [PATCH 89/90] initial file-based profile timing to avoid eval
 ordering issue

---
 Makefile | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/Makefile b/Makefile
index 1229abf..25c849f 100644
--- a/Makefile
+++ b/Makefile
@@ -73,15 +73,14 @@ endif
 .PHONY: toolchain-profile
 toolchain-profile:
 	$(call toolchain-profile-total)
-	$(call toolchain-profile-tracked)
 	@echo Build times:
 	@bash -c ' \
-		while IFS=, read -r target milliseconds; do \
-			echo $$target,$$(date -u -d @$$(( $$milliseconds / 1000 )) +%T); \
+		while IFS=, read -r target ms_start ms_stop; do \
+			ms_diff=$$(($$ms_stop - $$ms_start)); \
+			echo - $$target,$$(date -u -d @$$(( $$ms_diff / 1000 )) +%T); \
 		done < $(TOOLCHAIN_PROFILE_FILE)' \
 		| column -c 80 -s, -t
-	@echo "Real Total: $$(($(TOOLCHAIN_PROFILE_TOTAL)/1000))"
-	@echo "Tracked Total: $$(($(TOOLCHAIN_PROFILE_TRACKED)/1000))"
+	@echo "Total: $$(date -u -d @$$(( $(TOOLCHAIN_PROFILE_TOTAL) / 1000 )) +%T)";
 endif
 
 define toolchain-profile-total
@@ -97,12 +96,17 @@ define toolchain-profile-untracked
 endef
 
 define toolchain-profile-start
-	$(eval TOOLCHAIN_PROFILE_START_$(shell printf "$@"| openssl sha256 | awk '{ print $$2}')=$$($(shell printf $(call epochms))))
+	printf "%s,$(call epochms),\n" "$@" >> $(TOOLCHAIN_PROFILE_FILE);
 endef
 
 define toolchain-profile-stop
-printf "%s,%s\n" "$@" "$$(($(call epochms)-$(TOOLCHAIN_PROFILE_START_$(shell printf "$@" | openssl sha256 | awk '{ print $$2}'))))" \
-	>> $(TOOLCHAIN_PROFILE_FILE);
+	tmpfile=$$(mktemp -q "$(TOOLCHAIN_PROFILE_DIR)/tmp.XXXXXXXXX") \
+	&& cp $(TOOLCHAIN_PROFILE_FILE) $$tmpfile \
+	&& awk \
+		-v ms="$(call epochms)" \
+		'/^$(@),/ {$$0=$$0ms} 1' \
+		$$tmpfile \
+	> $(TOOLCHAIN_PROFILE_FILE)
 endef
 
 export

From 00ce00c246766b8a2ce09b96cf23b812a04d03a4 Mon Sep 17 00:00:00 2001
From: "Lance R. Vick" <lance@lrvick.net>
Date: Fri, 13 Oct 2023 05:04:57 -0700
Subject: [PATCH 90/90] avoid breaking on targets with special chars

---
 Makefile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 25c849f..9ceb1fc 100644
--- a/Makefile
+++ b/Makefile
@@ -96,7 +96,7 @@ define toolchain-profile-untracked
 endef
 
 define toolchain-profile-start
-	printf "%s,$(call epochms),\n" "$@" >> $(TOOLCHAIN_PROFILE_FILE);
+	printf "%s,$(call epochms),\n" "$@" >> "$(TOOLCHAIN_PROFILE_FILE)"
 endef
 
 define toolchain-profile-stop
@@ -104,9 +104,10 @@ define toolchain-profile-stop
 	&& cp $(TOOLCHAIN_PROFILE_FILE) $$tmpfile \
 	&& awk \
 		-v ms="$(call epochms)" \
-		'/^$(@),/ {$$0=$$0ms} 1' \
+		-v target="$(@)" \
+		'$$1 ~ "^" target {$$0=$$0ms} 1' \
 		$$tmpfile \
-	> $(TOOLCHAIN_PROFILE_FILE)
+	> "$(TOOLCHAIN_PROFILE_FILE)"
 endef
 
 export