init_mirror

#!/usr/bin/env bash
#
# Description:
# This script creates file_sets.tar.xz and repo.tar.xz. 
# file_sets.tar.xz comprises a configurable number of sub-files, set by NUM_FILE_SETS, which also 
# dictates the parallel mirror_file_set calls in subsequent GitHub pipeline stages.
#
# Usage:
# ./init_mirror
#
# Effects:
# - Generates 'file_sets.tar.xz': A tarball containing pool paths and respective hashes 
#   of binary and source packages. Each line follows the schema:
#     <debian pool path> <new pool path> <sha256sum>
# - Generates 'repo.tar.xz': A tarball containing repository metadata files such as:
#   - keyring.gpg
#   - InRelease files
#   - Packages.gz files
#   - Sources.gz files

set -exufo pipefail

version="$1"
t="$(date -u -Is)"

NUM_FILE_SETS="32"
GNUPGHOME="$(mktemp -d)"
chmod 700 "$GNUPGHOME"
export GNUPGHOME
gpg --generate-key --batch < gpg-key.conf
gpg --import < debian-archive-keyring.gpg

release="$(mktemp)"
curl -sSL http://deb.debian.org/debian/dists/testing/InRelease | gpg --verify --output - > "$release"

file_sets="$(mktemp -d)"
repo="$(mktemp -d)"
mkdir "$repo/main"

index_files="$(mktemp)"

for arch in all amd64 arm64; do
	file="main/binary-$arch/Packages"
	mkdir "$repo/$(dirname "$file")"
	hash="$(awk -v file="$file.xz" '!/^ / { flag=0 } flag && $3 == file { print $1 } /^SHA256:$/ { flag = 1 }' < "$release")"
	pkgs="$(./download_checked "$hash" "http://deb.debian.org/debian/dists/testing/$file.xz")"
	xz -d < "$pkgs" | ./packages.awk file_set_prefix="$file_sets/pkgs_${arch}_" num_file_sets=${NUM_FILE_SETS} > "$repo/$file"
	rm "$pkgs"
	size="$(wc -c "$repo/$file" | cut -d ' ' -f 1)"
	hash="$(sha256sum "$repo/$file" | head -c 64)"
	echo " $hash $size $file" >> "$index_files"
done

file="main/source/Sources"
mkdir "$repo/$(dirname "$file")"
hash="$(awk -v file="$file.xz" '!/^ / { flag=0 } flag && $3 == file { print $1 } /^SHA256:$/ { flag = 1 }' < "$release")"
srcs="$(./download_checked "$hash" "http://deb.debian.org/debian/dists/testing/$file.xz")"
xz -d < "$srcs" | ./sources.awk file_set_prefix="$file_sets/srcs_" num_file_sets=${NUM_FILE_SETS} > "$repo/$file"
rm "$srcs"
size="$(wc -c "$repo/$file" | cut -d ' ' -f 1)"
hash="$(sha256sum "$repo/$file" | head -c 64)"
echo " $hash $size $file" >> "$index_files"

rm "$release"

cat << EOF | ./strip_newlines.py | gpg --clearsign > "$repo/InRelease"
Codename: $version
Components: main
Architectures: all amd64 arm64
Date: $(date -R -u -d "$t")
Valid-Until: $(date -R -u -d "$t + 100 years")
SHA256:
$(cat "$index_files")
EOF

gpg --export root > "$repo/keyring.gpg"

tar -c -C "$repo" . | xz > repo.tar.xz
tar -c -C "$file_sets" . | xz > file_sets.tar.xz
rm -rf "$file_sets" "$GNUPGHOME"