mirror_file_set

#!/usr/bin/env bash
# Description:
# This script is designed to work as part of a pipeline to mirror Debian packages. 
# It takes a single file extracted from file_sets.tar.xz as input, downloads the 
# Debian file specified in the input file from the Debian repository, and uploads 
# it to a specified S3 bucket. Additionally, it validates the integrity of the 
# downloaded file by checking its hash.
#
# Usage:
# ./mirror_file_set <input_file>
#
# Arguments:
# <input_file>: A text file containing lines that follow the schema:
#               <debian pool path> <new pool path> <sha256sum>
#               Each line represents a Debian package to be downloaded and mirrored.
#
# Behavior:
# For each line in the input file:
# 1. Construct the Debian file URL using: http://deb.debian.org/debian/<debian pool path>
# 2. Download the Debian file.
# 3. Upload the file to the S3 bucket at the location specified by <new pool path>.
# 4. Verify the SHA256 hash of the downloaded file matches <sha256sum>.
#    If the hash does not match, report an error.



set -eufo pipefail

num_pkgs="$(wc -l "$1" | cut -d ' ' -f 1)"
cntr=0

existing_file_set="$(mktemp)"
aws s3api list-objects --bucket gardenlinux-repo-test | jq -r '.Contents // [] | .[].Key' > "$existing_file_set"

while read -r source target sha256sums; do
	cntr=$(( cntr + 1 ))
	tr ' ' '\n' <<< "$sha256sums" | while IFS=':' read -r hash file; do
		source_file="$source"
		target_file="$target"
		if [ -n "$file" ]; then
			source_file+="/$file"
			target_file+="/$file"
		fi

		target_file_transformed="$(tr '+' ' ' <<< "$target_file")"

		if grep -xF "$target_file" "$existing_file_set" > /dev/null && grep -xF "$target_file_transformed" "$existing_file_set" > /dev/null; then
			echo "$cntr/$num_pkgs $target_file already exists, skipped"
			continue
		fi

		if file="$(./download_checked "$hash" "http://deb.debian.org/debian/$source_file")"; then
			size="$(wc -c "$file" | cut -d ' ' -f 1)"
			aws s3 cp --quiet "$file" "s3://gardenlinux-repo-test/$target_file"
			[ "$target_file" = "$target_file_transformed" ] || aws s3 cp --quiet "$file" "s3://gardenlinux-repo-test/$target_file_transformed"
			rm "$file"
			echo "$cntr/$num_pkgs $target_file uploaded ($(numfmt --to=iec-i --suffix=B "$size"))"
		else
			exit 1
		fi
	done
done < "$1"