From eaec9859ed51180a00c6c2ee16273dc2d900d550 Mon Sep 17 00:00:00 2001 From: Michael Lustfield Date: Fri, 22 Sep 2023 03:24:01 -0500 Subject: [PATCH] Update to Debian 12 && Fix VM compatibility issues. - TH_SRC switched to "archive" to avoid update-removals. + Move iso retrieval stuff to Makefile. - Switch from packaged salt to pip-based install. --- .github/workflows/cicd.yml | 3 -- .github/workflows/release.yml | 4 --- Makefile | 25 +++++++++++--- bootstrap | 62 +++++++++++++++++++++++++---------- iso/build_iso | 53 ++++-------------------------- states/salt/init.sls | 49 +++++++++++++++++++++------ test/vbox_create | 15 +++++++-- 7 files changed, 125 insertions(+), 86 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index c0df8b6..d7dea2b 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -75,9 +75,6 @@ jobs: id: build_iso run: make teckhost-sda.iso env: - # current stable - TH_SRC: https://cdimage.debian.org/cdimage/unofficial/non-free/cd-including-firmware/current/amd64/iso-cd/firmware-11.7.0-amd64-netinst.iso - TH_CKSUM: "029500297f14bd4f6650fa4aa991c96027b8d5cd9cc91b38722ee2b914612e851fa81d60f4e7ad739565a83128a98461368fc95defc01ec7d66c62a32ca15bf9" THT_GRUBTEST: TEMPLATE_METHOD=lvm BS_pillar_root=test/pillar TH_SALTGPG=https://raw.githubusercontent.com/MTecknology/teckhost/master/test/pillar/skeys.gpg BS_gitfs_pillar_base=master BS_gitfs_base=${{ github.sha }} - name: Save ISO (teckhost-sda.iso) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c8a8264..0d0d347 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,10 +44,6 @@ jobs: - name: Build Teckhost ISO id: build_iso run: make teckhost.iso teckhost-sda.iso teckhost-nvme0n1.iso - env: - # current stable - TH_SRC: https://cdimage.debian.org/cdimage/unofficial/non-free/cd-including-firmware/current/amd64/iso-cd/firmware-11.7.0-amd64-netinst.iso - TH_CKSUM: "029500297f14bd4f6650fa4aa991c96027b8d5cd9cc91b38722ee2b914612e851fa81d60f4e7ad739565a83128a98461368fc95defc01ec7d66c62a32ca15bf9" # 3. Publish Release w/ Artifacts - name: Create Release diff --git a/Makefile b/Makefile index 22379fc..9e0862b 100644 --- a/Makefile +++ b/Makefile @@ -6,37 +6,54 @@ export WORKSPACE ?= $(abspath $(PWD)/) export GRUB_EXTRA ?= hostname=testpc1 +# Version Table +debian12_src ?= https://cdimage.debian.org/cdimage/archive/12.1.0/amd64/iso-cd/debian-12.1.0-amd64-netinst.iso +debian12_sha ?= 9f181ae12b25840a508786b1756c6352a0e58484998669288c4eec2ab16b8559 + ## # ISO ## -# Intended for production use (assumes nvme) -teckhost.iso: iso/preseed.cfg iso/grub-bios.cfg iso/grub-efi.cfg +# Intended for production use +teckhost.iso: upstream_debian12.iso iso/preseed.cfg iso/grub-bios.cfg iso/grub-efi.cfg ./iso/build_iso \ -s iso/preseed.cfg \ + -i upstream_debian12.iso \ -o teckhost.iso \ -x "$(GRUB_EXTRA)" \ -f iso/grub-bios.cfg -g iso/grub-efi.cfg # Intended for use with automated testing -teckhost-%.iso: testseed.cfg iso/grub-bios.cfg iso/grub-efi.cfg +teckhost-%.iso: upstream_debian12.iso testseed.cfg iso/grub-bios.cfg iso/grub-efi.cfg ./iso/build_iso \ -s testseed.cfg \ + -i upstream_debian12.iso \ -o "$@" \ -d "/dev/$*" \ -x "$(GRUB_EXTRA)" \ -f iso/grub-bios.cfg -g iso/grub-efi.cfg # Intended for local developmnt with virtualbox -teckhost-local.iso: testseed.cfg iso/grub-bios.cfg iso/grub-efi.cfg +teckhost-local.iso: upstream_debian12.iso testseed.cfg iso/grub-bios.cfg iso/grub-efi.cfg ./iso/build_iso \ -s testseed.cfg \ + -i upstream_debian12.iso \ -o teckhost-local.iso \ -d /dev/sda \ -x "hostname=devpc1 BS_devdir=/srv" \ -f iso/grub-bios.cfg -g iso/grub-efi.cfg +# Grab an upstream ISO and validate checksum +upstream_%.iso: + # Copy iso from parent directory or download fresh copy + cp "../$($*_sha).iso" ./ || wget --quiet -O "$($*_sha).iso" "$($*_src)" + # Verify checksum of pristine iso + echo "$($*_sha) $($*_sha).iso" | sha256sum -c + # Move into location to verify success + mv "$($*_sha).iso" "upstream_$*.iso" + + ## # Preeseed ## diff --git a/bootstrap b/bootstrap index ea01d26..68cfd71 100755 --- a/bootstrap +++ b/bootstrap @@ -1,4 +1,5 @@ #!/bin/bash +set -x ## # A quick/simple script to get masterless salt deployed and configured. # See help text (./bootstrap -h) and README.rst for requirements. @@ -9,27 +10,35 @@ # # BS_*: This script will dump all "BS_*" environment variables into a configuration # file for salt-minion; this provides a way to mangle teckhost.conf. +# +# Critical Order: +# 1. Prepare apt +# 2. Install apt/salt dependencies, +# 3. Add salt repo +# 4. Install salt +# 5. Chown salt ## main() { parse_options "$@" - # Prep + # Pre-flight safety_checks lock acquire "$0" || die 'Unable to acquire lock' + pristine_apt || die 'Failed to set pristine apt configuration' mkdir -p /etc/salt/minion.d + echo 'master: invalid.tld' >/etc/salt/minion.d/teckhost.conf - # Patch gai.conf + # Hack: IPv6 is often incorrectly implemented and first boot is very touchy echo 'precedence ::ffff:0:0/96 100' >>/etc/gai.conf + echo 'Acquire::ForceIPv4 "true";' >/etc/apt/apt.conf.d/99force-ipv4 - # Install Masterless Salt - configure_apt || die 'Failed to configure apt' - apt-get -y install gpg python3-pygit2 wget || die 'Failed to install dependencies' - configure_minion || die 'Failed to install salt-minion' + # Install and configure salt-minion (solo) + install_dependencies || die 'Failed to install dependencies' + install_salt || die 'Failed to install salt-minion service' + configure_minion || die 'Failed to configure salt-minion' deploy_gpgkeys || die 'Failed to unpack GPG keys' - #TODO: This is TEMPORARY hack because we need >= 3004.0 - apt-get install -y salt-minion=3004.1+dfsg-2 salt-common=3004.1+dfsg-2 || die 'Failed to install salt-minion' - #apt-get install -y salt-minion || die 'Failed to install salt-minion' + chown -R root:root /etc/salt/gpgkeys # Run Highstate and Configure System run_highstate || die 'Provisioning process (highstate) failed' @@ -86,7 +95,7 @@ safety_checks() { } # Ensure a clean apt state prior to salt management -configure_apt() { +pristine_apt() { # Find $OSCODENAME # Seems excessive when only one path is likely, but who knows how this might get used if [[ -n "$OSCODENAME" ]]; then @@ -105,13 +114,13 @@ configure_apt() { rm -rf /etc/apt/sources.list* mkdir /etc/apt/sources.list.d cat >/etc/apt/sources.list <<-EOF - deb http://deb.debian.org/debian $OSCODENAME main contrib non-free - deb http://security.debian.org/debian-security $OSCODENAME-security main contrib non-free - deb http://deb.debian.org/debian $OSCODENAME-updates main contrib non-free + deb http://deb.debian.org/debian $OSCODENAME main contrib non-free-firmware + deb http://security.debian.org/debian-security $OSCODENAME-security main contrib non-free-firmware + deb http://deb.debian.org/debian $OSCODENAME-updates main contrib non-free-firmware # Newer Packages (use with extreme caution) - deb http://deb.debian.org/debian testing main non-free contrib - deb http://deb.debian.org/debian sid main non-free contrib + deb http://deb.debian.org/debian testing main non-free-firmware contrib + deb http://deb.debian.org/debian sid main non-free-firmware contrib EOF cat >/etc/apt/preferences.d/pinning <<-EOF Package: * @@ -130,17 +139,36 @@ configure_apt() { Pin: release a=unstable Pin-Priority: 300 EOF + + # Update package cache apt-get update } +# Install build/runtime/etc dependencies for salt-master service +install_dependencies() { + apt-get update || return 1 + apt-get install -y wget python3-venv build-essential git || return 1 +} + +# Install the salt master and get vendor-garbage configured correctly +install_salt() { + # Application + python3 -m venv /opt/salt + /opt/salt/bin/pip3 install cryptography pygit2 salt==3006.4 + + # Directory structure + mkdir -p /etc/salt/minion.d + mkdir -p /etc/salt/pki/minion +} + # Run a highstate run_highstate() { # This is an ugly hack because of some networking hiccups during some deployments. - if ! salt-call --local -l debug state.highstate; then + if ! /opt/salt/bin/salt-call --local -l debug state.highstate; then log "$WARN" 'FIRST HIGHSTATE FAILED; Sleeping a few minutes before retrying.' sleep 240 # Less verbosity to help with information gathering - salt-call --local -l quiet --state-verbose=false state.highstate + /opt/salt/bin/salt-call --local -l quiet --state-verbose=false state.highstate fi } diff --git a/iso/build_iso b/iso/build_iso index 1dc59eb..37d6598 100755 --- a/iso/build_iso +++ b/iso/build_iso @@ -28,7 +28,6 @@ main() { # Read options into environment parse_options() { # Defaults - export TH_CKSUM="$TH_CKSUM" export TH_SRC="${TH_SRC:-./debian-netinst.iso}" export TH_DST="${TH_DST:-./teckhost.iso}" export TH_SEED="${TH_SEED:-preseed.cfg}" @@ -40,9 +39,8 @@ parse_options() { export THT_DEVICE="${THT_DEVICE:-/dev/nvme0n1}" export LOG_LEVEL="${LOG_LEVEL:-1}" - while getopts 'c:i:o:s:f:g:kd:b:x:l:h' OPT; do + while getopts 'i:o:s:f:g:kd:b:x:l:h' OPT; do case "$OPT" in - c) TH_CKSUM="$OPTARG";; i) TH_SRC="$OPTARG";; o) TH_DST="$OPTARG";; s) TH_SEED="$OPTARG";; @@ -57,6 +55,9 @@ parse_options() { *) die "Unexpected argument provided: '$OPT'";; esac done + + # Enable real debugging + [ ${LOG_LEVEL:-1} -lt 1 ] && set -x } # Show help text (an explanation of options) @@ -68,7 +69,6 @@ show_help() { Usage: build_iso [options] Options: - -c X${t}Checksum of resource -i X${t}Pristine ISO to be modified (can be http) -o X${t}Output ISO -s X${t}Preseed template to embed into ISO @@ -82,7 +82,6 @@ show_help() { -h${t}Print this help text and exit Defaults (can be set as environment variables): - TH_CKSUM${t} TH_SRC${t}./debian-netinst.iso TH_DST${t}./teckhost.iso TH_SEED${t}./preseed.cfg @@ -93,21 +92,15 @@ show_help() { THT_GRUBTEST${t}TEMPLATE_METHOD=lvm BS_pillar_root=test/pillar TH_SALTGPG=https://raw.githubusercontent.com/MTecknology/teckhost/master/test/pillar/skeys.gpg THT_GRUBTXTRA${t}"" LOG_LEVEL${t}1 (info) - - Tell build_iso to pull a remote iso and store in /tmp/\$TH_CKSUM.iso: - export \\ - TH_SRC=https://cdimage.debian.org/cdimage/unofficial/non-free/cd-including-firmware/current/amd64/iso-cd/firmware-11.3.0-amd64-netinst.iso \\ - TH_CKSUM=eba7ce7823681a610f9f23d6468976517ed92b6b90acec4ac55df62b0a090050bba0145ef5c07f544b92569cd10e9572f4e9f7c3415b3323abffa51cd7c5d4f4 - ./build_iso -o ~/teckhost.iso EOF } safety_checks() { - log "$DEBUG" 'Running sanity checks' + log "$INFO" 'Running sanity checks' # Check for required commands for cmd in 'bsdtar' 'syslinux' 'xorriso'; do - command_present "$cmd" || die 'Missing dependencies, must have: bsdtar syslinux xorriso' + command_present "$cmd" || die 'Missing dependencies, must have: bsdtar (libarchive-tools) syslinux xorriso' done # Verify provided values point at files @@ -119,20 +112,9 @@ safety_checks() { } unpack_source() { - # If source is http, grab/cache resource - if [[ "$TH_SRC" = http* ]]; then - _get_websource || die 'Failed to grab resource' - fi - # Verify source exists [[ ! -f "$TH_SRC" ]] && die "Source file ($TH_SRC) does not exist" - # Verify checksum of source - if [[ -n "$TH_CKSUM" ]]; then - sha512sum "$TH_SRC" | cut -d' ' -f1 | grep -q "$TH_CKSUM" || \ - die "cksum($TH_SRC) != $TH_CKSUM" - fi - # Actually unpack source bsdtar -C "$TH_TEMP" -xf "$TH_SRC" @@ -140,27 +122,6 @@ unpack_source() { chmod -R +w "$TH_TEMP" } -# Pull/cache a copy from the web and swap variables once verified -_get_websource() { - command_present 'wget' || die 'wget is required to get a web source' - - # Local cache location - iso="/tmp/$TH_CKSUM.iso" - - # If cache file exists, validate and return - if [[ -f "$iso" ]]; then - if sha512sum "$iso" | cut -d' ' -f1 | grep -q "$TH_CKSUM"; then - log "$DEBUG" "Valid cache for $TH_SRC" - TH_SRC="$iso" - return 0 - fi - log "$WARN" "Removing invalid cache at $iso" - rm -f "$iso" - fi - # Grab resource and store in cache location - wget -qO "$iso" "$TH_SRC" && TH_SRC="$iso" -} - # Insert an auto-only grub boot config inject_grubconfig() { log "$DEBUG" 'Copying grub config' @@ -195,7 +156,7 @@ inject_preseed() { # Build the modified ISO build_iso() { - log "$DEBUG" "Bulding ISO at $TH_DST" + log "$INFO" "Bulding ISO at $TH_DST" # Some weird requirement; things break without it chmod -R -w "$TH_TEMP" # command partially copied from iso in .disk/isofs diff --git a/states/salt/init.sls b/states/salt/init.sls index 6f378e4..65c36e8 100644 --- a/states/salt/init.sls +++ b/states/salt/init.sls @@ -1,28 +1,57 @@ -salt-minion: - pkg.installed: [] +{% set version = salt.pillar.get('versions:salt-minion', '3006.4') %} +include: + - helpers + +# Clean up obsolete or accidental installations +salt-cleanup: + pkg.purged: + - names: + - salt-minion + - salt-master + - salt-common + +/etc/salt/pki/minion: + file.directory: + - dir_mode: '0750' + - makedirs: True + - require: + - pkg: salt-cleanup + - require_in: + - cmd: salt-minion + +/etc/salt/minion.d/teckhost.conf: file.managed: - - name: /etc/salt/minion.d/teckhost.conf {% if salt.grains.get('bootstrap:devdir') and not salt.chroot.in_chroot() %} - source: salt://salt/saltlocal.conf {% else %} - source: salt://salt/saltsolo.conf {% endif %} - template: jinja + - dir_mode: '0750' + - makedirs: True - require: - - pkg: salt-minion + - pkg: salt-cleanup + - require_in: + - file: salt-minion salt-solo: - service.dead: - - name: salt-minion - - enable: False + pkg.installed: + - name: python3-venv - require: - - pkg: salt-minion + - pkg: salt-cleanup + cmd.run: + - name: 'python3 -m venv /opt/salt; /opt/salt/bin/pip3 install salt=={{ version }}' + - unless: '/opt/salt/bin/salt --version | grep {{ version }}' + - watch: + - pkg: salt-solo + - require: + - pkg: salt-solo cron.present: - - name: salt-call --local state.highstate + - name: /opt/salt/bin/salt-call --local state.highstate - identifier: highstate - special: '@hourly' - require: - - pkg: salt-minion + - pkg: salt-solo {% if salt.grains.get('bootstrap:devdir') %} ## diff --git a/test/vbox_create b/test/vbox_create index 6f8c6f8..cf5dcb4 100755 --- a/test/vbox_create +++ b/test/vbox_create @@ -25,7 +25,7 @@ show_usage() { cat <<-EOF Create a fresh VM for testing with - Usage: ./vbox_create [-h] + Usage: vbox_create [-h] Options: -i [path]${t}Location of installer ISO @@ -90,7 +90,12 @@ create_vm() { VBoxManage modifyvm "$TH_VMNAME" --cpus 4 VBoxManage modifyvm "$TH_VMNAME" --memory 1024 --vram 128 if [[ "$TH_BOOT" = 'efi' ]]; then - VBoxManage modifyvm "$TH_VMNAME" --firmware efi || return 1 + # Enable EFI + VBoxManage modifyvm "$TH_VMNAME" --firmware efi64 || return 1 + # Enable SecureBoot + VBoxManage modifynvram "$TH_VMNAME" inituefivarstore || return 1 + VBoxManage modifynvram "$TH_VMNAME" enrollmssignatures || return 1 + VBoxManage modifynvram "$TH_VMNAME" enrollorclpk || return 1 fi # Networking @@ -110,6 +115,12 @@ create_vm() { # Boot Order VBoxManage modifyvm "$TH_VMNAME" --boot1 dvd --boot2 disk --boot3 none --boot4 none || return 1 + # Recommended Settings + VBoxManage modifyvm "$TH_VMNAME" --graphicscontroller vmsvga --rtcuseutc on --pae off || return 1 + + # PATCH: https://bugs.debian.org/1036310 + VBoxManage modifyvm "$TH_VMNAME" --paravirtprovider legacy || return 1 + # Share Folder if [[ -n "$TH_DEVDIR" ]]; then VBoxManage sharedfolder add "$TH_VMNAME" --name devdir --hostpath "$TH_DEVDIR" || return 1