circular-snapshot

#!/bin/bash

# create circular-buffer snapshots
# reads files/folders to back up from /etc/circular-snapshot.txt by default
# or specify with -s,--source
# (one file/folder per line; # is a comment)

# Written by Zachary Robinson '24 (robinsonz)
# Last updated 2021-04-08

# -f, --filter can point to an rsync filter file, see
# https://man7.org/linux/man-pages/man1/rsync.1.html#FILTER_RULES

# -r, --rotations can be used to specify the total number of backups to keep (default 3)

# -t, --scripts can be used to specify a file containing scripts to run
# the script file should specify on each line the script to run
#
# the script will be run in a temp directory and any files in the directory
# after running will be synced to an output folder in <bkp-root>/script-backup
# equal to the name of the script minus any extensions

# -n, --remote makes rsync connect to the given remote over ssh

# exit on any command error
set -e

PROG_NAME="${0##*/}"

usage()
{
  echo "usage: $PROG_NAME [-s | --sources file-list] [-f | --filter filter-file]
                         [-t | --scripts script-list] [-r | --rotations num-rots]
                         [-n | --remote remote-addr]
                         <level> <backup-dir> [source1 source2 ...]"
  exit 2
}

## parse initial command line args

PARSED_ARGS=$(getopt -n "$PROG_NAME" -o s:f:r:t:n: -l sources:,filter:,rotations:,scripts:,remote: -- "$@")

VALID=$?
if [[ "$VALID" != "0" ]]; then
  usage
fi

eval set -- "$PARSED_ARGS"

SOURCES_DEFAULT="/etc/circular-snapshot.txt"
SOURCES="$SOURCES_DEFAULT"
FILTER=unset
SCRIPTS=unset
# 0-indexed; 3 = $LEVEL.0 through $LEVEL.3 is kept.
# must be nonzero, or the --link-dest option in rsync will break
ROT=3
REMOTE=unset

while :
do
  case $1 in
    -s | --sources) SOURCES="$2" ; shift 2 ;;
    -f | --filter) FILTER="$2" ; shift 2 ;;
    -r | --rotations) ROT="$2" ; shift 2 ;;
    -t | --scripts) SCRIPTS="$2" ; shift 2 ;;
    -n | --remote) REMOTE="$2:" ; shift 2 ;; # colon here for the host
    --) shift; break ;;
    *) echo "$PROG_NAME: unexpected option $1 - this should not happen" ; usage ;;
  esac
done

# make sure we have enough command line arguments

if [[ "$#" -lt 2 ]]; then
  usage
fi

LEVEL="$1"
shift
BACKUP_DIR="$1"
shift

if [[ -e "$BACKUP_DIR/circular-backup.lock" ]] ; then
  echo "$PROG_NAME: error: another instance of this script is currently backing
up to $BACKUP_DIR. Canceling backup to avoid conflicts. If you're sure there
are no other instances of this script, you can run rm
$BACKUP_DIR/circular-backup.lock."
  exit 1
fi

touch "$BACKUP_DIR/circular-backup.lock"

echo "$PROG_NAME: starting backup"

# array of files/folders we want to back up
declare -a backups

# read in from source list if it exists
if [[ -e "$SOURCES" ]]; then
  # this is some magical bash bullshit that I don't understand
  # but it reads each line and adds every non-blank line that doesn't start
  # with "#" to the backups array
  sourcetext=$(cat "$SOURCES")
  while IFS= read -r line ; do
    if [[ -n "$line" && "$line" != \#* ]]; then
      backups+=("$line")
    fi
  done <<< "$sourcetext"
elif [[ "$SOURCES" != "$SOURCES_DEFAULT" ]]; then
  echo "$PROG_NAME: warning: sources file $SOURCES does not exist"
fi

# read in from CLI
for item in "$@"; do
  backups+=("$item")
done

# check that we have something to work with; exit if not
if [[ ${#backups[@]} -eq 0 ]]; then
  echo "$PROG_NAME: error: $SOURCES does not exist or is empty and no sources were specified on the command line"
  exit 3
fi

# rotate the backups

# sliiiiiide to the left
if [[ -e "$BACKUP_DIR/$LEVEL.$ROT" ]]; then
  mv "$BACKUP_DIR/$LEVEL.$ROT" "$BACKUP_DIR/$LEVEL.0.new"
fi

for ((i=$ROT;i>0;i--)); do
  if [[ -e "$BACKUP_DIR/$LEVEL.$((i - 1))" ]]; then
    mv "$BACKUP_DIR/$LEVEL.$((i - 1))" "$BACKUP_DIR/$LEVEL.$i"
  fi
done

# sliiiiiiide to the right
if [[ -e "$BACKUP_DIR/$LEVEL.0.new" ]]; then
  mv "$BACKUP_DIR/$LEVEL.0.new" "$BACKUP_DIR/$LEVEL.0"
else
  mkdir -p "$BACKUP_DIR/$LEVEL.0"
fi

# this is EXTREMELY stupid but it's the only way to get rsync to delete files
# that have been removed from the backup set. Basically, we build a list of
# files and all of their subdirectories and use them to construct an rsync
# filter file (in addition to any filters that might be user-specified). Then
# we rsync the entire tree with everything excluded except for the specific
# files we want.

filterfile=`mktemp /tmp/circular-backup-XXXXXXXXXX`

if [[ -e "$FILTER" ]]; then
  cat "$FILTER" >> "$filterfile"
fi

for item in "${backups[@]}"; do
  subdir="$item/"
  absolute_subdir=
  if [[ -d "$item" ]]; then
    absolute_subdir=$(cd "$item"; pwd)
  else
    absolute_subdir=$(realpath "$item")
    absolute_subdir="${absolute_subdir%/*}"
    echo "+ $absolute_subdir/" >> "$filterfile"
  fi
  while true; do
    subdir="$absolute_subdir/.."
    absolute_subdir=$(cd "$subdir"; pwd)
    if [[ "$absolute_subdir" == "/" ]]; then break; fi
    echo "+ $absolute_subdir/" >> "$filterfile"
  done
  if [[ -d "$item" ]]; then
    echo "+ $(cd "$item"; pwd)/***" >> "$filterfile"
  else
    echo "+ $(realpath "$item")" >> "$filterfile"
  fi
done

echo "- *" >> "$filterfile"

rsync -aR --delete --delete-excluded --numeric-ids --filter="merge $filterfile" --link-dest="../$LEVEL.1/" / "$REMOTE$BACKUP_DIR/$LEVEL.0/"

rm "$filterfile"

echo "$PROG_NAME: file backup complete"

# ok, now we do scripts

# FIXME: right now I think the rsync step above deletes all the script outputs
# and we end up resyncing them. this isn't really a big deal as long as they
# don't change so they can get hardlinked but it's annoying and inefficient
# if script output changes often
if [[ -e "$SCRIPTS" ]]; then
  echo "$PROG_NAME: backing up scripts"
  declare -a scriptexecs

  scripttext=$(cat "$SCRIPTS")
  while IFS= read -r line ; do
    if [[ -n "$line" && "$line" != \#* ]]; then
      scriptexecs+=("$line")
    fi
  done <<< "$scripttext"

  for script in "${scriptexecs[@]}"; do
    tempdir=`mktemp -d "/tmp/circular-backup-script-XXXXXXXXXX"`

    echo "$PROG_NAME: running $script"
    (cd "$tempdir"; eval "$script")

    scriptname=$(basename "$script")
    destdir="$BACKUP_DIR/$LEVEL.0/script-backup/${scriptname##*/}"
    destdir="${destdir%.*}"

    echo "$PROG_NAME: copying result of $script"
    mkdir -p "$destdir"
    rsync -a --delete --numeric-ids --link-dest="$(realpath "$BACKUP_DIR")/$LEVEL.1/script-backup/$(basename `realpath "$destdir"`)" "$tempdir/" "$destdir/"
    rm -rf "$tempdir"
  done
fi

touch "$BACKUP_DIR/$LEVEL.0"

rm "$BACKUP_DIR/circular-backup.lock"
echo "$PROG_NAME: backup complete"