diff --git a/CHANGELOG.md b/CHANGELOG.md index c207ed5..c597fa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ First official snazzer release. ### Added ### ### Changed ### +- Snazzer doesn't treat all patterns in `/` like they were implicitly pre- and suffixed with `*`. + This behaviour now has to be specified explicitly. +- Snazzer now explicitly makes subvolumes absolute by prefixing with `/`. + - All entries in `/etc/snazzer/exclude.patterns` need to be either absolute + (start with `/`) or start with a `*`. + - Paths displayed by snazzer are now absolute as well. ### Deprecated ### ### Removed ### ### Fixed ### diff --git a/docs/examples/etc/snazzer/exclude.patterns b/docs/examples/etc/snazzer/exclude.patterns index 711a403..a0d2b30 100644 --- a/docs/examples/etc/snazzer/exclude.patterns +++ b/docs/examples/etc/snazzer/exclude.patterns @@ -1,6 +1,6 @@ -var/cache -var/lib/docker/* -.snapshots -tmp +/var/cache +/var/lib/docker/* +*/.snapshots +/tmp *backup* *secret* diff --git a/docs/snazzer-measure.md b/docs/snazzer-measure.md index 722640f..67584be 100644 --- a/docs/snazzer-measure.md +++ b/docs/snazzer-measure.md @@ -72,7 +72,7 @@ The output includes: Filename of newline separated list of shell glob patterns of subvolume pathnames which should be excluded from `snazzer --all` invocations; compatible with `--exclude-from` for **du** and **tar**. Examples of subvolume patterns to - exclude from regular snapshotting: \*secret\*, /var/cache, /var/lib/docker/btrfs, + exclude from regular snapshotting: \*secret\*, /var/cache, /var/lib/docker/\*, .snapshots. **NOTE:** `.snapshotz` is always excluded. Default: diff --git a/docs/snazzer.md b/docs/snazzer.md index 7943905..f5daff8 100644 --- a/docs/snazzer.md +++ b/docs/snazzer.md @@ -76,10 +76,8 @@ snapshots already measured by current hostname Filename of newline separated list of shell glob patterns of subvolume pathnames which should be excluded from `snazzer --all` invocations; compatible with `--exclude-from` for **du** and **tar**. Examples of subvolume patterns to - exclude from regular snapshotting: \*secret\*, var/cache, var/lib/docker/\*, - .snapshots. The patterns are matched against subvolumes as listed by - `btrfs subvolume list , without a leading /. - Note that **NOTE:** `.snapshotz` is always excluded. + exclude from regular snapshotting: \*secret\*, /var/cache, /var/lib/docker/\*, + \*/.snapshots. Note that **NOTE:** `.snapshotz` is always excluded. Default: SNAZZER_SUBVOLS_EXCLUDE_FILE="/etc/snazzer/exclude.patterns" @@ -117,7 +115,7 @@ snapshots already measured by current hostname `mkdir`: atimes always return with the current local time, which is obvioulsy different from one second to the next. So we have no hope of creating reproducible shasums or PGP signatures unless those directories are excluded - from our measurements of the snapshot. See also: + from our measurements of the snapshot. See also: [https://bugzilla.kernel.org/show\_bug.cgi?id=95201](https://bugzilla.kernel.org/show_bug.cgi?id=95201) # EXIT STATUS @@ -137,6 +135,7 @@ already in progress, check lock dir at /var/run/snazzer-measure.lock - 9. tried to display man page with a formatter which is not installed - 10. missing `snazzer-measure` or `snazzer-prune-candidates` from PATH - 11. missing `btrfs` command from PATH +- 12. syntax error in /etc/snazzer/exclude.patterns file. # SEE ALSO diff --git a/snazzer b/snazzer index b50d8d7..268db38 100755 --- a/snazzer +++ b/snazzer @@ -29,10 +29,10 @@ if ! $SUDO test -e "$SNAZZER_SUBVOLS_EXCLUDE_FILE"; then MISSING_SUBVOLS_EXCL_FILE="$SNAZZER_SUBVOLS_EXCLUDE_FILE" SNAZZER_SUBVOLS_EXCLUDE_FILE=$(mktemp) cat < "$SNAZZER_SUBVOLS_EXCLUDE_FILE" -var/cache -var/lib/docker/* -.snapshots -tmp +/var/cache +/var/lib/docker/* +*/.snapshots +/tmp *backup* *secret* HERE @@ -71,21 +71,30 @@ assert_btrfs_tools() { fi } +# function duplicated in snazzer-measure glob2grep_file() { FILE=$1 OUT=$(mktemp) + # check if every line starts with either / or *, if not, quit with a syntax error. + while read -r line; do + if ! echo "$line" | grep '^[/*]' >/dev/null; then + echo "SYNTAX ERROR: $1 contains line \"$line\" that starts with neither / nor *." >&2 + exit 12 + fi + done < "$1" + # first, escape $, . and ^ with a backslash so they're taken literally # then, extend * to .* to emulate shell globbing. # finally, add ^ and $ to the line ends so the lines are not evaluated as *line* sed 's|[$.^]|\\&|g' "$FILE" | sed 's/\*/\.*/g' | sed 's/^/\^/g' | sed 's/$/\$/g' > "$OUT" - echo "$OUT" } # This should only operate on a real mount(8) filesystem mountpoint, so call # assert_mountpoint before calling list_subvolumes. +# the DIR argument is assumed to be without a trailing slash, i.e. "/" or "/mnt" but not "/mnt/" list_subvolumes() { DIR=$1 DO_INVERT=$2 @@ -94,21 +103,30 @@ list_subvolumes() { else GREP="grep -v" fi + if [ "$DIR" != "/" ]; then - DIR="$DIR/" + PREFIX="$DIR" + else + PREFIX="" fi - EXCL_FILE=$(glob2grep_file "$SNAZZER_SUBVOLS_EXCLUDE_FILE") + + EXCL_FILE=$(glob2grep_file "$SNAZZER_SUBVOLS_EXCLUDE_FILE") || exit $? $SUDO btrfs subvolume list -t "$DIR" | tail -n+3 | \ - sed 's/^[0-9]*[ \t]*[0-9]*[ \t]*[0-9]*[ \t]*//g' | \ + # delete all columns except path, prefix the paths with a / + sed 's|^[0-9]*[ \t]*[0-9]*[ \t]*[0-9]*[ \t]*|/|g' | \ + # add / line, as the root subvolume is not included in btrfs subvolume list + (echo "/"; cat) | \ $GREP -f "$EXCL_FILE" | grep -v '\.snapshotz' | \ - while read -r SUBVOL; do echo "${DIR}$SUBVOL"; done + while read -r SUBVOL; do echo "${PREFIX}$SUBVOL"; done rm "$EXCL_FILE" } report_subvols_excluded() { DIR=$1 assert_mountpoint "$DIR" - NUM=$(list_subvolumes "$DIR" --excluded | wc -l) + SUBVOLS=$(list_subvolumes "$DIR" --excluded) || exit $? + # command substitution removes trailing newlines, we have to add it again for wc -l to give correct results. + NUM=$(printf "%s\n" "$SUBVOLS" | wc -l) if [ "$NUM" != "0" ]; then if [ "$DRY_RUN" = "1" ]; then printf "#"; fi @@ -123,7 +141,7 @@ report_subvols_excluded() { # subvols later on. # SMELL: what if a subvol is mounted some place other than its path name? list_btrfs_mountpoints() { - EXCL_FILE=$(glob2grep_file "$SNAZZER_SUBVOLS_EXCLUDE_FILE") + EXCL_FILE=$(glob2grep_file "$SNAZZER_SUBVOLS_EXCLUDE_FILE") || exit $? df -t btrfs 2>/dev/null | tail -n+2 | awk '{ print $1 }' | sort | uniq | \ while read -r DEV; do df --output=target "$DEV" | tail -n+2 ; done | \ grep -v '\.snapshotz' | grep -v -f "$EXCL_FILE" || true @@ -521,27 +539,23 @@ do_mountpoint() { if [ "$MOUNTPOINT" != "/" ]; then MOUNTPOINT=$(echo "$MOUNTPOINT" | sed 's|/$||g') fi - do_multiple "$DO_ACTION" "$MOUNTPOINT" report_subvols_excluded "$MOUNTPOINT" >> "$TMP_EXCL" - report_snapshot_dirs "$MOUNTPOINT" >> "$TMP_DIRS" assert_mountpoint "$MOUNTPOINT" - list_subvolumes "$MOUNTPOINT" | { - while read -r SUBVOL; do - do_multiple "$DO_ACTION" "$SUBVOL" - report_snapshot_dirs "$SUBVOL" >> "$TMP_DIRS" - done - } + SUBVOLS=$(list_subvolumes "$MOUNTPOINT") || exit $? + printf "%s\n" "$SUBVOLS" | while read -r SUBVOL; do + do_multiple "$DO_ACTION" "$SUBVOL" + report_snapshot_dirs "$SUBVOL" >> "$TMP_DIRS" + done } do_mountpoints() { TMP_EXCL=$(mktemp) TMP_DIRS=$(mktemp) if [ "$#" = 0 ]; then - list_btrfs_mountpoints | { - while read -r MOUNTPOINT; do - do_mountpoint "$MOUNTPOINT" "$TMP_EXCL" "$TMP_DIRS" - done - } + MOUNTPOINTS=$(list_btrfs_mountpoints) || exit $? + printf "%s\n" "$MOUNTPOINTS" | while read -r MOUNTPOINT; do + do_mountpoint "$MOUNTPOINT" "$TMP_EXCL" "$TMP_DIRS" + done else for MOUNTPOINT in "$@"; do do_mountpoint "$MOUNTPOINT" "$TMP_EXCL" "$TMP_DIRS" @@ -682,10 +696,8 @@ snapshots already measured by current hostname Filename of newline separated list of shell glob patterns of subvolume pathnames which should be excluded from C invocations; compatible with C<--exclude-from> for B and B. Examples of subvolume patterns to -exclude from regular snapshotting: *secret*, var/cache, var/lib/docker/*, -.snapshots. The patterns are matched against subvolumes as listed by -C>, without a leading /. -Note that B C<.snapshotz> is always excluded. +exclude from regular snapshotting: *secret*, /var/cache, /var/lib/docker/*, +*/.snapshots. Note that B C<.snapshotz> is always excluded. Default: SNAZZER_SUBVOLS_EXCLUDE_FILE="/etc/snazzer/exclude.patterns" @@ -727,7 +739,7 @@ these empty directories behave differently to empty directories created with C: atimes always return with the current local time, which is obvioulsy different from one second to the next. So we have no hope of creating reproducible shasums or PGP signatures unless those directories are excluded -from our measurements of the snapshot. See also: +from our measurements of the snapshot. See also: L =back @@ -761,6 +773,8 @@ already in progress, check lock dir at /var/run/snazzer-measure.lock =item 11. missing C command from PATH +=item 12. syntax error in /etc/snazzer/exclude.patterns file. + =back =head1 SEE ALSO diff --git a/snazzer-measure b/snazzer-measure index c3e06b0..15de62c 100755 --- a/snazzer-measure +++ b/snazzer-measure @@ -167,21 +167,35 @@ CMD echo "" } + +# function duplicated in snazzer glob2grep_file() { FILE=$1 OUT=$(mktemp) - sed 's|[$.^]|\\&|g' "$FILE" | sed 's/\*/\.*/g' > "$OUT" + # check if every line starts with either / or *, if not, quit with a syntax error. + while read -r line; do + if ! echo "$line" | grep '^[/*]' >/dev/null; then + echo "SYNTAX ERROR: $1 contains line \"$line\" that starts with neither / nor *." >&2 + exit 12 + fi + done < "$1" + + # first, escape $, . and ^ with a backslash so they're taken literally + # then, extend * to .* to emulate shell globbing. + # finally, add ^ and $ to the line ends so the lines are not evaluated as *line* + sed 's|[$.^]|\\&|g' "$FILE" | sed 's/\*/\.*/g' | sed 's/^/\^/g' | sed 's/$/\$/g' > "$OUT" echo "$OUT" } + assert_gpg_secring_excluded() { if [ "$(gpg2 --list-secret-keys | wc -l)" = "0" ]; then echo "ERROR: no gpg2 --list-secret-keys" >&2 exit 10 fi - EXCL_FILE=$( glob2grep_file "$SNAZZER_SUBVOLS_EXCLUDE_FILE" ) + EXCL_FILE=$(glob2grep_file "$SNAZZER_SUBVOLS_EXCLUDE_FILE") || exit $? N=$(gpg2 --list-secret-keys | grep '^/' | xargs readlink -f | \ grep -v -f "$EXCL_FILE" | \ (xargs --no-run-if-empty df -t btrfs 2>/dev/null) | wc -l) @@ -385,7 +399,7 @@ time+offset C Filename of newline separated list of shell glob patterns of subvolume pathnames which should be excluded from C invocations; compatible with C<--exclude-from> for B and B. Examples of subvolume patterns to -exclude from regular snapshotting: *secret*, /var/cache, /var/lib/docker/btrfs, +exclude from regular snapshotting: *secret*, /var/cache, /var/lib/docker/*, .snapshots. B C<.snapshotz> is always excluded. Default: diff --git a/tests/data/exclude.patterns b/tests/data/exclude.patterns index 711a403..a0d2b30 100644 --- a/tests/data/exclude.patterns +++ b/tests/data/exclude.patterns @@ -1,6 +1,6 @@ -var/cache -var/lib/docker/* -.snapshots -tmp +/var/cache +/var/lib/docker/* +*/.snapshots +/tmp *backup* *secret* diff --git a/tests/data/exclude.patterns.error b/tests/data/exclude.patterns.error new file mode 100644 index 0000000..19b69ee --- /dev/null +++ b/tests/data/exclude.patterns.error @@ -0,0 +1,7 @@ +/var/cache +/var/lib/docker/* +*/.snapshots +/tmp +foo +*backup* +*secret* diff --git a/tests/snazzer.bats b/tests/snazzer.bats index aaf54a3..dbdc3da 100755 --- a/tests/snazzer.bats +++ b/tests/snazzer.bats @@ -47,6 +47,13 @@ expected_snapshots_raw() { [ "$status" = "0" ] } +@test "snazzer --all check excludefile syntax" { + SNAZZER_SUBVOLS_EXCLUDE_FILE=$BATS_TEST_DIRNAME/data/exclude.patterns.error + run snazzer --all --dry-run "$MNT" + # 12 means that snazzer detected the errors in the file + [ "$status" = "12" ] +} + @test "snazzer --all [mountpoint]" { run snazzer --all "$MNT" expected_snapshots | sort > $(expected_file)