snazzer-measure

#!/bin/sh

set -e
SNAZZER_VERSION=0.5

# Keep in sync with the POD!
default_snazzer_sig_func() {
    gpg2 --quiet --no-greeting --batch --use-agent --armor --detach-sign -
}
# Keep in sync with the POD!
if [ -z "$SNAZZER_SIG_ENABLE" ]; then
    SNAZZER_SIG_ENABLE=1
fi
# Keep in sync with the POD!
if [ -z "$SNAZZER_MEASUREMENTS_EXCLUDE_FILE" ]; then
    SNAZZER_MEASUREMENTS_EXCLUDE_FILE=".snapshot_measurements.exclude"
fi

# Keep in sync with the POD!
if [ -z "$SNAZZER_SUBVOLS_EXCLUDE_FILE" ]; then
    SNAZZER_SUBVOLS_EXCLUDE_FILE="/etc/snazzer/exclude.patterns"
fi

if [ "$(id -u)" = "0" ]; then
    SUDO=""
else
    SUDO="sudo"
fi

# An empty dummy file to replace exclusion files if necessary
EMPTY_TMP=$(mktemp -t empty_dummy.XXXXXX)

do_snazzer_sig() {
    if type snazzer_sig_func | grep -i function >/dev/null; then
        snazzer_sig_func
    else
        default_snazzer_sig_func
    fi
}

get_measurements_exclude_file() {
    DIR=$1
    EXCL_FILE="$DIR/$SNAZZER_MEASUREMENTS_EXCLUDE_FILE"

    if [ -f "$EXCL_FILE" ]; then
        echo "$EXCL_FILE"
    else
        echo "$EMPTY_TMP"
    fi
}

alloc_empty() {
    if [ ! -f "$EMPTY_TMP" ]; then
        touch "$EMPTY_TMP"
    fi
}

remove_empty() {
    if [ -f "$EMPTY_TMP" ]; then
        rm "$EMPTY_TMP"
    fi
}

escape_quote() {
    echo "$1" | sed "s|'|'\\\\''|g"
}


get_date() {
    if [ "$SNAZZER_USE_UTC" = "1" ]; then
        date -u +"%Y-%m-%dT%H%M%SZ"
    else
        date +"%Y-%m-%dT%H%M%S%z"
    fi
}

host_datetime() {
    WHAT=$1
    echo "> on $(hostname) at $(get_date), $WHAT:"
}

build_tar_cmd() {
    DIR=$1
    DIR_ESC=$(escape_quote "$DIR")
    EXCL=$(get_measurements_exclude_file "$DIR")
    EXCL_ESC=$(escape_quote "$EXCL")

    cat <<CMD
find '$DIR_ESC' -xdev -not -path '$DIR_ESC' -printf '%P\0' | \
LC_ALL=C sort -z | \
tar --no-recursion --one-file-system --preserve-permissions \
--numeric-owner --null --create --to-stdout \
--directory '$DIR_ESC' \
--files-from - \
--exclude-from '$EXCL_ESC'
CMD
}

do_tar() {
    DIR=$1
    LIST_TMP=$(mktemp)
    SORT_TMP=$(mktemp)
    EXCL_FILE=$(get_measurements_exclude_file "$DIR")

    chmod 0700 "$LIST_TMP"
    # Keep tar/find out of the pipe in case they have errors
    $SUDO find "$DIR" -xdev -not -path "$DIR" -printf '%P\0' >"$LIST_TMP" && \
        LC_ALL=C sort -z "$LIST_TMP" > "$SORT_TMP" && \
        $SUDO tar --no-recursion --one-file-system --preserve-permissions \
            --numeric-owner --null --create --to-stdout \
            --directory "$DIR" \
            --files-from "$SORT_TMP" \
            --exclude-from "$EXCL_FILE" && \
        rm "$SORT_TMP" && \
        rm "$LIST_TMP"
}

measure_sha512sum() {
    DIR=$1

    host_datetime "sha512sum"
    cat <<CMD
($(build_tar_cmd "$DIR") | sha512sum -b)
$(sha512sum -b)
CMD
}

build_du_cmd() {
    DIR=$1
    DIR_ESC=$(escape_quote "$DIR")
    EXCL=$(get_measurements_exclude_file "$DIR")
    EXCL_ESC=$(escape_quote "$EXCL")

    cat <<CMD
(du -bs --one-file-system --exclude-from '$EXCL_ESC' '$DIR_ESC')
CMD
}

do_du() {
    DIR=$1
    EXCL_FILE=$(get_measurements_exclude_file "$DIR")
    EXCL_TMP=$(mktemp)
    # du doesn't understand relative exclude rules, so make them absolute
    # could use sed but that means complicated escapes to deal with hostile $DIR
    $SUDO cat "$EXCL_FILE" | while read EXCL_RULE
    do
        echo "$DIR/$EXCL_RULE" >> "$EXCL_TMP"
    done
    $SUDO du -bs --one-file-system --exclude-from "$EXCL_TMP" "$DIR"
    rm "$EXCL_TMP"
}

build_find_excl_switch() {
    DIR=$1
    EXCL_FILE=$(get_measurements_exclude_file "$DIR")

    sed "s/'/'\\\\''/g" "$EXCL_FILE" | while read RULE
    do
        printf " -path '*%s' -prune -o" "$RULE"
    done
}

dry_find() {
    DIR_ESC=$(echo "$1" | sed "s|'|'\\\\''|g")
    EXCL=$(build_find_excl_switch "$1")
    cat <<CMD
find '$DIR_ESC' -xdev -not -path '$DIR_ESC' -path '/tmp' -prune -o $EXCL -type f \
    -printf "%T@ '%P'\\\0" | sort -n | tail -1 | cut -f2- -d" " | \
    xargs -0 ls -la --time-style="+%Y-%m-%dT%H%M%S%z"
CMD
}

measure_latest() {
    FIND_CMD=$(dry_find "$1")
    host_datetime "find latest"
    echo "$FIND_CMD"
    echo ""
}

measure_du() {
    DIR=$1
    FAKE_DIR=$2
    host_datetime "du bytes"
    build_du_cmd "$FAKE_DIR"
    do_du "$DIR"
    echo ""
}

measure_tar_info() {
    host_datetime "tar info"
    TAR_VERSION=$(tar --version | head -n 1)
    TAR_DEFAULTS=$(tar --show-defaults)
    echo "$TAR_VERSION $TAR_DEFAULTS"
}

measure_gpg() {
    DIR=$1
    MEAS_FILE=$(basename "$DIR")
    HOST_DATETIME=$(host_datetime "gpg")
    echo "$HOST_DATETIME"
    cat <<CMD
(SIG=\$(mktemp) && grep -v '/,/' '$MEAS_FILE' | \
sed -n '/$HOST_DATETIME/,/-----END PGP SIGNATURE-----/ { /-----BEGIN PGP SIGNATURE-----/{x;d}; H }; \${x;p}' \
>"\$SIG" && $(build_tar_cmd "$DIR") | gpg2 --verify "\$SIG" - && rm "\$SIG")
CMD
    do_snazzer_sig
    echo ""
}

glob2grep_file() {
    FILE=$1
    OUT=$(mktemp)

    sed 's|[$.^]|\\&|g' "$FILE" | sed 's/\*/\.*/g' > "$OUT"

    echo "$OUT"
}

assert_gpg_secring_excluded() {
    if [ "$(gpg2 --list-secret-keys | wc -l)" = "0" ]; then
        echo "ERROR: no gpg2 --list-secret-keys" >&2
        exit 10
    fi
    EXCL_FILE=$( glob2grep_file "$SNAZZER_SUBVOLS_EXCLUDE_FILE" )
    N=$(gpg2 --list-secret-keys | grep '^/' | xargs readlink -f | \
        grep -v -f "$EXCL_FILE" | \
        (xargs --no-run-if-empty df -t btrfs 2>/dev/null) | wc -l)
    rm "$EXCL_FILE"
    #N=$(gpg2 --list-secret-keys | grep '^/' | xargs readlink -f | \
    #    grep -v -f /etc/snazzer/exclude.patterns | wc -l)
    GPG_SECRING=$(gpg2 --list-secret-keys | head -n 1)

    if [ "$N" -ne 0 ] && ( [ -z "$MY_KEYFILES_ARE_INVINCIBLE" ] || \
        [ "$MY_KEYFILES_ARE_INVINCIBLE" -ne "1" ]); then
        show_usage <<HERE
gpg secret key(s) are included in the default snazzer shapshots, including
(perhaps?) $GPG_SECRING!

Move the private keyfiles you wish to use for measurement signing to a subvolume
excluded in "$SNAZZER_SUBVOLS_EXCLUDE_FILE", and ensure that the gpg environment
and config is set correctly. Stubbornly refusing to shoot self in foot; if you
feel this is in error, try again with MY_KEYFILES_ARE_INVINCIBLE=1.
HERE
        exit 4
    fi
}

# tar invocation inspired by
# https://wiki.debian.org/ReproducibleBuilds/FileOrderInTarballs
measure() {
    MEAS_DIR=$1
    FAKE_DIR=$2

    alloc_empty

    if [ -z "$FAKE_DIR" ]; then
        FAKE_DIR="$MEAS_DIR"
    fi
    cat <<HERE
################################################################################
HERE
    measure_du "$MEAS_DIR" "$FAKE_DIR"
    #measure_latest "$MEAS_DIR"
    # Sometimes, it's mildly faster to run sha512sum and gpg signing in parallel
    # especially on I/O starved VMs.
    if [ "$SNAZZER_SIG_ENABLE" != "0" ]; then
        assert_gpg_secring_excluded
        # Put the fifo somewhere it can't be listed by anyone except us, and
        # especially set permissions so that it can only be read by us.
        TMP_DIR=$(mktemp -d)
        chmod 0700 "$TMP_DIR"
        TMP_FIFO=$(mktemp --tmpdir="$TMP_DIR" -u)
        TMP_RC=$(mktemp --tmpdir="$TMP_DIR")
        mkfifo --mode=0700 "$TMP_FIFO"
        SHA512_MEASUREMENT=$(mktemp --tmpdir="$TMP_DIR")
        GPG_MEASUREMENT=$(mktemp --tmpdir="$TMP_DIR")
        ( do_tar "$MEAS_DIR" || echo "$?" > "$TMP_RC" && exit "$?" ) | \
            tee "$TMP_FIFO" | \
            measure_sha512sum "$FAKE_DIR" > "$SHA512_MEASUREMENT" &
        sleep 1 # SMELLLLLL... avoid stupid race condition where it seems that
                # measure_gpg starts before tee does
        if [ -s "$TMP_RC" -a "$(cat "$TMP_RC")" != "0" ]; then
            exit "$(cat "$TMP_RC")"
        fi
        measure_gpg "$FAKE_DIR" < "$TMP_FIFO" > "$GPG_MEASUREMENT"
        # And if we put their respective output straight to stdout, we sometimes
        # get their output intermixedue
        cat "$SHA512_MEASUREMENT"
        cat "$GPG_MEASUREMENT"
        rm "$GPG_MEASUREMENT"
        rm "$SHA512_MEASUREMENT"
        rm "$TMP_RC"
        rm "$TMP_FIFO"
        rmdir "$TMP_DIR"
    else
        do_tar "$MEAS_DIR" | measure_sha512sum "$FAKE_DIR"
    fi
    measure_tar_info

    remove_empty
}

show_usage() {
    if [ -n "$1" ]; then
        pod2usage -exit 0 "$0" >&2
        printf "\nERROR: %s" "$1" >&2
    else
        pod2usage -exit 0 "$0"
    fi
}

case "$1" in
    -h | --help ) pod2usage -exit 0 "$0"; exit ;;
         --man ) pod2usage -exit 0 -verbose 3 "$0"; exit ;;
         --man-roff ) pod2man --release=$SNAZZER_VERSION "$0"; exit ;;
         --man-markdown )
         cat <<HERE | perl -Mstrict
if ( eval { require Pod::Markdown; 1; } ) {
    Pod::Markdown->new->filter('$0');
}
else {
    print STDERR "ERROR: --man-markdown requires Pod::Markdown\n\$@\n";
    exit 9;
}
HERE
             exit ;;
    -* ) echo "ERROR: Invalid argument '$1'" >&2 ; exit 1 ;;
esac

if [ -z "$1" ]; then
    show_usage "No path specified"
    exit 2
else
    measure "$1" "$2"
fi

<<__DNE__
__END__
=head1 NAME

snazzer-measure - report shasums & PGP signatures of content under a given path,
along with commands to reproduce or verify data is unchanged

=head1 SYNOPSIS

  snazzer-measure /measured/path [/reported/path] >> path_measurements

=head1 DESCRIPTION

Creates reproducible fingerprints of the given directory, along with commands
necessary (relative to measured path, or if supplied - the optional reported
path) to reproduce the measurement using only standard core GNU userland.

The output includes:

=over

=item * hostname and datetime of B<snazzer-measure> invocation

=item * C<du -bs> (bytes used)

=item * C<sha512sum> of the result of a reproducible tarball of the directory

=item * C<gpg2 --armor --sign> of the same

=item * instructions for reproducing or verifying each of the above

=item * C<tar --version>, C<tar --show-defaults>

=back

=head1 OPTIONS

=over

=item B<--help>: Brief help message

=item B<--man>: Full documentation

=item B<--man-roff>: Full documentation as *roff output, Eg:

    snazzer-measure --man-roff | nroff -man

=item B<--man-markdown>: Full documentation as markdown output, Eg:

    snazzer-measure --man-markdown > snazzer-measure-manpage.md

=back

=head1 ENVIRONMENT

=over

=item * snazzer_sig_func

Function generating PGP SIGNATURE text. Takes input from stdin, output to
stdout. Signatures can be disabled with L<SNAZZER_SIG_ENABLE>. Default:

    snazzer_sig_func() {
        gpg2 --quiet --no-greeting --batch --use-agent --armor --detach-sign -
    }

=item * SNAZZER_SIG_ENABLE

If set to 0, GPG signing is disabled and snazzer_sig_func() is not called.

=item * SNAZZER_MEASUREMENTS_EXCLUDE_FILE

A filename within the measured directory of a newline-separated list of shell
glob patterns to exclude from measurements. Default:

  SNAZZER_MEASUREMENTS_EXCLUDE_FILE=".snapshot_measurements.exclude"

=item * MY_KEYFILES_ARE_INVINCIBLE=1

Skip sanity check/abort when gpg secret key exists on a subvolume included in
default snazzer snapshots

=item * SNAZZER_USE_UTC

Use UTC times of the form C<YYYY-MM-DDTHHMMSSZ> instead of the default local
time+offset C<YYYY-MM-DDTHHMMSS+hhmm>

=item * SNAZZER_SUBVOLS_EXCLUDE_FILE

Filename of newline separated list of shell glob patterns of subvolume pathnames
which should be excluded from C<snazzer --all> invocations; compatible with
C<--exclude-from> for B<du> and B<tar>.  Examples of subvolume patterns to
exclude from regular snapshotting: *secret*, /var/cache, /var/lib/docker/btrfs,
.snapshots.  B<NOTE:> C<.snapshotz> is always excluded.
Default:

  SNAZZER_SUBVOLS_EXCLUDE_FILE="/etc/snazzer/exclude.patterns"

=back

=head2 sudo requirements

When running B<snazzer-measure> as a non-root user, certain commands will be
prefixed with C<sudo>. The following lines in C</etc/sudoers> or
C</etc/sudoers.d/snazzer> should suffice for scripted jobs such as cron (replace
C<measureuser> with the actual user name you are setting up for this task):

    measureuser ALL=(root:nobody) NOPASSWD:NOEXEC: \
        /bin/cat */.snapshotz/*/.snapshot_measurements.exclude, \
        /usr/bin/du -bs --one-file-system --exclude-from * */.snapshotz/*, \
        /usr/bin/find */.snapshotz/* \
            -xdev -not -path /*/.snapshotz/* -printf %P\\\\0, \
        /bin/tar --no-recursion --one-file-system --preserve-permissions \
            --numeric-owner --null --create --to-stdout \
            --directory */.snapshotz/* --files-from * \
            --exclude-from */.snapshotz/*/.snapshot_measurements.exclude

=head1 EXIT STATUS

B<snazzer-measure> will abort with an error message printed to STDERR and
non-zero exit status under the following conditions:

=over

=item 1. Invalid argument

=item 2. Path string not specified

=item 4. GPG signature would have been generated with a secret keyfile stored
in a subvolume which has not been excluded from default snazzer snapshots, see
L<IMPORTANT> below

=item 5. Expected the .snazzer_measurements.exclude file to contain an entry
for the .snazzer_measurements file

=back

=head1 IMPORTANT

Please note that if you are using this tool to gain some form of integrity
measurement (Eg. you want to detect tampering), GPG private keys used for the
signing operation mustn't be exposed among the directories being measured.

Put another way: it makes no sense to GPG-sign measurements of a directory if
those very same directories contain the GPG private key material required to
re-sign modifications made by anyone who happens to be looking.

=head1 BUGS AND LIMITATIONS

=over

=item * MY_KEYFILES_ARE_INVINCIBLE

The sanity check for location of GPG secret keyfile may be more annoying than
helpful on installations using smartcards, TPMs, or other methods of protecting
keyfiles - hence the B<MY_KEYFILES_ARE_INVINCIBLE> work-around.

=item * Temporary files

To avoid unnecessary I/O, gpg signing and shasumming are done in parallel from
the same C<tar --to-stdout> pipe; this involves creating a temporary named pipe
which is normally removed at the end of a successful run, but will be left
behind should a failure occur. These are randomly named with C<mktemp> and mode
0700, inside a C<mktemp -d> directory also with 0700 permissions.

=back

=head1 SEE ALSO

snazzer, snazzer-prune-candidates, snazzer-receive

=head1 AUTHOR

Paul Harvey <csirac2@gmail.com>, https://github.com/csirac2/snazzer

=head1 LICENSE AND COPYRIGHT

Copyright (c) 2015, Paul Harvey <csirac2@gmail.com> All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
=cut
__DNE__