Implement testing - also on travis.ci

.gitignore

@@ -0,0 +1 @@
+tests/all.log

.travis.yml

@@ -0,0 +1,16 @@
+language: python
+# Use containers instead of VMs
+sudo: false
+
+addons:
+    apt:
+        packages:
+        - libnss3
+
+python:
+    - "2.7"
+    - "3.4"
+    - "3.5"
+
+script:
+    - cd tests && ./run_all -v

README.md

@@ -61,8 +61,24 @@ Exporting overwrites existing passwords without warning. Make sure you have a
 backup or are using the `pass git` functionality.
 
 
+#### Testing
+
+If you wish to run the testsuite locally chdir into `tests/` and run `./run_all`
+
+If any test fails on your system please ensure `libnss` is installed.
+
+If afterwards tests still fail, re-run with `./run_all -v` and file a bug
+report including this output. Please include some information about your
+system, including linux distribution, and version of libnss/firefox.
+
+It is much appreciated.
+
+
 #### Changelog
 
+##### 0.5.1
+- Testsuite is now in place
+
 ##### 0.5
 - Fix encoding/decoding problems in python 2 - #5
 - Exporting passwords to *pass* now includes the login name

tests/bash_tap.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# For more information, see https://github.com/wbsch/bash_tap
+# Subject to the MIT License. See LICENSE file or http://opensource.org/licenses/MIT
+# Copyright (c) 2015 Wilhelm Schürmann
+
+function bashtap_on_error {
+    # A command in the parent script failed, interpret this as a test failure.
+    # $bashtap_line contains the last executed line, or an error.
+    echo -n "$bashtap_output"
+    echo "not ok 1 - ${bashtap_line}"
+    bashtap_clean_tmpdir
+}
+
+function bashtap_run_testcase {
+    # Run each line in the parent script up to the first "exit".
+    bashtap_output=""
+    while IFS= read -r bashtap_line && [ "${bashtap_line:0:4}" != "exit" ]; do
+        # Skip shebang.
+        if [ "${bashtap_line:0:2}" == "#!" ]; then
+            continue
+        fi
+
+        # Avoid recursively sourcing this script, and any helper scripts.
+        if [[ "$bashtap_line" =~ ^(\.|source).*(/|[[:blank:]])bash_tap[^/]*$ ]]; then
+            continue
+        fi
+
+        # Include comments as-is.
+        if [ "${bashtap_line:0:1}" == "#" ]; then
+            bashtap_output+="$bashtap_line"
+            bashtap_output+=$'\n'
+            continue
+        fi
+
+        # Run file line by line.
+        if [ ! -z "$bashtap_line" ] && [ "${bashtap_line:0:2}" != "#!" ]; then
+            bashtap_output+="# $ $bashtap_line"
+            bashtap_output+=$'\n'
+            local cmd_output
+            local cmd_ret
+
+            eval "$bashtap_line" &> bashtap_out_tmp
+            cmd_ret=$?
+            cmd_output="$(sed 's/^/# >>> /' < bashtap_out_tmp)"
+
+            if [ ! -z "$cmd_output" ]; then
+                bashtap_output+="$cmd_output"
+                bashtap_output+=$'\n'
+            fi
+            if [ "$cmd_ret" -ne 0 ]; then
+                exit $cmd_ret
+            fi
+        fi
+    done <"$bashtap_org_script"
+}
+
+function bashtap_clean_tmpdir {
+    if [ ! -z "$bashtap_tmpdir" ] && [ -d "$bashtap_tmpdir" ]; then
+        cd "$bashtap_org_pwd"
+        rm -rf "$bashtap_tmpdir"
+    fi
+    if [ -f bashtap_out_tmp ]; then
+        rm bashtap_out_tmp
+    fi
+}
+
+function bashtap_get_absolute_path {
+    # NOTE: No actual thought put into this. Might break. Horribly.
+    # Using this instead of readlink/realpath for OSX compatibility.
+    echo $(cd "$(dirname "$1")" && pwd)/$(basename "$1")
+}
+
+
+bashtap_org_pwd=$(pwd)
+bashtap_org_script=$(bashtap_get_absolute_path "$0")
+
+if [ "${0:(-2)}" == ".t" ] || [ "$1" == "-t" ]; then
+    # Make sure any failing commands are caught.
+    set -e
+    set -o pipefail
+
+    # TAP header. Hardcoded number of tests, 1.
+    echo "1..1"
+
+    # Output TAP failure on early exit.
+    trap bashtap_on_error EXIT
+
+    # The different calls to mktemp are necessary for OSX compatibility.
+    bashtap_tmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'bash_tap')
+    if [ ! -z "$bashtap_tmpdir" ]; then
+        cd "$bashtap_tmpdir"
+    else
+        bashtap_line="Unable to create temporary directory."
+        exit 1
+    fi
+
+    # Scripts sourced before bash_tap.sh may declare this function.
+    if declare -f bashtap_setup >/dev/null; then
+        bashtap_setup
+    fi
+
+    # Run test file interpreting failing commands as a test failure.
+    bashtap_run_testcase && echo "ok 1"
+
+    # Since we're in a sourced file and just ran the parent script,
+    # exit without running it a second time.
+    trap - EXIT
+    bashtap_clean_tmpdir
+    exit
+else
+    if declare -f bashtap_setup >/dev/null; then
+        bashtap_setup
+    fi
+fi

tests/bash_tap_fd.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# This file only contains helper functions for making testing easier.
+# The magic happens in bash_tap.sh sourced at the end of this file.
+#
+# Subject to the MIT License. See LICENSE file or http://opensource.org/licenses/MIT
+# Copyright (c) 2015-2016 Wilhelm Schürmann
+
+function get_password {
+    echo "$(cat ${bashtap_org_pwd}/test_data/master_password)"
+}
+
+function get_script {
+    echo "/usr/bin/env python ${bashtap_org_pwd}/../firefox_decrypt.py"
+}
+
+function get_test_data {
+    echo "${bashtap_org_pwd}/test_data"
+}
+
+function get_user_data {
+    echo -e "$(cat ${bashtap_org_pwd}/test_data/users/${1}.user)"
+}
+
+# Include the base script that does the actual work.
+source bash_tap.sh

tests/direct_profile_20.t

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+source bash_tap_fd.sh
+
+PASSWD=$(get_password)
+CMD=$(get_script)
+TEST="$(get_test_data)/test_profile_firefox_20/"
+
+
+# Python 2 tests
+diff -u <(echo ${PASSWD} | ${CMD} ${TEST} | grep -C1 doesntexist) <(get_user_data "doesntexist")
+diff -u <(echo ${PASSWD} | ${CMD} ${TEST} | grep -C1 onemore) <(get_user_data "onemore")
+diff -u <(echo ${PASSWD} | ${CMD} ${TEST} | grep -C1 cömplex) <(get_user_data "complex")
+
+# vim: ai sts=4 et sw=4

tests/direct_profile_46.t

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+source bash_tap_fd.sh
+
+PASSWD=$(get_password)
+CMD=$(get_script)
+TEST="$(get_test_data)/test_profile_firefox_46/"
+
+
+# Python 2 tests
+diff -u <(echo ${PASSWD} | ${CMD} ${TEST} | grep -C1 doesntexist) <(get_user_data "doesntexist")
+diff -u <(echo ${PASSWD} | ${CMD} ${TEST} | grep -C1 onemore) <(get_user_data "onemore")
+diff -u <(echo ${PASSWD} | ${CMD} ${TEST} | grep -C1 cömplex) <(get_user_data "complex")
+
+# vim: ai sts=4 et sw=4

tests/problems

@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import sys
+import re
+import argparse
+from collections import defaultdict
+
+
+def color(text, c):
+    """
+    Add color on the keyword that identifies the state of the test
+    """
+    if sys.stdout.isatty():
+        clear = "\033[0m"
+
+        colors = {
+            "red": "\033[1m\033[91m",
+            "yellow": "\033[1m\033[93m",
+            "green": "\033[1m\033[92m",
+        }
+        return colors[c] + text + clear
+    else:
+        return text
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Report on test results")
+    parser.add_argument('--summary', action="store_true",
+                        help="Display only the totals in each category")
+    parser.add_argument('tapfile', default="all.log", nargs="?",
+                        help="File containing TAP output")
+    return parser.parse_args()
+
+
+def print_category(tests):
+    if not cmd_args.summary:
+        for key in sorted(tests):
+            print("%-32s %4d" % (key, tests[key]))
+
+
+def pad(i):
+    return " " * i
+
+
+if __name__ == "__main__":
+    cmd_args = parse_args()
+
+    errors = defaultdict(int)
+    skipped = defaultdict(int)
+    expected = defaultdict(int)
+    unexpected = defaultdict(int)
+    passed = defaultdict(int)
+
+    file = re.compile("^# (?:./)?(\S+\.t)(?:\.exe)?$")
+    timestamp = re.compile("^# (\d+(?:\.\d+)?) ==>.*$")
+    start = None
+    stop = None
+
+    with open(cmd_args.tapfile) as fh:
+        for line in fh:
+            if start is None:
+                # First line contains the starting timestamp
+                start = float(timestamp.match(line).group(1))
+                continue
+
+            match = file.match(line)
+            if match:
+                filename = match.group(1)
+
+            if line.startswith("ok "):
+                passed[filename] += 1
+
+            if line.startswith("not "):
+                errors[filename] += 1
+
+            if line.startswith("skip "):
+                skipped[filename] += 1
+
+            if line.startswith("# EXPECTED_FAILURE:"):
+                expected[filename] += 1
+
+            if line.startswith("# UNEXPECTED_SUCCESS:"):
+                unexpected[filename] += 1
+
+        # Last line contains the ending timestamp
+        stop = float(timestamp.match(line).group(1))
+
+    # Remove expected failures from the skipped tests category
+    for filename, value in expected.items():
+        if skipped[filename] == value:
+            del skipped[filename]
+        else:
+            skipped[filename] -= value
+
+    v = "{0:>5d}"
+    passed_str = "Passed:" + pad(24)
+    passed_int = v.format(sum(passed.values()))
+    error_str = "Failed:" + pad(24)
+    error_int = v.format(sum(errors.values()))
+    unexpected_str = "Unexpected successes:" + pad(10)
+    unexpected_int = v.format(sum(unexpected.values()))
+    skipped_str = "Skipped:" + pad(23)
+    skipped_int = v.format(sum(skipped.values()))
+    expected_str = "Expected failures:" + pad(13)
+    expected_int = v.format(sum(expected.values()))
+    runtime_str = "Runtime:" + pad(20)
+    runtime_int = "{0:>8.2f} seconds".format(stop - start)
+
+    if cmd_args.summary:
+        print(color(passed_str, "green"), passed_int)
+        print(color(error_str, "red"), error_int)
+        print(color(unexpected_str, "red"), unexpected_int)
+        print(color(skipped_str, "yellow"), skipped_int)
+        print(color(expected_str, "yellow"), expected_int)
+        print(runtime_str, runtime_int)
+
+    else:
+        print(color(error_str, "red"))
+        print_category(errors)
+        print()
+        print(color(unexpected_str, "red"))
+        print_category(unexpected)
+        print()
+        print(color(skipped_str, "yellow"))
+        print_category(skipped)
+        print()
+        print(color(expected_str, "yellow"))
+        print_category(expected)
+
+    # If we encoutered any failures, return non-zero code
+    sys.exit(1 if int(error_int) or int(unexpected_int) else 0)

tests/profile_ini_20.t

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+source bash_tap_fd.sh
+
+PASSWD=$(get_password)
+CMD=$(get_script)
+TEST="$(get_test_data)"
+PAYLOAD="1\n${PASSWD}"
+
+
+# Python 2 tests
+diff -u <(echo -e ${PAYLOAD} | ${CMD} ${TEST} | grep -C1 doesntexist) <(get_user_data "doesntexist")
+diff -u <(echo -e ${PAYLOAD} | ${CMD} ${TEST} | grep -C1 onemore) <(get_user_data "onemore")
+diff -u <(echo -e ${PAYLOAD} | ${CMD} ${TEST} | grep -C1 cömplex) <(get_user_data "complex")
+
+# vim: ai sts=4 et sw=4

tests/profile_ini_46.t

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+source bash_tap_fd.sh
+
+PASSWD=$(get_password)
+CMD=$(get_script)
+TEST="$(get_test_data)"
+PAYLOAD="2\n${PASSWD}"
+
+
+# Python 2 tests
+diff -u <(echo -e ${PAYLOAD} | ${CMD} ${TEST} | grep -C1 doesntexist) <(get_user_data "doesntexist")
+diff -u <(echo -e ${PAYLOAD} | ${CMD} ${TEST} | grep -C1 onemore) <(get_user_data "onemore")
+diff -u <(echo -e ${PAYLOAD} | ${CMD} ${TEST} | grep -C1 cömplex) <(get_user_data "complex")
+
+# vim: ai sts=4 et sw=4