From c8ba41a4cd5754e9b88931d6172cdd6bf7374153 Mon Sep 17 00:00:00 2001
From: Martin Fischer <martin@push-f.com>
Date: Tue, 1 Feb 2022 15:51:07 +0100
Subject: [PATCH] spec_tests.py: introduce --track option

The --track option is meant to help developers impement new CommonMark
parsers from scratch. When you start from scratch, you start out with
hundreds of test failures (for everything you haven't impemented yet).

For that scenario it's more helpful to only print the test cases
that changed compared to the last run. When you run spec_tests.py with
--track then fix one test case and run it again it only prints e.g.

  Example 107 (lines 1750-1757) Indented code blocks
  fixed!

omitting all the test failures that haven't changed.
---
 test/spec_tests.py | 65 ++++++++++++++++++++++++++++++++--------------
 1 file changed, 46 insertions(+), 19 deletions(-)

diff --git a/test/spec_tests.py b/test/spec_tests.py
index 766df5cd..f8cb7092 100755
--- a/test/spec_tests.py
+++ b/test/spec_tests.py
@@ -30,6 +30,8 @@
             default=False, help='filter stdin through normalizer for testing')
     parser.add_argument('-n', '--number', type=int, default=None,
             help='only consider the test with the given number')
+    parser.add_argument('--track', metavar='path',
+            help='track which test cases pass/fail in the given JSON file and only report changes')
     args = parser.parse_args(sys.argv[1:])
 
 def out(str):
@@ -38,12 +40,13 @@ def out(str):
 def print_test_header(test):
     out("Example %d (lines %d-%d) %s\n" % (test['example'], test['start_line'], test['end_line'], test['section']))
 
-def do_test(test, normalize):
+def do_test(test, normalize, prev_result):
     [retcode, actual_html_bytes, err] = cmark.to_html(test['markdown'])
     if retcode != 0:
-        print_test_header(test)
-        out("program returned error code %d\n" % retcode)
-        sys.stdout.buffer.write(err)
+        if prev_result != 'error':
+            print_test_header(test)
+            out("program returned error code %d\n" % retcode)
+            sys.stdout.buffer.write(err)
         return 'error'
 
     expected_html = test['html']
@@ -51,12 +54,13 @@ def do_test(test, normalize):
     try:
         actual_html = actual_html_bytes.decode('utf-8')
     except UnicodeDecodeError as e:
-        print_test_header(test)
-        out(test['markdown'] + '\n')
-        out("Unicode error: " + str(e) + '\n')
-        out("Expected: " + repr(expected_html) + '\n')
-        out("Got:      " + repr(actual_html_bytes) + '\n')
-        out('\n')
+        if prev_result != 'fail':
+            print_test_header(test)
+            out(test['markdown'] + '\n')
+            out("Unicode error: " + str(e) + '\n')
+            out("Expected: " + repr(expected_html) + '\n')
+            out("Got:      " + repr(actual_html_bytes) + '\n')
+            out('\n')
         return 'fail'
 
     if normalize:
@@ -64,16 +68,21 @@ def do_test(test, normalize):
         expected_html = normalize_html(expected_html) + '\n'
 
     if actual_html != expected_html:
-        print_test_header(test)
-        out(test['markdown'] + '\n')
-        expected_html_lines = expected_html.splitlines(True)
-        actual_html_lines = actual_html.splitlines(True)
-        for diffline in unified_diff(expected_html_lines, actual_html_lines,
-                        "expected HTML", "actual HTML"):
-            out(diffline)
-        out('\n')
+        if prev_result != 'fail':
+            print_test_header(test)
+            out(test['markdown'] + '\n')
+            expected_html_lines = expected_html.splitlines(True)
+            actual_html_lines = actual_html.splitlines(True)
+            for diffline in unified_diff(expected_html_lines, actual_html_lines,
+                            "expected HTML", "actual HTML"):
+                out(diffline)
+            out('\n')
         return 'fail'
 
+    if prev_result and prev_result != 'pass':
+        print_test_header(test)
+        print('fixed!')
+
     return 'pass'
 
 def get_tests(specfile):
@@ -139,8 +148,26 @@ def get_tests(specfile):
         skipped = len(all_tests) - len(tests)
         cmark = CMark(prog=args.program, library_dir=args.library_dir)
         result_counts = {'pass': 0, 'fail': 0, 'error': 0, 'skip': skipped}
+
+        previous = {}
+
+        if args.track:
+            try:
+                with open(args.track) as f:
+                    previous = json.load(f)
+            except FileNotFoundError:
+                pass
+
+        results = {}
+
         for test in tests:
-            result = do_test(test, args.normalize)
+            result = do_test(test, args.normalize, previous.get(str(test['example'])))
             result_counts[result] += 1
+            results[test['example']] = result
+
+        if args.track:
+            with open(args.track, 'w') as f:
+                json.dump(results, f)
+
         out("{pass} passed, {fail} failed, {error} errored, {skip} skipped\n".format(**result_counts))
         exit(result_counts['fail'] + result_counts['error'])