From b509236030edc1a9dbfe6f6f7506f9e3925bb3d5 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tmgross@umich.edu>
Date: Thu, 14 Mar 2024 21:29:05 -0700
Subject: [PATCH] Add tests for runtime-ignored trials

Co-authored-by: Ben Widawsky <ben@bwidawsk.net>

(most of this is from Ben)
---
 tests/all_passing.rs | 130 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 99 insertions(+), 31 deletions(-)

diff --git a/tests/all_passing.rs b/tests/all_passing.rs
index 40be8d2..f23e136 100644
--- a/tests/all_passing.rs
+++ b/tests/all_passing.rs
@@ -1,44 +1,62 @@
 use common::{args, check};
-use libtest_mimic::{Trial, Conclusion};
+use libtest_mimic::{Completion, Conclusion, Trial};
 use pretty_assertions::assert_eq;
 
-use crate::common::{assert_reordered_log, conclusion_to_output, do_run};
+use crate::common::do_run;
 
 #[macro_use]
 mod common;
 
-
 fn tests() -> Vec<Trial> {
     vec![
         Trial::test("foo", || Ok(())),
         Trial::test("bar", || Ok(())),
         Trial::test("barro", || Ok(())),
+        // Passed
+        Trial::skippable_test("baz", || Ok(Completion::Completed)),
+        // Ignored with a reason
+        Trial::skippable_test("qux", || {
+            Ok(Completion::Ignored {
+                reason: "very valid reason".into(),
+            })
+        }),
+        // Ignored with no reason
+        Trial::skippable_test("quux", || Ok(Completion::Ignored { reason: "".into(), })),
     ]
 }
 
 #[test]
 fn normal() {
-    check(args([]), tests, 3,
+    check(
+        args([]),
+        tests,
+        6,
         Conclusion {
             num_filtered_out: 0,
-            num_passed: 3,
+            num_passed: 4,
             num_failed: 0,
-            num_ignored: 0,
+            num_ignored: 2,
             num_measured: 0,
         },
         "
             test foo   ... ok
             test bar   ... ok
             test barro ... ok
-        "
+            test baz   ... ok
+            test qux   ... ignored, very valid reason
+            test quux  ... ignored
+        ",
     );
 }
 
 #[test]
 fn filter_one() {
-    check(args(["foo"]), tests, 1,
+    check(
+        args(["foo"]),
+        tests,
+        1,
         Conclusion {
-            num_filtered_out: 2,
+            num_filtered_out: 5,
             num_passed: 1,
             num_failed: 0,
             num_ignored: 0,
@@ -50,9 +68,12 @@ fn filter_one() {
 
 #[test]
 fn filter_two() {
-    check(args(["bar"]), tests, 2,
+    check(
+        args(["bar"]),
+        tests,
+        2,
         Conclusion {
-            num_filtered_out: 1,
+            num_filtered_out: 4,
             num_passed: 2,
             num_failed: 0,
             num_ignored: 0,
@@ -65,12 +86,14 @@ fn filter_two() {
     );
 }
 
-
 #[test]
 fn filter_exact() {
-    check(args(["bar", "--exact"]), tests, 1,
+    check(
+        args(["bar", "--exact"]),
+        tests,
+        1,
         Conclusion {
-            num_filtered_out: 2,
+            num_filtered_out: 5,
             num_passed: 1,
             num_failed: 0,
             num_ignored: 0,
@@ -82,9 +105,12 @@ fn filter_exact() {
 
 #[test]
 fn filter_two_and_skip() {
-    check(args(["--skip", "barro", "bar"]), tests, 1,
+    check(
+        args(["--skip", "barro", "bar"]),
+        tests,
+        1,
         Conclusion {
-            num_filtered_out: 2,
+            num_filtered_out: 5,
             num_passed: 1,
             num_failed: 0,
             num_ignored: 0,
@@ -94,52 +120,89 @@ fn filter_two_and_skip() {
     );
 }
 
+#[test]
+fn filter_runtime_ignored() {
+    check(
+        args(["qux", "--exact"]),
+        tests,
+        1,
+        Conclusion {
+            num_filtered_out: 5,
+            num_passed: 0,
+            num_failed: 0,
+            num_ignored: 1,
+            num_measured: 0,
+        },
+        "test qux ... ignored, very valid reason",
+    );
+}
+
 #[test]
 fn skip_nothing() {
-    check(args(["--skip", "peter"]), tests, 3,
+    check(
+        args(["--skip", "peter"]),
+        tests,
+        6,
         Conclusion {
             num_filtered_out: 0,
-            num_passed: 3,
+            num_passed: 4,
             num_failed: 0,
-            num_ignored: 0,
+            num_ignored: 2,
             num_measured: 0,
         },
         "
             test foo   ... ok
             test bar   ... ok
             test barro ... ok
-        "
+            test baz   ... ok
+            test qux   ... ignored, very valid reason
+            test quux  ... ignored
+        ",
     );
 }
 
 #[test]
 fn skip_two() {
-    check(args(["--skip", "bar"]), tests, 1,
+    check(
+        args(["--skip", "bar"]),
+        tests,
+        4,
         Conclusion {
             num_filtered_out: 2,
-            num_passed: 1,
+            num_passed: 2,
             num_failed: 0,
-            num_ignored: 0,
+            num_ignored: 2,
             num_measured: 0,
         },
-        "test foo ... ok"
+        "
+            test foo  ... ok
+            test baz  ... ok
+            test qux  ... ignored, very valid reason
+            test quux ... ignored
+        ",
     );
 }
 
 #[test]
 fn skip_exact() {
-    check(args(["--exact", "--skip", "bar"]), tests, 2,
+    check(
+        args(["--exact", "--skip", "bar"]),
+        tests,
+        5,
         Conclusion {
             num_filtered_out: 1,
-            num_passed: 2,
+            num_passed: 3,
             num_failed: 0,
-            num_ignored: 0,
+            num_ignored: 2,
             num_measured: 0,
         },
         "
             test foo   ... ok
             test barro ... ok
-        "
+            test baz   ... ok
+            test qux   ... ignored, very valid reason
+            test quux  ... ignored
+        ",
     );
 }
 
@@ -148,10 +211,15 @@ fn terse_output() {
     let (c, out) = do_run(args(["--format", "terse"]), tests());
     assert_eq!(c, Conclusion {
         num_filtered_out: 0,
-        num_passed: 3,
+        num_passed: 4,
         num_failed: 0,
-        num_ignored: 0,
+        num_ignored: 2,
         num_measured: 0,
     });
-    assert_reordered_log(out.as_str(), 3, &["..."], &conclusion_to_output(&c));
+    assert_log!(out, "
+        running 6 tests
+        ....SS
+        test result: ok. 4 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; \
+            finished in 0.00s
+    ");
 }