fix validation (RagnarGrootKoerkamp#344)

* fix validation

* fix parallel?

* fix?

* improved message

bin/problem.py

@@ -224,7 +224,7 @@ def testcases(
             for prefix in {
                 validate.Mode.INPUT: ['secret', 'sample'],
                 validate.Mode.ANSWER: ['secret', 'sample'],
-                validate.Mode.INVALID: ['bad', 'invalid_*'],
+                validate.Mode.INVALID: config.INVALID_CASE_DIRECTORIES,
             }[mode]:
                 in_paths += glob(p.path, f'data/{prefix}/**/*.in')
         else:
@@ -243,7 +243,9 @@ def testcases(
         testcases.sort(key=lambda t: t.name)
 
         if len(testcases) == 0:
-            warn(f'Didn\'t find any testcases{" with answer" if needans else ""} for {p.name}')
+            ans = ' with answer' if needans else ''
+            val = f' skipping {mode} validation' if mode is not None else ''
+            warn(f'Didn\'t find any testcases{ans} for problem {p.name}{val}')
             testcases = False
 
         p._testcases[key] = testcases
@@ -658,7 +660,7 @@ def validate_data(problem, mode: validate.Mode, constraints: dict | bool | None
             case _:
                 ValueError(mode)
 
-        needans = mode != validate.Mode.INPUT  # TODO
+        needans = mode != validate.Mode.INPUT
 
         if testcases is False:
             return True
@@ -668,10 +670,10 @@ def validate_data(problem, mode: validate.Mode, constraints: dict | bool | None
             return True
 
         action = (
-            "Invalidation"
+            'Invalidation'
             if mode == validate.Mode.INVALID
             else (
-                f"{mode} validation" if not check_constraints else f"Collecting {mode} constraints"
+                f'{mode} validation' if not check_constraints else f'Collecting {mode} constraints'
             ).capitalize()
         )
 
@@ -681,22 +683,29 @@ def validate_data(problem, mode: validate.Mode, constraints: dict | bool | None
 
         # validate the testcases
         bar = ProgressBar(action, items=[t.name for t in testcases])
-        for testcase in testcases:
-            bar.start(testcase.name)
+
+        def process_testcase(testcase):
+            nonlocal success
+
+            localbar = bar.start(testcase.name)
 
             if (
                 mode == validate.Mode.INPUT
                 and not testcase.in_path.is_symlink()
-                and not testcase.root == "invalid_answers"
-                and not testcase.root == "invalid_outputs"
+                and not testcase.root == 'invalid_answers'
+                and not testcase.root == 'invalid_outputs'
             ):
                 t2 = problem.matches_existing_testcase(testcase)
                 if t2 is not None:
-                    bar.error(f'Duplicate testcase: identical to {t2.name}')
-                    continue
+                    localbar.error(f'Duplicate testcase: identical to {t2.name}')
+                    return
+
+            ok = testcase.validate_format(mode, bar=localbar, constraints=constraints)
+            success &= ok
+            if ok:
+                localbar.done()
 
-            success &= testcase.validate_format(mode, bar=bar, constraints=constraints)
-            bar.done()
+        parallel.run_tasks(process_testcase, testcases)
 
         bar.finalize(print_done=True)
 

bin/testcase.py

@@ -205,34 +205,86 @@ def validate_format(
         bar,
         constraints=None,
         warn_instead_of_error=False,
-        args=None,  # TODO never used?
+        args=None,
     ) -> bool:
         check_constraints = constraints is not None
+
         match mode:
             case Mode.INPUT:
-                validators = self.problem.validators(
-                    InputValidator, check_constraints=check_constraints
+                return self._run_validators(
+                    Mode.INPUT,
+                    self.problem.validators(InputValidator, check_constraints=check_constraints),
+                    self.root == 'invalid_inputs',
+                    bar=bar,
+                    constraints=constraints,
+                    warn_instead_of_error=warn_instead_of_error,
+                    args=args,
                 )
-                expect_rejection = self.root == 'invalid_inputs'
             case Mode.ANSWER:
-                validators = self.problem.validators(
-                    AnswerValidator, check_constraints=check_constraints
-                ) + self.problem.validators(OutputValidator, check_constraints=check_constraints)
-                expect_rejection = self.root == 'invalid_answers'
+                return self._run_validators(
+                    Mode.ANSWER,
+                    self.problem.validators(AnswerValidator, check_constraints=check_constraints)
+                    + self.problem.validators(OutputValidator, check_constraints=check_constraints),
+                    self.root == 'invalid_answers',
+                    bar=bar,
+                    constraints=constraints,
+                    warn_instead_of_error=warn_instead_of_error,
+                    args=args,
+                )
             case Mode.INVALID:
-                validators = self.problem.validators(InputValidator)[::]
-                if self.root in ['invalid_answers', 'invalid_outputs']:
-                    validators += self.problem.validators(
-                        AnswerValidator
-                    ) + self.problem.validators(OutputValidator)
-                expect_rejection = True
+                assert self.root in config.INVALID_CASE_DIRECTORIES[:-1]
+
+                ok = self.validate_format(
+                    Mode.INPUT,
+                    bar=bar,
+                    constraints=constraints,
+                    warn_instead_of_error=warn_instead_of_error,
+                    args=args,
+                )
+                if not ok or self.root == 'invalid_inputs':
+                    return ok
+
+                ok = self.validate_format(
+                    Mode.ANSWER,
+                    bar=bar,
+                    constraints=constraints,
+                    warn_instead_of_error=warn_instead_of_error,
+                    args=args,
+                )
+                if not ok or self.root == 'invalid_answers':
+                    return ok
+
+                return self._run_validators(
+                    Mode.INVALID,
+                    self.problem.validators(OutputValidator),
+                    True,
+                    bar=bar,
+                    constraints=constraints,
+                    warn_instead_of_error=warn_instead_of_error,
+                    args=args,
+                )
             case _:
                 raise ValueError
 
+    def _run_validators(
+        self,
+        mode: Mode,
+        validators,
+        expect_rejection,
+        *,
+        bar,
+        constraints=None,
+        warn_instead_of_error=False,
+        args=None,
+    ) -> bool:
+        if args is None:
+            args = []
         results = []
         for validator in validators:
-            if type(validator) == OutputValidator and self.root.startswith("invalid"):
-                args = ['case_sensitive', 'space_change_sensitive']
+            name = validator.name
+            if type(validator) == OutputValidator and mode == Mode.ANSWER:
+                args += ['case_sensitive', 'space_change_sensitive']
+                name = f'{name} (ans)'
             flags = self.testdata_yaml_validator_flags(validator)
             if flags is False:
                 continue
@@ -241,7 +293,7 @@ def validate_format(
             ret = validator.run(self, mode=mode, constraints=constraints, args=flags)
             results.append(ret.status)
 
-            message = validator.name + ': '
+            message = name + ': '
             if ret.status:
                 message += 'accepted'
             elif ret.status == ExecStatus.TIMEOUT:
@@ -287,7 +339,7 @@ def validate_format(
                     warn_instead_of_error=warn_instead_of_error,
                 )
 
-            if ret.status or expect_rejection:
+            if ret.status or self.root in config.INVALID_CASE_DIRECTORIES:
                 continue
 
             # Move testcase to destination directory if specified.
@@ -317,10 +369,12 @@ def validate_format(
         if expect_rejection:
             success = ExecStatus.REJECTED in results
             if not success:
-                bar.error(f"was not rejected by {mode} validation")
+                bar.error(f'was not rejected by {mode} validation')
         else:
             success = all(results)
             if success:
                 sanity_check(self.in_path if mode == Mode.INPUT else self.ans_path, bar)
+            else:
+                bar.done(False)
 
         return success

bin/validate.py

@@ -185,6 +185,9 @@ def run(self, testcase, mode=Mode.INPUT, constraints=None, args=None) -> ExecRes
         """
         if mode == Mode.ANSWER:
             raise ValueError("InputValidators do not support Mode.ANSWER")
+        if mode == Mode.INVALID:
+            raise ValueError("InputValidators do no support Mode.INVALID")
+
         cwd, constraints_path, arglist = self._run_helper(testcase, constraints, args)
 
         if self.language in Validator.FORMAT_VALIDATOR_LANGUAGES:
@@ -229,6 +232,8 @@ def run(self, testcase, mode=Mode.ANSWER, constraints=None, args=None):
 
         if mode == Mode.INPUT:
             raise ValueError("AnswerValidators do no support Mode.INPUT")
+        if mode == Mode.INVALID:
+            raise ValueError("AnswerValidators do no support Mode.INVALID")
 
         cwd, constraints_path, arglist = self._run_helper(testcase, constraints, args)
 
@@ -280,13 +285,22 @@ def run(self, testcase, mode, constraints=None, args=None):
         The ExecResult
         """
 
+        if mode == Mode.INPUT:
+            raise ValueError("OutputValidator do no support Mode.INPUT")
+
         in_path = testcase.in_path.resolve()
         ans_path = testcase.ans_path.resolve()
-        path = (
-            mode.out_path
-            if hasattr(mode, 'out_path')
-            else (testcase.out_path.resolve() if testcase.root == 'invalid_outputs' else ans_path)
-        )
+        if hasattr(mode, 'out_path'):
+            path = mode.out_path
+        elif mode == Mode.ANSWER:
+            path = ans_path
+        else:
+            # mode == Mode.INVALID
+            if testcase.root != 'invalid_outputs':
+                raise ValueError(
+                    "OutputValidator in Mode.INVALID should only be run for data/invalid_outputs"
+                )
+            path = testcase.out_path.resolve()
 
         if self.language in Validator.FORMAT_VALIDATOR_LANGUAGES:
             raise ValueError("Invalid output validator language")