First pass validate bids examples

.github/workflows/ci.yaml

@@ -69,6 +69,7 @@ jobs:
 
       - name: Run spec_test coverage
         run: HED_GITHUB_TOKEN=${{ secrets.HED_GITHUB_TOKEN }} coverage run --append -m unittest spec_tests/test_errors.py
+        continue-on-error: true
 
       - name: Archive code coverage results
         if: ${{matrix.python-version == '3.9'}}

.github/workflows/spec_tests.yaml

@@ -31,13 +31,30 @@ jobs:
           python -m pip install --upgrade --upgrade-strategy eager pip
           pip install -r requirements.txt
 
-      - name: Test with unittest
+      - name: Spec Error Tests
+        id: spec_error_tests
+        continue-on-error: true
         run: |
-           python -m unittest spec_tests/test_errors.py > test_results.txt
+           python -m unittest spec_tests/test_errors.py > error_results.txt
 
-      - name: Upload spec test results
+      - name: Bids Validation Test
+        id: bids_validation_test
+        continue-on-error: true
+        run: |
+           python -m unittest spec_tests/validate_bids.py > validate_bids_results.txt
+
+      - name: Upload error test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: error-test-results
+          path: error_results.txt
+
+      - name: Upload bids test results
         uses: actions/upload-artifact@v3
         with:
-          name: spec-test-results
-          path: test_results.txt
+          name: bids-test-results
+          path: validate_bids_results.txt
 
+      - name: Fail if Tests Failed
+        if: steps.spec_error_tests.outcome == 'failure' || steps.bids_validation_test.outcome == 'failure'
+        run: exit 1

.gitmodules

@@ -2,3 +2,8 @@
 	path = spec_tests/hed-specification
 	url = https://github.com/hed-standard/hed-specification/
 	branch = develop
+
+[submodule "spec_tests/hed-examples"]
+	path = spec_tests/hed-examples
+	url = https://github.com/hed-standard/hed-examples/
+	branch = develop

spec_tests/validate_bids.py

@@ -0,0 +1,35 @@
+import os
+import unittest
+
+from hed.tools import BidsDataset
+from hed.errors import get_printable_issue_string
+
+
+class MyTestCase(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.base_dir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                                                 'hed-examples/datasets'))
+        cls.fail_count = []
+
+    @classmethod
+    def tearDownClass(cls):
+        pass
+
+    def test_validation(self):
+        base_dir = self.base_dir
+        for directory in os.listdir(base_dir):
+            dataset_path = os.path.join(base_dir, directory)
+            if not os.path.isdir(dataset_path):
+                continue
+
+            bids_data = BidsDataset(dataset_path)
+            issues = bids_data.validate(check_for_warnings=False)
+            if issues:
+                self.fail_count.append(issues)
+        print(f"{len(self.fail_count)} tests got an unexpected result")
+        print("\n".join(get_printable_issue_string(issue, skip_filename=False) for issue in self.fail_count))
+        self.assertEqual(0, len(self.fail_count))
+
+if __name__ == '__main__':
+    unittest.main()