From bb139e972b3683d6d4edeeef93bb4c763ef4ccd8 Mon Sep 17 00:00:00 2001
From: Chris Markiewicz <markiewicz@stanford.edu>
Date: Wed, 13 Nov 2024 18:38:03 -0500
Subject: [PATCH 1/5] feat(cli): Add tool for filename validation for use in
 pre-receive hooks (#1986)

* ENH: Create a pre-receive hook that accepts ignore rules and file listing

* FIX: Use consistent logger, set sensible default level, enable -q

* ENH: Determine dataset type from bundled dataset_description.json

* RF: Stream filename validation, rewrite protocol

The 0001 is a particular git-protocol-ism, and 0000 does not behave as I
expected. Instead of adding multiple 0001s and attempting to identify
the meaning, the new protocol preceeds the old protocol with a header
line and a single JSON line.

* DOC: Update docstring

* Apply suggestions from code review

Co-authored-by: Nell Hardcastle <nell@dev-nell.com>

---------

Co-authored-by: Nell Hardcastle <nell@dev-nell.com>
---
 tools/schemacode/bidsschematools/__main__.py | 138 ++++++++++++++++++-
 tools/schemacode/bidsschematools/utils.py    |   2 +-
 2 files changed, 136 insertions(+), 4 deletions(-)

diff --git a/tools/schemacode/bidsschematools/__main__.py b/tools/schemacode/bidsschematools/__main__.py
index a250e37cab..777bea2d87 100644
--- a/tools/schemacode/bidsschematools/__main__.py
+++ b/tools/schemacode/bidsschematools/__main__.py
@@ -1,6 +1,9 @@
+import json
 import logging
 import os
+import re
 import sys
+from itertools import chain
 
 import click
 
@@ -9,15 +12,18 @@
 else:
     from importlib.resources import files
 
-
+from .rules import regexify_filename_rules
 from .schema import export_schema, load_schema
+from .validator import _bidsignore_check
 
 
 @click.group()
 @click.option("-v", "--verbose", count=True)
-def cli(verbose):
+@click.option("-q", "--quiet", count=True)
+def cli(verbose, quiet):
     """BIDS Schema Tools"""
-    logging.getLogger("bidsschematools").setLevel(logging.INFO - verbose * 10)
+    verbose = verbose - quiet
+    logging.getLogger("bidsschematools").setLevel(logging.WARNING - verbose * 10)
 
 
 @cli.command()
@@ -53,5 +59,131 @@ def export_metaschema(ctx, output):
             fobj.write(metaschema)
 
 
+@cli.command("pre-receive-hook")
+@click.option("--schema", "-s", type=click.Path(), help="Path to the BIDS schema")
+@click.option(
+    "--input", "-i", "input_", default="-", type=click.Path(), help="Input file (default: stdin)"
+)
+@click.option(
+    "--output",
+    "-o",
+    "output",
+    default="-",
+    type=click.Path(),
+    help="Output file (default: stdout)",
+)
+def pre_receive_hook(schema, input_, output):
+    """Validate filenames from a list of files against the BIDS schema
+
+    The expected input takes the following form:
+
+    ```
+    bids-hook-v2
+    {"Name": "My dataset", "BIDSVersion": "1.9.0", "DatasetType": "raw"}
+    ignore-pattern1
+    ...
+    ignore-patternN
+    0001
+    .datalad/config
+    .gitattributes
+    CHANGES
+    README
+    dataset_description.json
+    participants.tsv
+    sub-01/anat/sub-01_T1w.nii.gz
+    ...
+    ```
+
+    The header identifies the protocol version. For protocol ``bids-hook-v2``,
+    the second line MUST be the dataset_description.json file, with any newlines removed.
+    The following lines, up to the line containing "0001", are ignore patterns
+    from the .bidsignore file. The lines following "0001" are the filenames to
+    be validated.
+
+    This is intended to be used in a git pre-receive hook.
+    """
+    logger = logging.getLogger("bidsschematools")
+    schema = load_schema(schema)
+
+    # Slurp inputs for now; we can think about streaming later
+    if input_ == "-":
+        stream = sys.stdin
+    else:
+        stream = open(input_)
+
+    first_line = next(stream)
+    if first_line == "bids-hook-v2\n":
+        # V2 format: header line, description JSON, followed by legacy format
+        description_str = next(stream)
+        fail = False
+        try:
+            description: dict = json.loads(description_str)
+        except json.JSONDecodeError:
+            fail = True
+        if fail or not isinstance(description, dict):
+            logger.critical("Protocol error: invalid JSON in description")
+            logger.critical(
+                "Dataset description must be one JSON object, written to a single line"
+            )
+            logger.critical("Received: %s", description_str)
+            stream.close()
+            sys.exit(2)
+    else:
+        # Legacy: ignore patterns, followed by "0001", followed by filenames
+        stream = chain([first_line], stream)
+        description = {}
+
+    dataset_type = description.get("DatasetType", "raw")
+    logger.info("Dataset type: %s", dataset_type)
+
+    ignore = []
+    for line in stream:
+        if line == "0001\n":
+            break
+        ignore.append(line.strip())
+    logger.info("Ignore patterns found: %d", len(ignore))
+
+    all_rules = chain.from_iterable(
+        regexify_filename_rules(group, schema, level=2)
+        for group in (schema.rules.files.common, schema.rules.files.raw)
+    )
+    if dataset_type == "derivative":
+        all_rules = chain(
+            all_rules,
+            regexify_filename_rules(schema.rules.files.derivatives, schema, level=2),
+        )
+
+    regexes = [rule["regex"] for rule in all_rules]
+    # XXX Hack for phenotype files - this can be removed once we
+    # have a schema definition for them
+    regexes.append(r"phenotype/.*\.(tsv|json)")
+
+    output = sys.stdout if output == "-" else open(output, "w")
+
+    rc = 0
+    any_files = False
+    valid_files = 0
+    with output:
+        for filename in stream:
+            if not any_files:
+                logger.debug("Validating files, first file: %s", filename)
+                any_files = True
+            filename = filename.strip()
+            if any(_bidsignore_check(pattern, filename, "") for pattern in ignore):
+                continue
+            if not any(re.match(regex, filename) for regex in regexes):
+                print(filename, file=output)
+                rc = 1
+            else:
+                valid_files += 1
+
+    if valid_files == 0:
+        logger.error("No files to validate")
+        rc = 2
+
+    stream.close()
+    sys.exit(rc)
+
+
 if __name__ == "__main__":
     cli()
diff --git a/tools/schemacode/bidsschematools/utils.py b/tools/schemacode/bidsschematools/utils.py
index ec2947fb37..6f9850edb0 100644
--- a/tools/schemacode/bidsschematools/utils.py
+++ b/tools/schemacode/bidsschematools/utils.py
@@ -29,7 +29,7 @@ def get_logger(name=None):
     logging.Logger
         logger object.
     """
-    return logging.getLogger("bids-schema" + (".%s" % name if name else ""))
+    return logging.getLogger("bidsschematools" + (".%s" % name if name else ""))
 
 
 def set_logger_level(lgr, level):

From 9540a823ed70fa65dcf914e992adabd11c7285fc Mon Sep 17 00:00:00 2001
From: Chris Markiewicz <markiewicz@stanford.edu>
Date: Thu, 14 Nov 2024 07:56:21 -0500
Subject: [PATCH 2/5] chore: Remove excess test data from bidsschematools
 installation directories (#1985)

* chore: Move bidsschematools to src-layout

* chore: Update locations in .pre-commit-config.yaml

* chore: Ignore tests/data/bids*-examples

* test: Save test data into tests/data

* chore: Update package data discovery

* chore(ci): Make sure schemacode tests are triggered on maintenance branches

* chore(ci): Update pytest invocation

* chore: Ensure coverage is collected from tests and installed modules
---
 .github/workflows/schemacode_ci.yml           | 17 +++++--
 .pre-commit-config.yaml                       |  4 +-
 tools/schemacode/MANIFEST.in                  |  1 +
 .../bidsschematools/data/metaschema.json      |  1 -
 tools/schemacode/bidsschematools/data/schema  |  1 -
 .../tests/test_make_testdata.py               | 48 -------------------
 tools/schemacode/pyproject.toml               | 10 ++--
 tools/schemacode/setup.cfg                    |  7 +--
 .../{ => src}/bidsschematools/__init__.py     |  0
 .../{ => src}/bidsschematools/__main__.py     |  0
 .../{ => src}/bidsschematools/conftest.py     | 34 ++++++++-----
 .../bidsschematools/data/__init__.py          |  0
 .../src/bidsschematools/data/metaschema.json  |  1 +
 .../src/bidsschematools/data/schema           |  1 +
 .../bidsschematools/data/tests/test_rules.py  |  0
 .../{ => src}/bidsschematools/expressions.py  |  0
 .../bidsschematools/render/__init__.py        |  0
 .../bidsschematools/render/tables.py          |  0
 .../{ => src}/bidsschematools/render/text.py  |  0
 .../{ => src}/bidsschematools/render/utils.py |  0
 .../{ => src}/bidsschematools/rules.py        |  0
 .../{ => src}/bidsschematools/schema.py       |  0
 .../bidsschematools/tests/__init__.py         |  0
 .../bidsschematools/tests/data/__init__.py    |  0
 .../data/broken_dataset_description.json      |  0
 .../data/expected_bids_validator_xs_write.log |  0
 .../bidsschematools/tests/test_expressions.py |  0
 .../tests/test_make_testdata.py               | 38 +++++++++++++++
 .../tests/test_render_tables.py               |  0
 .../bidsschematools/tests/test_render_text.py |  0
 .../tests/test_render_utils.py                |  0
 .../bidsschematools/tests/test_rules.py       |  0
 .../bidsschematools/tests/test_schema.py      |  0
 .../bidsschematools/tests/test_validator.py   |  0
 .../bidsschematools/types/__init__.py         |  0
 .../bidsschematools/types/namespace.py        |  0
 .../{ => src}/bidsschematools/utils.py        |  0
 .../{ => src}/bidsschematools/validator.py    |  0
 tools/schemacode/tests/data/.gitignore        |  1 +
 39 files changed, 86 insertions(+), 78 deletions(-)
 create mode 100644 tools/schemacode/MANIFEST.in
 delete mode 120000 tools/schemacode/bidsschematools/data/metaschema.json
 delete mode 120000 tools/schemacode/bidsschematools/data/schema
 delete mode 100644 tools/schemacode/bidsschematools/tests/test_make_testdata.py
 rename tools/schemacode/{ => src}/bidsschematools/__init__.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/__main__.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/conftest.py (82%)
 rename tools/schemacode/{ => src}/bidsschematools/data/__init__.py (100%)
 create mode 120000 tools/schemacode/src/bidsschematools/data/metaschema.json
 create mode 120000 tools/schemacode/src/bidsschematools/data/schema
 rename tools/schemacode/{ => src}/bidsschematools/data/tests/test_rules.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/expressions.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/render/__init__.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/render/tables.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/render/text.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/render/utils.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/rules.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/schema.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/tests/__init__.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/tests/data/__init__.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/tests/data/broken_dataset_description.json (100%)
 rename tools/schemacode/{ => src}/bidsschematools/tests/data/expected_bids_validator_xs_write.log (100%)
 rename tools/schemacode/{ => src}/bidsschematools/tests/test_expressions.py (100%)
 create mode 100644 tools/schemacode/src/bidsschematools/tests/test_make_testdata.py
 rename tools/schemacode/{ => src}/bidsschematools/tests/test_render_tables.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/tests/test_render_text.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/tests/test_render_utils.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/tests/test_rules.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/tests/test_schema.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/tests/test_validator.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/types/__init__.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/types/namespace.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/utils.py (100%)
 rename tools/schemacode/{ => src}/bidsschematools/validator.py (100%)
 create mode 100644 tools/schemacode/tests/data/.gitignore

diff --git a/.github/workflows/schemacode_ci.yml b/.github/workflows/schemacode_ci.yml
index db6e77700a..f193cd5601 100644
--- a/.github/workflows/schemacode_ci.yml
+++ b/.github/workflows/schemacode_ci.yml
@@ -4,10 +4,13 @@ on:
   push:
     branches:
       - "master"
+      - "maint/*"
     tags:
       - "schema-*"
   pull_request:
     branches:
+      - "master"
+      - "maint/*"
       - "*"
 
 concurrency:
@@ -87,14 +90,15 @@ jobs:
 
       - name: "Run tests"
         run: |
-          python -m pytest -vs --pyargs bidsschematools -m "not validate_schema" \
-              --cov-append --cov-report=xml --cov=bidsschematools --doctest-modules
+          python -m pytest -vs --doctest-modules -m "not validate_schema" \
+              --cov-append --cov-report=xml --cov-report=term --cov=src/bidsschematools
+        working-directory: tools/schemacode
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
           name: unit_${{ matrix.os }}_${{ matrix.python-version }}
-          path: coverage.xml
+          path: tools/schemacode/coverage.xml
         if: success()
 
   publish:
@@ -143,13 +147,16 @@ jobs:
           python -m pip install -e ./tools/schemacode[all]
 
       - name: Run schema validation tests
-        run: python -m pytest --pyargs bidsschematools -m "validate_schema" --cov-append --cov-report=xml --cov=bidsschematools
+        run: |
+          python -m pytest -vs --doctest-modules -m "not validate_schema" \
+              --cov-append --cov-report=xml --cov-report=term --cov=src/bidsschematools
+        working-directory: tools/schemacode
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
           name: schema_validation
-          path: coverage.xml
+          path: tools/schemacode/coverage.xml
         if: success()
 
   upload_to_codecov:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 277632e355..c2261af7e0 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 ci:
   skip: [shellcheck]
-exclude: 'tools/schemacode/bidsschematools/tests/data/broken_dataset_description.json'
+exclude: 'tools/schemacode/src/bidsschematools/tests/data/broken_dataset_description.json'
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0
@@ -83,7 +83,7 @@ repos:
           - types-jsonschema
           - jsonschema
           - httpx
-        args: ["tools/schemacode/bidsschematools"]
+        args: ["tools/schemacode/src"]
         pass_filenames: false
   - repo: https://github.com/koalaman/shellcheck-precommit
     rev: v0.10.0
diff --git a/tools/schemacode/MANIFEST.in b/tools/schemacode/MANIFEST.in
new file mode 100644
index 0000000000..37f630a4ae
--- /dev/null
+++ b/tools/schemacode/MANIFEST.in
@@ -0,0 +1 @@
+recursive-include tests *
diff --git a/tools/schemacode/bidsschematools/data/metaschema.json b/tools/schemacode/bidsschematools/data/metaschema.json
deleted file mode 120000
index ae3cfc38f9..0000000000
--- a/tools/schemacode/bidsschematools/data/metaschema.json
+++ /dev/null
@@ -1 +0,0 @@
-../../../../src/metaschema.json
\ No newline at end of file
diff --git a/tools/schemacode/bidsschematools/data/schema b/tools/schemacode/bidsschematools/data/schema
deleted file mode 120000
index a06955547a..0000000000
--- a/tools/schemacode/bidsschematools/data/schema
+++ /dev/null
@@ -1 +0,0 @@
-../../../../src/schema
\ No newline at end of file
diff --git a/tools/schemacode/bidsschematools/tests/test_make_testdata.py b/tools/schemacode/bidsschematools/tests/test_make_testdata.py
deleted file mode 100644
index 5721b630d6..0000000000
--- a/tools/schemacode/bidsschematools/tests/test_make_testdata.py
+++ /dev/null
@@ -1,48 +0,0 @@
-import os
-import shutil
-
-try:
-    from importlib.resources import files
-except ImportError:  # PY<3.9
-    from importlib_resources import files
-
-import pytest
-
-
-def require_env(var):
-    env = os.environ.get(var)
-    return pytest.mark.skipif(
-        not env, reason=f"To activate this test/feature `export {var}=1` and re-run."
-    )
-
-
-@require_env("BIDSSCHEMATOOLS_RELEASE")
-def test_make_archive(bids_examples, bids_error_examples):
-    """
-    ATTENTION! This is not a test!
-    Create static testdata archive containing the bidsschematools data reference whitelist.
-
-    Notes
-    -----
-    Due to intricacies arising from:
-
-    (1) fixtures not working outside of pytest
-    (2) implicit teardown leveraging tempdata removal (while held open by yield)
-    (3) wrappers evaluating the yield statement
-    (4) the desire to not download testdata twice for archive creation
-
-    testdata archive creation is now inconspicuously posing as a test.
-    """
-
-    testdata_dir = files("bidsschematools.tests.data")
-    ignore_git = shutil.ignore_patterns(".git*")
-    shutil.copytree(bids_examples, testdata_dir / "bids-examples", ignore=ignore_git)
-    shutil.copytree(bids_error_examples, testdata_dir / "bids-error-examples", ignore=ignore_git)
-
-    # Keeping this for now, it would be really nice to have a separate archive someday.
-    # archive_name = f"bidsschematools-testdata-{__version__}"
-    # archive_path = f"/tmp/{archive_name}.tar.gz"
-
-    # with tarfile.open(archive_path, "w:gz") as tar:
-    #     tar.add(bids_examples, arcname=f"{archive_name}/bids-examples")
-    #     tar.add(bids_error_examples, arcname=f"{archive_name}/bids-error-examples")
diff --git a/tools/schemacode/pyproject.toml b/tools/schemacode/pyproject.toml
index 22547f33ef..ce49dcd818 100644
--- a/tools/schemacode/pyproject.toml
+++ b/tools/schemacode/pyproject.toml
@@ -30,9 +30,11 @@ markers = [
   "validate_schema: tests that validate the schema itself",
 ]
 
+[tool.coverage.paths]
+source = [
+  "src/bidsschematools",
+  "**/site-packages/bidsschematools",
+]
+
 [tool.coverage.run]
 parallel = true
-omit = [
-  "*/*/tests/*",
-  "**/tests/*"
-]
diff --git a/tools/schemacode/setup.cfg b/tools/schemacode/setup.cfg
index f75732b593..aa3d19de1b 100644
--- a/tools/schemacode/setup.cfg
+++ b/tools/schemacode/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = bidsschematools
-version = file:bidsschematools/data/schema/SCHEMA_VERSION
+version = file:src/bidsschematools/data/schema/SCHEMA_VERSION
 url = https://github.com/bids-standard/bids-specification
 author = bids-standard developers
 author_email = bids.maintenance@gmail.com
@@ -25,8 +25,6 @@ install_requires =
     pyyaml
     importlib_resources; python_version < "3.9"
     jsonschema
-packages = find:
-include_package_data = false
 zip_safe = false
 
 [options.extras_require]
@@ -59,8 +57,7 @@ bidsschematools =
     data/schema/BIDS_VERSION
     data/schema/SCHEMA_VERSION
     data/schema/**/*.yaml
-    tests/data/**/*
-    tests/data/**/.bidsignore
+    tests/data/*
 
 [options.entry_points]
 console_scripts =
diff --git a/tools/schemacode/bidsschematools/__init__.py b/tools/schemacode/src/bidsschematools/__init__.py
similarity index 100%
rename from tools/schemacode/bidsschematools/__init__.py
rename to tools/schemacode/src/bidsschematools/__init__.py
diff --git a/tools/schemacode/bidsschematools/__main__.py b/tools/schemacode/src/bidsschematools/__main__.py
similarity index 100%
rename from tools/schemacode/bidsschematools/__main__.py
rename to tools/schemacode/src/bidsschematools/__main__.py
diff --git a/tools/schemacode/bidsschematools/conftest.py b/tools/schemacode/src/bidsschematools/conftest.py
similarity index 82%
rename from tools/schemacode/bidsschematools/conftest.py
rename to tools/schemacode/src/bidsschematools/conftest.py
index 2e5eaec86f..0a47d26535 100644
--- a/tools/schemacode/bidsschematools/conftest.py
+++ b/tools/schemacode/src/bidsschematools/conftest.py
@@ -1,12 +1,8 @@
 import logging
 import tempfile
+from pathlib import Path
 from subprocess import run
 
-try:
-    from importlib.resources import as_file, files
-except ImportError:  # PY<3.9
-    from importlib_resources import as_file, files
-
 import pytest
 
 lgr = logging.getLogger()
@@ -35,23 +31,37 @@
 ]
 
 
+@pytest.fixture(scope="session")
+def tests_data_dir():
+    try:
+        this_file = Path(__file__)
+    except NameError:
+        return None
+
+    data_dir = this_file.parent.parent.parent / "tests" / "data"
+
+    if data_dir.exists():
+        return data_dir
+
+
 def get_gitrepo_fixture(url, whitelist):
     @pytest.fixture(scope="session")
-    def fixture():
+    def fixture(tests_data_dir):
+        if tests_data_dir is None:
+            pytest.skip("No test data directory found; probably in an installed package")
         archive_name = url.rsplit("/", 1)[-1]
-        testdata_dir = files("bidsschematools.tests.data") / archive_name
-        if testdata_dir.is_dir():
+        archive_dir = tests_data_dir / archive_name
+        if archive_dir.is_dir():
             lgr.info(
-                f"Found static testdata archive under `{testdata_dir}`. "
+                f"Found static testdata archive under `{archive_dir}`. "
                 "Not downloading latest data from version control."
             )
-            with as_file(testdata_dir) as path:
-                yield path
+            yield archive_dir
         else:
             lgr.info(
                 "No static testdata available under `%s`. "
                 "Attempting to fetch live data from version control.",
-                testdata_dir,
+                archive_dir,
             )
             with tempfile.TemporaryDirectory() as path:
                 lgr.debug("Cloning %r into %r", url, path)
diff --git a/tools/schemacode/bidsschematools/data/__init__.py b/tools/schemacode/src/bidsschematools/data/__init__.py
similarity index 100%
rename from tools/schemacode/bidsschematools/data/__init__.py
rename to tools/schemacode/src/bidsschematools/data/__init__.py
diff --git a/tools/schemacode/src/bidsschematools/data/metaschema.json b/tools/schemacode/src/bidsschematools/data/metaschema.json
new file mode 120000
index 0000000000..399e1beffb
--- /dev/null
+++ b/tools/schemacode/src/bidsschematools/data/metaschema.json
@@ -0,0 +1 @@
+../../../../../src/metaschema.json
\ No newline at end of file
diff --git a/tools/schemacode/src/bidsschematools/data/schema b/tools/schemacode/src/bidsschematools/data/schema
new file mode 120000
index 0000000000..1e33d0b966
--- /dev/null
+++ b/tools/schemacode/src/bidsschematools/data/schema
@@ -0,0 +1 @@
+../../../../../src/schema
\ No newline at end of file
diff --git a/tools/schemacode/bidsschematools/data/tests/test_rules.py b/tools/schemacode/src/bidsschematools/data/tests/test_rules.py
similarity index 100%
rename from tools/schemacode/bidsschematools/data/tests/test_rules.py
rename to tools/schemacode/src/bidsschematools/data/tests/test_rules.py
diff --git a/tools/schemacode/bidsschematools/expressions.py b/tools/schemacode/src/bidsschematools/expressions.py
similarity index 100%
rename from tools/schemacode/bidsschematools/expressions.py
rename to tools/schemacode/src/bidsschematools/expressions.py
diff --git a/tools/schemacode/bidsschematools/render/__init__.py b/tools/schemacode/src/bidsschematools/render/__init__.py
similarity index 100%
rename from tools/schemacode/bidsschematools/render/__init__.py
rename to tools/schemacode/src/bidsschematools/render/__init__.py
diff --git a/tools/schemacode/bidsschematools/render/tables.py b/tools/schemacode/src/bidsschematools/render/tables.py
similarity index 100%
rename from tools/schemacode/bidsschematools/render/tables.py
rename to tools/schemacode/src/bidsschematools/render/tables.py
diff --git a/tools/schemacode/bidsschematools/render/text.py b/tools/schemacode/src/bidsschematools/render/text.py
similarity index 100%
rename from tools/schemacode/bidsschematools/render/text.py
rename to tools/schemacode/src/bidsschematools/render/text.py
diff --git a/tools/schemacode/bidsschematools/render/utils.py b/tools/schemacode/src/bidsschematools/render/utils.py
similarity index 100%
rename from tools/schemacode/bidsschematools/render/utils.py
rename to tools/schemacode/src/bidsschematools/render/utils.py
diff --git a/tools/schemacode/bidsschematools/rules.py b/tools/schemacode/src/bidsschematools/rules.py
similarity index 100%
rename from tools/schemacode/bidsschematools/rules.py
rename to tools/schemacode/src/bidsschematools/rules.py
diff --git a/tools/schemacode/bidsschematools/schema.py b/tools/schemacode/src/bidsschematools/schema.py
similarity index 100%
rename from tools/schemacode/bidsschematools/schema.py
rename to tools/schemacode/src/bidsschematools/schema.py
diff --git a/tools/schemacode/bidsschematools/tests/__init__.py b/tools/schemacode/src/bidsschematools/tests/__init__.py
similarity index 100%
rename from tools/schemacode/bidsschematools/tests/__init__.py
rename to tools/schemacode/src/bidsschematools/tests/__init__.py
diff --git a/tools/schemacode/bidsschematools/tests/data/__init__.py b/tools/schemacode/src/bidsschematools/tests/data/__init__.py
similarity index 100%
rename from tools/schemacode/bidsschematools/tests/data/__init__.py
rename to tools/schemacode/src/bidsschematools/tests/data/__init__.py
diff --git a/tools/schemacode/bidsschematools/tests/data/broken_dataset_description.json b/tools/schemacode/src/bidsschematools/tests/data/broken_dataset_description.json
similarity index 100%
rename from tools/schemacode/bidsschematools/tests/data/broken_dataset_description.json
rename to tools/schemacode/src/bidsschematools/tests/data/broken_dataset_description.json
diff --git a/tools/schemacode/bidsschematools/tests/data/expected_bids_validator_xs_write.log b/tools/schemacode/src/bidsschematools/tests/data/expected_bids_validator_xs_write.log
similarity index 100%
rename from tools/schemacode/bidsschematools/tests/data/expected_bids_validator_xs_write.log
rename to tools/schemacode/src/bidsschematools/tests/data/expected_bids_validator_xs_write.log
diff --git a/tools/schemacode/bidsschematools/tests/test_expressions.py b/tools/schemacode/src/bidsschematools/tests/test_expressions.py
similarity index 100%
rename from tools/schemacode/bidsschematools/tests/test_expressions.py
rename to tools/schemacode/src/bidsschematools/tests/test_expressions.py
diff --git a/tools/schemacode/src/bidsschematools/tests/test_make_testdata.py b/tools/schemacode/src/bidsschematools/tests/test_make_testdata.py
new file mode 100644
index 0000000000..77c6ae9d00
--- /dev/null
+++ b/tools/schemacode/src/bidsschematools/tests/test_make_testdata.py
@@ -0,0 +1,38 @@
+import os
+import shutil
+
+import pytest
+
+
+def require_env(var):
+    env = os.environ.get(var)
+    return pytest.mark.skipif(
+        not env, reason=f"To activate this test/feature `export {var}=1` and re-run."
+    )
+
+
+@require_env("BIDSSCHEMATOOLS_RELEASE")
+def test_make_archive(tests_data_dir, bids_examples, bids_error_examples):
+    """
+    ATTENTION! This is not a test!
+    Create static testdata archive containing the bidsschematools data reference whitelist.
+
+    Notes
+    -----
+    Due to intricacies arising from:
+
+    (1) fixtures not working outside of pytest
+    (2) implicit teardown leveraging tempdata removal (while held open by yield)
+    (3) wrappers evaluating the yield statement
+    (4) the desire to not download testdata twice for archive creation
+
+    testdata archive creation is now inconspicuously posing as a test.
+    """
+
+    ignore_git = shutil.ignore_patterns(".git*")
+    target_examples = tests_data_dir / "bids-examples"
+    target_error_examples = tests_data_dir / "bids-error-examples"
+    if bids_examples != target_examples:
+        shutil.copytree(bids_examples, target_examples, ignore=ignore_git)
+    if bids_error_examples != target_error_examples:
+        shutil.copytree(bids_error_examples, target_error_examples, ignore=ignore_git)
diff --git a/tools/schemacode/bidsschematools/tests/test_render_tables.py b/tools/schemacode/src/bidsschematools/tests/test_render_tables.py
similarity index 100%
rename from tools/schemacode/bidsschematools/tests/test_render_tables.py
rename to tools/schemacode/src/bidsschematools/tests/test_render_tables.py
diff --git a/tools/schemacode/bidsschematools/tests/test_render_text.py b/tools/schemacode/src/bidsschematools/tests/test_render_text.py
similarity index 100%
rename from tools/schemacode/bidsschematools/tests/test_render_text.py
rename to tools/schemacode/src/bidsschematools/tests/test_render_text.py
diff --git a/tools/schemacode/bidsschematools/tests/test_render_utils.py b/tools/schemacode/src/bidsschematools/tests/test_render_utils.py
similarity index 100%
rename from tools/schemacode/bidsschematools/tests/test_render_utils.py
rename to tools/schemacode/src/bidsschematools/tests/test_render_utils.py
diff --git a/tools/schemacode/bidsschematools/tests/test_rules.py b/tools/schemacode/src/bidsschematools/tests/test_rules.py
similarity index 100%
rename from tools/schemacode/bidsschematools/tests/test_rules.py
rename to tools/schemacode/src/bidsschematools/tests/test_rules.py
diff --git a/tools/schemacode/bidsschematools/tests/test_schema.py b/tools/schemacode/src/bidsschematools/tests/test_schema.py
similarity index 100%
rename from tools/schemacode/bidsschematools/tests/test_schema.py
rename to tools/schemacode/src/bidsschematools/tests/test_schema.py
diff --git a/tools/schemacode/bidsschematools/tests/test_validator.py b/tools/schemacode/src/bidsschematools/tests/test_validator.py
similarity index 100%
rename from tools/schemacode/bidsschematools/tests/test_validator.py
rename to tools/schemacode/src/bidsschematools/tests/test_validator.py
diff --git a/tools/schemacode/bidsschematools/types/__init__.py b/tools/schemacode/src/bidsschematools/types/__init__.py
similarity index 100%
rename from tools/schemacode/bidsschematools/types/__init__.py
rename to tools/schemacode/src/bidsschematools/types/__init__.py
diff --git a/tools/schemacode/bidsschematools/types/namespace.py b/tools/schemacode/src/bidsschematools/types/namespace.py
similarity index 100%
rename from tools/schemacode/bidsschematools/types/namespace.py
rename to tools/schemacode/src/bidsschematools/types/namespace.py
diff --git a/tools/schemacode/bidsschematools/utils.py b/tools/schemacode/src/bidsschematools/utils.py
similarity index 100%
rename from tools/schemacode/bidsschematools/utils.py
rename to tools/schemacode/src/bidsschematools/utils.py
diff --git a/tools/schemacode/bidsschematools/validator.py b/tools/schemacode/src/bidsschematools/validator.py
similarity index 100%
rename from tools/schemacode/bidsschematools/validator.py
rename to tools/schemacode/src/bidsschematools/validator.py
diff --git a/tools/schemacode/tests/data/.gitignore b/tools/schemacode/tests/data/.gitignore
new file mode 100644
index 0000000000..2dc3af1b14
--- /dev/null
+++ b/tools/schemacode/tests/data/.gitignore
@@ -0,0 +1 @@
+bids*-examples

From d3e575eff814317b90157a3a69fdf4c077f70cc2 Mon Sep 17 00:00:00 2001
From: Chris Markiewicz <effigies@gmail.com>
Date: Thu, 14 Nov 2024 08:06:10 -0500
Subject: [PATCH 3/5] chore(ci): Fix pytest call for make_archive

---
 .github/workflows/schemacode_ci.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/schemacode_ci.yml b/.github/workflows/schemacode_ci.yml
index f193cd5601..bb2ec67c4a 100644
--- a/.github/workflows/schemacode_ci.yml
+++ b/.github/workflows/schemacode_ci.yml
@@ -39,7 +39,9 @@ jobs:
         run: pip install --upgrade tools/schemacode[all]
         if: ${{ startsWith(github.ref, 'refs/tags/schema-') }}
       - name: "Build archive on tag"
-        run: pytest tools/schemacode/bidsschematools -k make_archive
+        run: |
+          python -m pytest -k make_archive
+        working-directory: tools/schemacode
         env:
           BIDSSCHEMATOOLS_RELEASE: 1
         if: ${{ startsWith(github.ref, 'refs/tags/schema-') }}

From 063f55e10c390c2021509010c2e880c9ec6547a8 Mon Sep 17 00:00:00 2001
From: Chris Markiewicz <effigies@gmail.com>
Date: Thu, 14 Nov 2024 08:00:19 -0500
Subject: [PATCH 4/5] schema-0.11.3.post3

---
 src/schema/SCHEMA_VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/schema/SCHEMA_VERSION b/src/schema/SCHEMA_VERSION
index 4bd93f8234..162e85a5a9 100644
--- a/src/schema/SCHEMA_VERSION
+++ b/src/schema/SCHEMA_VERSION
@@ -1 +1 @@
-0.11.3.post3-dev
+0.11.3.post3

From e8595461dba81f628c01e7a8107123a0e565ff83 Mon Sep 17 00:00:00 2001
From: Chris Markiewicz <effigies@gmail.com>
Date: Thu, 14 Nov 2024 08:01:38 -0500
Subject: [PATCH 5/5] chore: Bump schema post-dev version

---
 src/schema/SCHEMA_VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/schema/SCHEMA_VERSION b/src/schema/SCHEMA_VERSION
index 162e85a5a9..4fd02b3796 100644
--- a/src/schema/SCHEMA_VERSION
+++ b/src/schema/SCHEMA_VERSION
@@ -1 +1 @@
-0.11.3.post3
+0.11.3.post4-dev