Update to match neurodatascience/nipoppy:main (#51)

.flake8

@@ -0,0 +1,14 @@
+[flake8]
+exclude =
+    .git,
+    __pycache__,
+    build,
+    dist,
+	env,
+	venv,
+per-file-ignores =
+    # - docstrings rules that should not be applied to tests
+	tests/*: D100, D103, D104
+    # allow "weird indentation"
+	tests/test_workflow_*.py: D100, D103, D104, E131, E127, E501
+max-line-length = 90

.github/workflows/tests.yml

@@ -7,7 +7,7 @@ on:
   pull_request:
     branches: ['*']
 
-# cancel previous runs if new one is triggered 
+# cancel previous runs if new one is triggered
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
 
     # only trigger on upstream repo
-    if: github.repository_owner == 'neurodatascience' && github.event.repository.name == 'nipoppy' 
+    if: github.repository_owner == 'neurodatascience' && github.event.repository.name == 'nipoppy'
 
     steps:
 
@@ -31,11 +31,25 @@ jobs:
       with:
         python-version: '3.11'
 
-    - name: Install 
+    - name: Install dependencies
+      run: |
+          sudo apt-get update
+          sudo apt-get install -y git-annex
+
+    - name: Install nipoppy
       run: |
         pip install -U pip
         pip install .[tests]
 
+    - name: Install data
+      run: make -C tests data/ds004097
+
     - name: Test
-      run: |
-        python -m pytest
+      run: python -m pytest tests --cov-report=xml
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        file: ./coverage.xml
+        name: codecov-umbrella
+        fail_ci_if_error: false

.gitignore

@@ -17,12 +17,16 @@ share/python-wheels/
 nipoppy/notebooks/.ipynb_checkpoints/*
 
 # dataset-specific files
-bids_filter.json 
+bids_filter.json
 workflow/proc_pipe/fmriprep/scripts/fmriprep_sge.sh
 workflow/proc_pipe/fmriprep/scripts/fmriprep_slurm.sh
 workflow/dicom_org/dicom_dir_func.py
 
 # logs/testing
 *.out
 *.log
+.coverage
+htmlcov
+
+
 env/

.pre-commit-config.yaml

@@ -3,6 +3,19 @@
 # See https://pre-commit.com/hooks.html for more hooks
 
 repos:
+
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    -   id: check-ast
+    -   id: check-case-conflict
+    -   id: check-json
+    -   id: check-merge-conflict
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+    -   id: check-toml
+
 -   repo: https://github.com/codespell-project/codespell
     rev: v2.2.5
     hooks:

nipoppy/workflow/proc_pipe/fmriprep/run_fmriprep.py

@@ -42,7 +42,7 @@ def run_fmriprep(participant_id: str,
     fmriprep_home_dir = f"{fmriprep_out_dir}/fmriprep_home_{participant_id}/"
     Path(f"{fmriprep_home_dir}").mkdir(parents=True, exist_ok=True)
 
-    bids_db_dir = f"/fmripre_proc/{DNAME_BIDS_DB}"
+    bids_db_dir = f"/fmriprep_proc/{DNAME_BIDS_DB}"
 
     bids_db_dir_outside_container = f"{proc_dir}/{DNAME_BIDS_DB}"
     if not Path(bids_db_dir_outside_container).exists():
@@ -60,12 +60,12 @@ def run_fmriprep(participant_id: str,
                 logger.error(f"Expected to find only files in {bids_db_dir_outside_container} but found directory {path}")
                 sys.exit(1)
 
-    # Singularity CMD 
+    # Singularity CMD
     SINGULARITY_CMD=f"singularity run \
         -B {bids_dir}:{bids_dir} \
         -B {fmriprep_home_dir}:/home/fmriprep --home /home/fmriprep --cleanenv \
         -B {fmriprep_out_dir}:/output \
-        -B {proc_dir}:/fmripre_proc \
+        -B {proc_dir}:/fmriprep_proc \
         -B {templateflow_dir}:{SINGULARITY_TEMPLATEFLOW_DIR} \
         -B {fmriprep_dir}:/work \
         -B {fs_dir}:{SINGULARITY_FS_DIR} \
@@ -89,15 +89,15 @@ def run_fmriprep(participant_id: str,
     # Append optional args
     if use_bids_filter:
         logger.info("Using bids_filter.json")
-        bids_filter_str = f"--bids-filter-file /fmripre_proc/bids_filter_fmriprep.json"
+        bids_filter_str = f"--bids-filter-file /fmriprep_proc/bids_filter_fmriprep.json"
         fmriprep_CMD = f"{fmriprep_CMD} {bids_filter_str}"
 
     if anat_only:
         logger.info("Using anat_only workflow")
         anat_only_str = "--anat-only"
         fmriprep_CMD = f"{fmriprep_CMD} {anat_only_str}"
 
-    CMD_ARGS = SINGULARITY_CMD + fmriprep_CMD 
+    CMD_ARGS = SINGULARITY_CMD + fmriprep_CMD
     CMD = CMD_ARGS.split()
 
     logger.info("Running fmriprep...")
@@ -108,7 +108,7 @@ def run_fmriprep(participant_id: str,
         fmriprep_proc = subprocess.run(CMD)
     except Exception as e:
         logger.error(f"fmriprep run failed with exceptions: {e}")
-    
+
     logger.info(f"Successfully completed fmriprep run for participant: {participant_id}")
     logger.info("-"*75)
     logger.info("")
@@ -164,7 +164,7 @@ def run(participant_id: str,
     # Copy bids_filter.json `<DATASET_ROOT>/bids/bids_filter.json`
     if use_bids_filter:
         logger.info(f"Copying ./bids_filter.json to {proc_dir}/bids_filter_fmriprep.json (to be seen by Singularity container)")
-        shutil.copyfile(f"{CWD}/bids_filter.json", f"{proc_dir}/bids_filter_fmriprep.json")
+        shutil.copyfile(f"{CWD}/sample_bids_filter.json", f"{proc_dir}/bids_filter_fmriprep.json")
 
     # launch fmriprep
     run_fmriprep(participant_id,
@@ -183,15 +183,15 @@ def run(participant_id: str,
 if __name__ == '__main__':
     # argparse
     HELPTEXT = """
-    Script to run fMRIPrep 
+    Script to run fMRIPrep
     """
 
     parser = argparse.ArgumentParser(description=HELPTEXT)
 
     parser.add_argument('--global_config', type=str, help='path to global configs for a given nipoppy dataset')
     parser.add_argument('--participant_id', type=str, help='participant id')
     parser.add_argument('--session_id', type=str, help='session id for the participant')
-    parser.add_argument('--output_dir', type=str, default=None, 
+    parser.add_argument('--output_dir', type=str, default=None,
                         help='specify custom output dir (if None --> <DATASET_ROOT>/derivatives)')
     parser.add_argument('--use_bids_filter', action='store_true', help='use bids filter or not')
     parser.add_argument('--anat_only', action='store_true', help='run only anatomical workflow or not')

nipoppy/workflow/proc_pipe/mriqc/run_mriqc.py

@@ -106,6 +106,8 @@ def run(participant_id, global_configs, session_id, output_dir, modalities, logg
     except Exception as e:
         logger.error(f"mriqc run failed with exceptions: {e}")
 
+    return CMD
+
 
 if __name__ == '__main__':
 

pyproject.toml

@@ -19,7 +19,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-test = ["pytest"]
+test = ["pytest", "pytest-cov", "datalad"]
 
 # alias in case of typo
 tests = ["nipoppy[test]"]
@@ -30,12 +30,23 @@ packages = ["nipoppy"]
 [tool.hatch.build]
 include = [
     "nipoppy/workflow/proc_pipe/tractoflow/tractoflow",
+    "nipoppy/workflow/proc_pipe/tractoflow",
 ]
 
 [tool.codespell]
 skip = '.git,.github,*.pdf,*.svg,pyproject.toml,*.ipynb,*.html,ppmi_imaging_descriptions.json'
 ignore-words-list = 'te,ines'
 
 [tool.pytest.ini_options]
-addopts = "-ra -q -vv "
-testpaths = ["tests/"]
+addopts = "-ra -q -vv --cov nipoppy"
+testpaths = ["tests/"]
+norecursedirs = ["tests/data"]
+
+[tool.isort]
+combine_as_imports = true
+line_length = 79
+profile = "black"
+skip_gitignore = true
+
+[tool.black]
+line-length = 79

tests/Makefile

@@ -0,0 +1,7 @@
+.PHONY: data/ds004097
+
+data/ds004097:
+	rm -fr data/ds004097
+	datalad install -s ///openneuro/ds004097 data/ds004097
+	cd data/ds004097 && datalad get sub-NDARDD890AYU/ses-01/anat/ -J 12
+	cd data/ds004097 && datalad get sub-NDARDD890AYU/ses-01/dwi/ -J 12

tests/conftest.py

@@ -1,20 +1,55 @@
 from __future__ import annotations
 
 import json
+import shutil
 from pathlib import Path
 
 
-def global_config_file() -> Path:
+def _global_config_file() -> Path:
     return Path(__file__).parent / "data" / "test_global_configs.json"
 
 
 def global_config_for_testing(pth: Path) -> dict:
     """Set up configuration for testing and create required directories."""
-    with open(global_config_file(), "r") as f:
+    with open(_global_config_file(), "r") as f:
         global_configs = json.load(f)
 
     global_configs["DATASET_ROOT"] = str(pth)
 
     (pth / "bids").mkdir(parents=True, exist_ok=True)
 
     return global_configs
+
+
+def create_dummy_bids_filter(
+    pth: Path, filename: str = "bids_filter.json"
+) -> None:
+    """Use a modified content from the tractoflow sample."""
+    bids_filter = {
+        "t1w": {
+            "datatype": "anat",
+            "session": "01",
+            "run": "1",
+            "suffix": "T1w",
+        },
+        "dwi": {"session": "01", "run": "1", "suffix": "dwi"},
+    }
+
+    pth.mkdir(parents=True, exist_ok=True)
+    with open(pth / filename, "w") as f:
+        json.dump(bids_filter, f)
+
+
+def mock_bids_dataset(pth: Path, dataset: str) -> Path:
+    """Copy a BIDS example dataset to the given path."""
+    ds = Path(__file__).parent / "data" / dataset
+    print(f"\nCopying {ds} to {pth}\n")
+    shutil.copytree(
+        ds,
+        pth,
+        symlinks=True,
+        ignore_dangling_symlinks=True,
+        dirs_exist_ok=True,
+    )
+
+    return pth / "bids"

tests/data/.gitignore

@@ -0,0 +1 @@
+ds**

tests/data/test_global_configs.json

@@ -3,7 +3,7 @@
     "DATASET_ROOT": "",
 
     "CONTAINER_STORE": "",
-    
+
     "SINGULARITY_PATH": "singularity",
 
     "TEMPLATEFLOW_DIR": "",
@@ -13,7 +13,7 @@
 
     "BIDS": {
         "heudiconv": {
-            "VERSION": "0.11.6",    
+            "VERSION": "0.11.6",
             "CONTAINER": "heudiconv_{}.sif",
             "URL": ""
         },
@@ -35,6 +35,11 @@
             "CONTAINER": "fmriprep_{}.sif",
             "URL": ""
         },
+        "tractoflow": {
+            "VERSION": "XX.X.X",
+            "CONTAINER": "tractoflow_{}.sif",
+            "URL": ""
+        },
         "freesurfer": {
             "VERSION": "6.0.1",
             "CONTAINER": "fmriprep_{}.sif",
@@ -47,6 +52,6 @@
             "PATH": "",
             "VERSION": "",
             "URL": ""
-        }     
+        }
     }
 }