[ENH] Update global config specification (nipoppy#238)

* add NAME, VERSION, and VARIANT to PipelineConfig

* remove VARIANT and add BidsPipelineConfig (with STEP)

* apply new pipeline config schemas to Config

* apply changes to workflows and workflow tests

* improve test for duplicate pipelines

* rename BIDS -> BIDS_PIPELINES

* add GLOBALS field to config to minimize repetition

* make sample global config file only contain the "latest" version of each supported pipelines

* rename global_configs.json -> global_config.json (singular)

* update config file insert in docs (quickstart)

* rename global_configs.json -> global_config.json in documentation and test files too

* increase test coverage

* fix tests so that dummy Templateflow file is written in tmp_path

* put CONTAINER and URI inside CONTAINER_INFO in PipelineConfig

* implement pipeline steps with INVOCATION_FILE/DESCRIPTOR_FILE instead of INVOCATION/DESCRIPTOR (non-config tests failing)

* make workflows use pipeline steps with invocation/descriptor files (all tests pass except test_supported_pipelines.py)

* make supported pipeline tests pass

* rename GLOBALS field to SUBSTITUTIONS

* apply substitutions to descriptor/invocation file content

* make `nipoppy init` copy sample invocation files

* apply layout path replacement right after loading config file

* add `--pipeline-step` argument to `nipoppy run`

* make `pipeline_version` argument optional

* do not propagate container configs on load (fix triple --cleanenv issue)

* move container subcommand field to Boutiques descriptor instead of CONTAINER_CONFIG

* make ContainerConfig propagation optionally include COMMAND propagation

* move manifest and config file to dataset root directory

* use PYBIDS_IGNORE_FILE instead of PYBIDS_IGNORE

* move tracker config(s) to separate file too

* add "CUSTOM" field to Config and do not allow extra fields

* update file schemas doc page

* fix path in docs

* uncomment caplog assert statements now that caplog is working

* add "container store" substitution by default

* add [[NIPOPPY_PIPELINE_VERSION]] and [[NIPOPPY_PIPELINE_NAME]] substitutions to config

* fix incorrect freesurfer output paths

* add descriptor file paths to global config and move descriptors in DatasetInitWorkflow

* remove flaky test_capture_warnings test

* fix dcm2bids/heudiconv default value for sourcedata directory

* fix DICOM_DIR_MAP_FILE validation error caused by substitution logic

* fix typo

* make default values explicit for container command and CUSTOM field

* rename CONTAINER_INFO.PATH -> CONTAINER_INFO.FILE

* turn get_boutiques_config() into a cached property boutiques_config

* make sure highlighted lines in docs are correct

* add second visit/session example to sample global config

* add note in tabular model classes about "model" terminology

* update highlighted lines in example config in docs

nipoppy_cli/docs/scripts/pydantic_to_jsonschema.py

@@ -6,32 +6,34 @@
 
 from nipoppy.config.boutiques import BoutiquesConfig
 from nipoppy.config.main import Config
+from nipoppy.config.tracker import TrackerConfig
 from nipoppy.layout import LayoutConfig
 from nipoppy.tabular.bagel import BagelModel
 from nipoppy.tabular.doughnut import DoughnutModel
 from nipoppy.tabular.manifest import ManifestModel
 
-DPATH_SCHEMA = Path(__file__).parent / ".." / "source" / "schemas"
+DPATH_SCHEMAS = Path(__file__).parent / ".." / "source" / "schemas"
 
 MODEL_FILENAME_MAP = {
     BoutiquesConfig: "boutiques.json",
     Config: "config.json",
     LayoutConfig: "layout.json",
+    TrackerConfig: "tracker.json",
     BagelModel: "bagel.json",
     DoughnutModel: "doughnut.json",
     ManifestModel: "manifest.json",
 }
 
 if __name__ == "__main__":
     # make sure schemas directory exists
-    if not DPATH_SCHEMA.exists():
-        print(f"\tCreating {DPATH_SCHEMA}")
-        DPATH_SCHEMA.mkdir(parents=True)
+    if not DPATH_SCHEMAS.exists():
+        print(f"\tCreating {DPATH_SCHEMAS}")
+        DPATH_SCHEMAS.mkdir(parents=True)
 
     # generate schema files
     for model, filename in MODEL_FILENAME_MAP.items():
         print(f"\tWriting JSON schema for {model.__name__} to {filename}")
-        fpath_schema = DPATH_SCHEMA / filename
+        fpath_schema = DPATH_SCHEMAS / filename
 
         schema = model.model_json_schema()
 

nipoppy_cli/docs/source/quickstart.md

@@ -46,10 +46,10 @@ You can also delete (or not) any configuration for a software/version that you d
 class: dropdown
 ---
 Here is the default content of {{fpath_config}}:
-```{literalinclude} ../../nipoppy/data/examples/sample_global_configs.json
+```{literalinclude} ../../nipoppy/data/examples/sample_global_config-latest_pipelines.json
 ---
 linenos: True
-emphasize-lines: 2, 4, 7, 28, 35, 51, 56, 69, 82, 91, 104
+emphasize-lines: 2, 4-5, 8-9, 12-16
 language: json
 ---
 ```

nipoppy_cli/docs/source/schemas/index.rst

@@ -11,27 +11,25 @@ This page contains auto-generated :term:`JSON` schemas [#f1]_ for the text files
 Global configuration file
 -------------------------
 
-.. include:: schema_howto.md
-    :parser: myst_parser.sphinx_
+.. tip::
 
-.. admonition:: Info
+    See the :ref:`Quickstart guide <customizing-config>` for an example config file.
 
-    ``PROC_PIPELINES`` and ``BIDS`` are nested objects (i.e. dictionaries),
-    where the final "leaf" values are ``PipelineConfig`` objects. All keys should
-    be strings.
+.. include:: schema_howto.md
+    :parser: myst_parser.sphinx_
 
-    ``PROC_PIPELINES`` expects two levels of nesting: one for the **pipeline name**,
-    and the other for the **pipeline version**.
+Below is the schema used for the global configuration :term:`JSON` file.
 
-    ``BIDS`` expects three levels of nesting: one for the **pipeline name**, one for
-    the **pipeline version**, and the last one for the **name of the** :term:`BIDS` **conversion step**.
+.. jsonschema:: config.json
 
-    See the :ref:`Quickstart guide <customizing-config>` for an example config file
-    that shows these nested structures.
+Tracker configuration file
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Below is the schema used for the global configuration :term:`JSON` file.
+The tracker configuration file specified in the `PipelineConfig`_ should be a :term:`JSON` file that contains **a list** of tracker configurations.
+The schema for each individual tracker configuration is shown below.
 
-.. jsonschema:: config.json
+.. jsonschema:: tracker.json
+    :lift_title: False
 
 .. _manifest-schema:
 

nipoppy_cli/docs/source/schemas/schema_howto.md

@@ -4,7 +4,7 @@ class: note
 ---
 - Read the table from top to bottom, left to right
 - **Required properties** are in **bold**
-- The *type* of each property is in *italics*
+- The *type* of each property is in *italics*, to the right of a cell labelled "type"
     - Unless if that property is an object described by another schema, in which case it is a link to that schema
 - Default values are shown for optional properties (`None` if empty)
 - See the [JSON schema docs](https://www.learnjsonschema.com/) more details about keyword meanings

nipoppy_cli/nipoppy/cli/parser.py

@@ -73,13 +73,24 @@ def add_args_pipeline(parser: _ActionsContainer) -> _ActionsContainer:
         "--pipeline",
         type=str,
         required=True,
-        help="Pipeline name, as written in the config file.",
+        help="Pipeline name, as specified in the config file.",
     )
     parser.add_argument(
         "--pipeline-version",
         type=str,
         required=False,
-        help="Pipeline version, as written in the config file.",
+        help="Pipeline version, as specified in the config file.",
+    )
+    return parser
+
+
+def add_arg_pipeline_step(parser: _ActionsContainer) -> _ActionsContainer:
+    """Add a --pipeline-step argument to the parser."""
+    parser.add_argument(
+        "--pipeline-step",
+        type=str,
+        required=False,
+        help="Pipeline step, as specified in the config file (default: first step).",
     )
     return parser
 
@@ -242,12 +253,7 @@ def add_subparser_bids_conversion(
     )
     parser = add_arg_dataset_root(parser)
     parser = add_args_pipeline(parser)
-    parser.add_argument(
-        "--pipeline-step",
-        type=str,
-        required=False,
-        help="Pipeline step, as written in the config file.",
-    )
+    parser = add_arg_pipeline_step(parser)
     parser = add_args_participant_and_session(parser)
     parser = add_arg_simulate(parser)
     return parser
@@ -267,6 +273,7 @@ def add_subparser_pipeline_run(
     )
     parser = add_arg_dataset_root(parser)
     parser = add_args_pipeline(parser)
+    parser = add_arg_pipeline_step(parser)
     parser = add_args_participant_and_session(parser)
     parser = add_arg_simulate(parser)
     return parser

nipoppy_cli/nipoppy/cli/run.py

@@ -1,5 +1,6 @@
 """Command-line interface."""
 
+import logging
 import sys
 from typing import Sequence
 
@@ -14,7 +15,7 @@
     COMMAND_PIPELINE_TRACK,
     get_global_parser,
 )
-from nipoppy.logger import add_logfile, get_logger
+from nipoppy.logger import add_logfile, capture_warnings, get_logger
 from nipoppy.workflows.bids_conversion import BidsConversionRunner
 from nipoppy.workflows.dataset_init import InitWorkflow
 from nipoppy.workflows.dicom_reorg import DicomReorgWorkflow
@@ -76,6 +77,7 @@ def cli(argv: Sequence[str] = None) -> None:
                 dpath_root=dpath_root,
                 pipeline_name=args.pipeline,
                 pipeline_version=args.pipeline_version,
+                pipeline_step=args.pipeline_step,
                 participant=args.participant,
                 session=args.session,
                 simulate=args.simulate,
@@ -97,6 +99,10 @@ def cli(argv: Sequence[str] = None) -> None:
         if command != COMMAND_INIT:
             add_logfile(logger, workflow.generate_fpath_log())
 
+        # capture warnings
+        logging.captureWarnings(True)
+        capture_warnings(workflow.logger)
+
         # run the workflow
         workflow.run()
 

nipoppy_cli/nipoppy/config/__init__.py

@@ -4,3 +4,5 @@
 from .container import ContainerConfig
 from .main import Config
 from .pipeline import PipelineConfig
+from .pipeline_step import PipelineStepConfig
+from .tracker import TrackerConfig

nipoppy_cli/nipoppy/config/boutiques.py

@@ -1,19 +1,22 @@
 """Boutiques configuration model and utility functions."""
 
-from pydantic import ConfigDict
+from pydantic import ConfigDict, Field
 
-from nipoppy.config.container import ModelWithContainerConfig
+from nipoppy.config.container import SchemaWithContainerConfig
 
 BOUTIQUES_CUSTOM_KEY = "custom"  # as defined by Boutiques schema
 BOUTIQUES_CONFIG_KEY = "nipoppy"
 
 
-class BoutiquesConfig(ModelWithContainerConfig):
-    """Model for custom configuration within a Boutiques descriptor."""
+class BoutiquesConfig(SchemaWithContainerConfig):
+    """Schema for custom configuration within a Boutiques descriptor."""
 
+    CONTAINER_SUBCOMMAND: str = Field(
+        default="run", description="Subcommand for Apptainer/Singularity call"
+    )
     # dpath_participant_session_result (for tarring/zipping/extracting)
     # run_on (for choosing which participants/sessions to run on)
-    # bids_input (for pybids)
+    # with_pybids (for pybids)
 
     model_config = ConfigDict(extra="forbid")
 

nipoppy_cli/nipoppy/config/container.py

@@ -20,15 +20,16 @@
 
 
 class ContainerConfig(BaseModel):
-    """Model for container configuration."""
+    """
+    Schema for container configuration.
+
+    Does not include information about the container image.
+    """
 
     COMMAND: str = Field(
         default="apptainer",
         description="Name of or path to Apptainer/Singularity executable",
     )
-    SUBCOMMAND: str = Field(
-        default="run", description="Subcommand for Apptainer/Singularity call"
-    )
     ARGS: list[str] = Field(
         default=[],
         description=(
@@ -46,7 +47,7 @@ class ContainerConfig(BaseModel):
         default=True,
         description=(
             "Whether this config should inherit from higher-lever container configs."
-            " If false, will override higher-level configs"
+            " If false, will ignore higher-level configs"
         ),
     )
 
@@ -66,19 +67,27 @@ def add_bind_path(
             mode=mode,
         )
 
-    def merge_args_and_env_vars(self, other: Any):
+    def merge(self, other: Any, overwrite_command=False):
         """
-        Merge arguments and environment variables with another instance.
+        Combine with another ContainerConfig instance.
 
-        Arguments from other are appended to arguments of the current instance,
-        but environment variables from other do not overwrite those of the current
-        instance.
+        By default this only changes arguments and environment variables, and no
+        information is overwritten:
+        - Arguments from other are appended to arguments of the current instance
+        - Environment variables from other do not overwrite those of the current
+        instance
+
+        If overwrite_command is True, the command of the current instance is
+        replaced with that of the other instance.
         """
         if not isinstance(other, self.__class__):
             raise TypeError(
                 f"Cannot merge {self.__class__} with object of type {type(other)}"
             )
 
+        if overwrite_command:
+            self.COMMAND = other.COMMAND
+
         if self.ARGS != other.ARGS:
             self.ARGS.extend(other.ARGS)
 
@@ -89,10 +98,31 @@ def merge_args_and_env_vars(self, other: Any):
         return self
 
 
-class ModelWithContainerConfig(BaseModel):
-    """To be inherited by configs that have a ContaienrConfig sub-config."""
+class ContainerInfo(BaseModel):
+    """Schema for container image (i.e., file) information."""
+
+    FILE: Optional[Path] = Field(
+        default=None,
+        description=(
+            "Path to the container associated with the pipeline"
+            ", relative to the root directory of the dataset"
+        ),
+    )
+    URI: Optional[str] = Field(
+        default=None,
+        description="The Docker or Apptainer/Singularity URI for the container",
+    )
+
+    model_config = ConfigDict(extra="forbid")
+
 
-    CONTAINER_CONFIG: ContainerConfig = ContainerConfig()
+class SchemaWithContainerConfig(BaseModel):
+    """To be inherited by configs that have a ContainerConfig sub-config."""
+
+    CONTAINER_CONFIG: ContainerConfig = Field(
+        default=ContainerConfig(),
+        description="Configuration for running a container",
+    )
 
     def get_container_config(self) -> ContainerConfig:
         """Return the pipeline's ContainerConfig object."""
@@ -222,6 +252,7 @@ def check_container_command(command: str) -> str:
 
 def prepare_container(
     container_config: ContainerConfig,
+    subcommand: str = "run",
     check=True,
     logger: Optional[logging.Logger] = None,
 ) -> str:
@@ -243,7 +274,6 @@ def prepare_container(
         The command string
     """
     command = container_config.COMMAND
-    subcommand = container_config.SUBCOMMAND
     args = container_config.ARGS
     env_vars = container_config.ENV_VARS