Merge branch 'master' of https://github.com/nipype/pydra into feature…

…/oar

.github/workflows/publish.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
       uses: actions/setup-python@v4
       with:

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     if: "!contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.head_commit.message, 'skip ci')"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Prepare repository
         # Fetch full git history and tags

.github/workflows/testdask.yml

@@ -0,0 +1,45 @@
+name: Dask
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ['3.9', '3.10', '3.11']
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository }}
+
+      - name: Setup Python version ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies for Dask
+        run: |
+          pip install -e ".[test,dask]"
+
+      - name: Run tests for Dask
+        run: |
+          pytest -v --dask pydra/engine --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
+
+      - name: Upload to codecov
+        run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW

.github/workflows/testpsijlocal.yml

@@ -0,0 +1,45 @@
+name: PSI/J-Local
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: ['3.11']
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository }}
+
+      - name: Setup Python version ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies for PSI/J
+        run: |
+          pip install -e ".[test, psij]"
+
+      - name: Run tests for PSI/J
+        run: |
+          pytest --color=yes -vs --psij=local -n auto pydra/engine --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml
+
+      - name: Upload to codecov
+        run: codecov -f cov.xml -F unittests -e GITHUB_WORKFLOW

.github/workflows/testpsijslurm.yml

@@ -0,0 +1,54 @@
+name: PSI/J-SLURM
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: [3.11.5]
+      fail-fast: false
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_IMAGE: adi611/docker-centos7-slurm:23.02.1
+
+    steps:
+    - name: Disable etelemetry
+      run: echo "NO_ET=TRUE" >> $GITHUB_ENV
+    - uses: actions/checkout@v4
+    - name: Pull docker image
+      run: |
+        docker pull $DOCKER_IMAGE
+        # Have image running in the background
+        docker run `bash <(curl -s https://codecov.io/env)` -itd -h slurmctl --cap-add sys_admin -d --name slurm -v `pwd`:/pydra -e NO_ET=$NO_ET $DOCKER_IMAGE
+    - name: Display previous jobs with sacct
+      run: |
+        echo "Allowing ports/daemons time to start" && sleep 10
+        docker exec slurm bash -c "sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'"
+        docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null
+        if [ $? -ne 0 ]; then
+            echo "Slurm docker image error"
+            exit 1
+        fi
+    - name: Setup Python
+      run: |
+        docker exec slurm bash -c "echo $NO_ET"
+        docker exec slurm bash -c "ls -la && echo list top level dir"
+        docker exec slurm bash -c "ls -la /pydra && echo list pydra dir"
+        if [[ "${{ matrix.python-version }}" == "3.11.5" ]]; then
+            docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v 3.11.5"
+        fi
+        docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}"
+        docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test,psij] && python -c 'import pydra; print(pydra.__version__)'"
+    - name: Run pytest
+      run: |
+        docker exec slurm bash -c "pytest --color=yes -vs -n auto --psij=slurm --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'"
+    - name: Upload to codecov
+      run: |
+        docker exec slurm bash -c "pip install urllib3==1.26.6"
+        docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests"
+        docker rm -f slurm

.github/workflows/testpydra.yml

@@ -22,7 +22,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: actions/setup-python@v4
@@ -80,7 +80,7 @@ jobs:
           name: archive
           path: archive/
       - name: Fetch repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         if: matrix.install == 'repo'
 
       - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}

.github/workflows/testsingularity.yml

@@ -21,7 +21,7 @@ jobs:
           echo "RELEASE_VERSION=v3.7.1" >> $GITHUB_ENV
           echo "NO_ET=TRUE" >> $GITHUB_ENV
       - name: Setup Singularity
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: hpcng/singularity
           ref: 'v3.7.1'
@@ -57,7 +57,7 @@ jobs:
 
 
       - name: Checkout Pydra repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: ${{ github.repository }}
       - name: Install pydra (test)

.github/workflows/testslurm.yml

@@ -8,28 +8,27 @@ on:
 
 jobs:
   build:
+    strategy:
+      matrix:
+        python-version: [3.8.16, 3.9.16, 3.10.9, 3.11.5]
+      fail-fast: false
     runs-on: ubuntu-latest
     env:
-      DOCKER_IMAGE: mgxd/slurm:19.05.1
+      DOCKER_IMAGE: adi611/docker-centos7-slurm:23.02.1
 
     steps:
     - name: Disable etelemetry
       run: echo "NO_ET=TRUE" >> $GITHUB_ENV
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Pull docker image
       run: |
         docker pull $DOCKER_IMAGE
-        # Have image running in background
-        docker run `bash <(curl -s https://codecov.io/env)` -itd -h ernie --name slurm -v `pwd`:/pydra -e NO_ET=$NO_ET $DOCKER_IMAGE
-    - name: Update python
-      run: docker exec slurm bash -c "conda install python==3.8.15"
+        # Have image running in the background
+        docker run `bash <(curl -s https://codecov.io/env)` -itd -h slurmctl --cap-add sys_admin -d --name slurm -v `pwd`:/pydra -e NO_ET=$NO_ET $DOCKER_IMAGE
     - name: Display previous jobs with sacct
       run: |
         echo "Allowing ports/daemons time to start" && sleep 10
-        docker exec slurm bash -c "sacctmgr -i add cluster name=linux \
-          && supervisorctl restart slurmdbd \
-          && supervisorctl restart slurmctld \
-          && sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'"
+        docker exec slurm bash -c "sacctmgr -i add account none,test Cluster=linux Description='none' Organization='none'"
         docker exec slurm bash -c "sacct && sinfo && squeue" 2&> /dev/null
         if [ $? -ne 0 ]; then
             echo "Slurm docker image error"
@@ -40,10 +39,16 @@ jobs:
         docker exec slurm bash -c "echo $NO_ET"
         docker exec slurm bash -c "ls -la && echo list top level dir"
         docker exec slurm bash -c "ls -la /pydra && echo list pydra dir"
+        if [[ "${{ matrix.python-version }}" == "3.11.5" ]]; then
+            docker exec slurm bash -c "CONFIGURE_OPTS=\"-with-openssl=/opt/openssl\" pyenv install -v 3.11.5"
+        fi
+        docker exec slurm bash -c "pyenv global ${{ matrix.python-version }}"
         docker exec slurm bash -c "pip install --upgrade pip && pip install -e /pydra[test] && python -c 'import pydra; print(pydra.__version__)'"
     - name: Run pytest
-      run: docker exec slurm bash -c "pytest --color=yes -vs -n auto --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra"
+      run: |
+        docker exec slurm bash -c "pytest --color=yes -vs --cov pydra --cov-config /pydra/.coveragerc --cov-report xml:/pydra/cov.xml --doctest-modules /pydra/pydra/ -k 'not test_audit_prov and not test_audit_prov_messdir_1 and not test_audit_prov_messdir_2 and not test_audit_prov_wf and not test_audit_all'"
     - name: Upload to codecov
       run: |
+        docker exec slurm bash -c "pip install urllib3==1.26.6"
         docker exec slurm bash -c "codecov --root /pydra -f /pydra/cov.xml -F unittests"
         docker rm -f slurm

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: check-yaml
     -   id: check-added-large-files
 -   repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 23.9.1
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell

.zenodo.json

@@ -75,6 +75,11 @@
       "name": "Vaillant, Ghislain",
       "orcid": "0000-0003-0267-3033"
     },
+    {
+      "affiliation": "Indian Institute of Information Technology Kalyani",
+      "name": "Agarwal, Aditya",
+      "orcid": "0009-0008-2639-5334"
+    },
     {
       "affiliation": "MIT, HMS",
       "name": "Ghosh, Satrajit",

pydra/conftest.py

@@ -7,6 +7,12 @@
 
 def pytest_addoption(parser):
     parser.addoption("--dask", action="store_true", help="run all combinations")
+    parser.addoption(
+        "--psij",
+        action="store",
+        help="run with psij subtype plugin",
+        choices=["local", "slurm"],
+    )
 
 
 def pytest_generate_tests(metafunc):
@@ -21,6 +27,16 @@ def pytest_generate_tests(metafunc):
         except ValueError:
             # Called as --pyargs, so --dask isn't available
             pass
+        try:
+            if metafunc.config.getoption("psij"):
+                Plugins.append("psij-" + metafunc.config.getoption("psij"))
+                if (
+                    bool(shutil.which("sbatch"))
+                    and metafunc.config.getoption("psij") == "slurm"
+                ):
+                    Plugins.remove("slurm")
+        except ValueError:
+            pass
         metafunc.parametrize("plugin_dask_opt", Plugins)
 
     if "plugin" in metafunc.fixturenames:
@@ -35,6 +51,16 @@ def pytest_generate_tests(metafunc):
             Plugins = ["slurm"]
         else:
             Plugins = ["cf"]
+        try:
+            if metafunc.config.getoption("psij"):
+                Plugins.append("psij-" + metafunc.config.getoption("psij"))
+                if (
+                    bool(shutil.which("sbatch"))
+                    and metafunc.config.getoption("psij") == "slurm"
+                ):
+                    Plugins.remove("slurm")
+        except ValueError:
+            pass
         metafunc.parametrize("plugin", Plugins)
 
 

pydra/engine/helpers.py

@@ -261,7 +261,8 @@ def make_klass(spec):
                     type=tp,
                     **kwargs,
                 )
-            type_checker = TypeParser[newfield.type](newfield.type)
+            checker_label = f"'{name}' field of {spec.name}"
+            type_checker = TypeParser[newfield.type](newfield.type, label=checker_label)
             if newfield.type in (MultiInputObj, MultiInputFile):
                 converter = attr.converters.pipe(ensure_list, type_checker)
             elif newfield.type in (MultiOutputObj, MultiOutputFile):
@@ -652,7 +653,11 @@ def argstr_formatting(argstr, inputs, value_updates=None):
     for fld in inp_fields:
         fld_name = fld[1:-1]  # extracting the name form {field_name}
         fld_value = inputs_dict[fld_name]
-        if fld_value is attr.NOTHING:
+        fld_attr = getattr(attrs.fields(type(inputs)), fld_name)
+        if fld_value is attr.NOTHING or (
+            fld_value is False
+            and TypeParser.matches_type(fld_attr.type, ty.Union[Path, bool])
+        ):
             # if value is NOTHING, nothing should be added to the command
             val_dict[fld_name] = ""
         else: