98BT refactoring (#119)

* [BUG] inside unit_tests workflow

* Refactoring pipeline 98BT

* [TEST] update tests [REFAC] pep8 and bug correction

* Using narps_open.data.task module inside 98BT

* [TEST] update + dependancy to niflow

* [REFAC][skip ci]

* [BUG] input of get_contrats [skip ci]

* Light bugs + remove large files [skip ci]

* [TEST] adding tests for 98BT

* [TEST] adding tests for 98BT

* [TEST] adding tests for 98BT

* Output names for 98BT

* Preprocessing output names

* Subject info issue + output names

* 98BT - Refac get_parameters_files

* [TEST] 98BT get_parameters_file

* Using fixture for temporary test dir creation / removal

* Linting pipelines test files

* Bug with regressors naming

* pylintrc for pylint config

narps_open/pipelines/__init__.py

@@ -35,7 +35,7 @@
     '6VV2': None,
     '80GC': None,
     '94GU': None,
-    '98BT': None,
+    '98BT': 'PipelineTeam98BT',
     '9Q6R': None,
     '9T8E': None,
     '9U7M': None,

setup.py

@@ -19,7 +19,8 @@
     'networkx>=2.0,<3.0', # a workaround to nipype's bug (issue 3530)
     'nilearn>=0.10.0,<0.11',
     'nipype>=1.8.6,<1.9',
-    'pandas>=1.5.2,<1.6'
+    'pandas>=1.5.2,<1.6',
+    'niflow-nipype1-workflows>=0.0.5,<0.1.0'
 ]
 extras_require = {
     'tests': [

tests/conftest.py

@@ -14,6 +14,7 @@
 from numpy import isclose
 from pytest import helpers, fixture
 from pathvalidate import is_valid_filepath
+from numpy import isclose
 
 from narps_open.pipelines import Pipeline
 from narps_open.runner import PipelineRunner

tests/pipelines/templates/template_test.py

@@ -3,21 +3,21 @@
 
 """ This template can be use to test a pipeline.
 
-    - Replace all occurrences of XXXX by the actual id of the team.
+    - Replace all occurrences of 2T6S by the actual id of the team.
     - All lines starting with [INFO], are meant to help you during the reproduction,
     these can be removed eventually.
     - Also remove lines starting with [TODO], once you did what they suggested.
     - Remove this docstring once you are done with coding the tests.
 """
 
-""" Tests of the 'narps_open.pipelines.team_XXXX' module.
+""" Tests of the 'narps_open.pipelines.team_2T6S' module.
 
 Launch this test with PyTest
 
 Usage:
 ======
-    pytest -q test_team_XXXX.py
-    pytest -q test_team_XXXX.py -k <selected_test>
+    pytest -q test_team_2T6S.py
+    pytest -q test_team_2T6S.py -k <selected_test>
 """
 
 # [INFO] About these imports :
@@ -28,12 +28,12 @@
 # [INFO] Only for type testing
 from nipype import Workflow
 
-# [INFO] Of course, import the class you want to test, here the Pipeline class for the team XXXX
-from narps_open.pipelines.team_XXXX import PipelineTeamXXXX
+# [INFO] Of course, import the class you want to test, here the Pipeline class for the team 2T6S
+from narps_open.pipelines.team_2T6S import PipelineTeam2T6S
 
 # [INFO] All tests should be contained in the following class, in order to sort them.
-class TestPipelinesTeamXXXX:
-    """ A class that contains all the unit tests for the PipelineTeamXXXX class."""
+class TestPipelinesTeam2T6S:
+    """ A class that contains all the unit tests for the PipelineTeam2T6S class."""
 
     # [TODO] Write one or several unit_test (and mark them as such)
     # [TODO]    ideally for each method of the class you test.
@@ -42,19 +42,19 @@ class TestPipelinesTeamXXXX:
     @staticmethod
     @mark.unit_test
     def test_create():
-        """ Test the creation of a PipelineTeamXXXX object """
+        """ Test the creation of a PipelineTeam2T6S object """
 
-        pipeline = PipelineTeamXXXX()
+        pipeline = PipelineTeam2T6S()
         assert pipeline.fwhm == 8.0
-        assert pipeline.team_id == 'XXXX'
+        assert pipeline.team_id == '2T6S'
 
     # [INFO] Here is one example for the methods returning workflows
     @staticmethod
     @mark.unit_test
     def test_workflows():
-        """ Test the workflows of a PipelineTeamXXXX object """
+        """ Test the workflows of a PipelineTeam2T6S object """
 
-        pipeline = PipelineTeamXXXX()
+        pipeline = PipelineTeam2T6S()
         assert pipeline.get_preprocessing() is None
         assert pipeline.get_run_level_analysis() is None
         assert isinstance(pipeline.get_subject_level_analysis(), Workflow)
@@ -68,24 +68,16 @@ def test_workflows():
     @staticmethod
     @mark.unit_test
     def test_outputs():
-        """ Test the expected outputs of a PipelineTeamXXXX object """
-        pipeline = PipelineTeamXXXX()
+        """ Test the expected outputs of a PipelineTeam2T6S object """
+        pipeline = PipelineTeam2T6S()
 
         # 1 - 1 subject outputs
         pipeline.subject_list = ['001']
-        assert len(pipeline.get_preprocessing_outputs()) == 0
-        assert len(pipeline.get_run_level_outputs()) == 0
-        assert len(pipeline.get_subject_level_outputs()) == 7
-        assert len(pipeline.get_group_level_outputs()) == 63
-        assert len(pipeline.get_hypotheses_outputs()) == 18
+        helpers.test_pipeline_outputs(pipeline, [0, 0, 7, 63, 18])
 
         # 2 - 4 subjects outputs
         pipeline.subject_list = ['001', '002', '003', '004']
-        assert len(pipeline.get_preprocessing_outputs()) == 0
-        assert len(pipeline.get_run_level_outputs()) == 0
-        assert len(pipeline.get_subject_level_outputs()) == 28
-        assert len(pipeline.get_group_level_outputs()) == 63
-        assert len(pipeline.get_hypotheses_outputs()) == 18
+        helpers.test_pipeline_outputs(pipeline, [0, 0, 28, 63, 18])
 
     # [TODO] Feel free to add other methods, e.g. to test the custom node functions of the pipeline
 
@@ -95,8 +87,8 @@ def test_outputs():
     @staticmethod
     @mark.pipeline_test
     def test_execution():
-        """ Test the execution of a PipelineTeamXXXX and compare results """
+        """ Test the execution of a PipelineTeam2T6S and compare results """
 
         # [INFO] We use the `test_pipeline_evaluation` helper which is responsible for running the
         # [INFO]    pipeline, iterating over subjects and comparing output with expected results.
-        helpers.test_pipeline_evaluation('XXXX')
+        helpers.test_pipeline_evaluation('2T6S')

tests/pipelines/test_team_98BT.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python
+# coding: utf-8
+
+""" Tests of the 'narps_open.pipelines.team_98BT' module.
+
+Launch this test with PyTest
+
+Usage:
+======
+    pytest -q test_team_98BT.py
+    pytest -q test_team_98BT.py -k <selected_test>
+"""
+from os.path import join, exists
+from filecmp import cmp
+
+from pytest import helpers, mark
+from nipype import Workflow
+from nipype.interfaces.base import Bunch
+
+from narps_open.utils.configuration import Configuration
+from narps_open.pipelines.team_98BT import PipelineTeam98BT
+
+TEMPORARY_DIR = join(Configuration()['directories']['test_runs'], 'test_98BT')
+
+class TestPipelinesTeam98BT:
+    """ A class that contains all the unit tests for the PipelineTeam98BT class."""
+
+    @staticmethod
+    @mark.unit_test
+    def test_create():
+        """ Test the creation of a PipelineTeam98BT object """
+
+        pipeline = PipelineTeam98BT()
+
+        # 1 - check the parameters
+        assert pipeline.fwhm == 8.0
+        assert pipeline.team_id == '98BT'
+
+        # 2 - check workflows
+        processing = pipeline.get_preprocessing()
+        assert len(processing) == 2
+        for sub_workflow in processing:
+            assert isinstance(sub_workflow, Workflow)
+
+        assert pipeline.get_run_level_analysis() is None
+        assert isinstance(pipeline.get_subject_level_analysis(), Workflow)
+
+        group_level = pipeline.get_group_level_analysis()
+        assert len(group_level) == 3
+        for sub_workflow in group_level:
+            assert isinstance(sub_workflow, Workflow)
+
+    @staticmethod
+    @mark.unit_test
+    def test_outputs():
+        """ Test the expected outputs of a PipelineTeam98BT object """
+        pipeline = PipelineTeam98BT()
+        # 1 - 1 subject outputs
+        pipeline.subject_list = ['001']
+        helpers.test_pipeline_outputs(pipeline, [1 + 1*1 + 4*1*5,0,9,84,18])
+
+        # 2 - 4 subjects outputs
+        pipeline.subject_list = ['001', '002', '003', '004']
+        helpers.test_pipeline_outputs(pipeline, [1 + 4*1 + 4*4*5,0,36,84,18])
+
+    @staticmethod
+    @mark.unit_test
+    def test_fieldmap_info():
+        """ Test the get_fieldmap_info method """
+
+        filedmap_file_1 = join(
+            Configuration()['directories']['test_data'], 'pipelines', 'phasediff_1.json')
+        filedmap_file_2 = join(
+            Configuration()['directories']['test_data'], 'pipelines', 'phasediff_2.json')
+
+        test_result = PipelineTeam98BT.get_fieldmap_info(
+            filedmap_file_1, ['magnitude_1', 'magnitude_2'])
+        assert test_result[0] == (0.00492, 0.00738)
+        assert test_result[1] == 'magnitude_1'
+        test_result = PipelineTeam98BT.get_fieldmap_info(
+            filedmap_file_2, ['magnitude_1', 'magnitude_2'])
+        assert test_result[0] == (0.00492, 0.00738)
+        assert test_result[1] == 'magnitude_2'
+
+    @staticmethod
+    @mark.unit_test
+    @mark.parametrize('remove_test_dir', TEMPORARY_DIR)
+    def test_parameters_files(remove_test_dir):
+        """ Test the get_parameters_files method
+            For this test, we created the two following files by downsampling output files
+                from the preprocessing pipeline :
+                - wc2sub-001_T1w-32.nii (white matter file)
+                - uasub-001_task-MGT_run-01_bold_resampled-32.nii (motion corrected file)
+            Voxel dimension was multiplied by 32, number of slices was reduced to 4.
+        """
+        parameters_file = join(
+            Configuration()['directories']['test_data'], 'pipelines', 'confounds.tsv')
+        func_file = join(Configuration()['directories']['test_data'], 'pipelines',
+            'team_98BT', 'uasub-001_task-MGT_run-01_bold_resampled-32.nii')
+        wc2_file = join(Configuration()['directories']['test_data'], 'pipelines',
+            'team_98BT', 'wc2sub-001_T1w-32.nii')
+        reference_file = join(
+            Configuration()['directories']['test_data'], 'pipelines',
+            'team_98BT', 'parameters_file.tsv')
+
+        # Get new parameters file
+        PipelineTeam98BT.get_parameters_file(
+            parameters_file, wc2_file, func_file, 'sid', 'rid', TEMPORARY_DIR)
+
+        # Check parameters file was created
+        created_parameters_file = join(
+            TEMPORARY_DIR, 'parameters_files', 'parameters_file_sub-sid_run-rid.tsv')
+        assert exists(created_parameters_file)
+
+        # Check contents
+        assert cmp(reference_file, created_parameters_file)
+
+    @staticmethod
+    @mark.unit_test
+    def test_subject_information():
+        """ Test the get_subject_information method """
+
+        # Get test files
+        test_file = join(Configuration()['directories']['test_data'], 'pipelines', 'events.tsv')
+
+        bunch = PipelineTeam98BT.get_subject_information(test_file, 1)
+
+        # Compare bunches to expected
+        assert isinstance(bunch, Bunch)
+        assert bunch.conditions == ['gamble_run1']
+        helpers.compare_float_2d_arrays(bunch.onsets, [
+            [4.071, 11.834, 19.535, 27.535, 36.435]])
+        helpers.compare_float_2d_arrays(bunch.durations, [
+            [4.0, 4.0, 4.0, 4.0, 4.0]])
+        assert bunch.amplitudes is None
+        assert bunch.tmod is None
+        assert bunch.regressor_names is None
+        assert bunch.regressors is None
+        pmod = bunch.pmod[0]
+        assert isinstance(pmod, Bunch)
+        assert pmod.name == ['gain_run1', 'loss_run1', 'answers_run1']
+        assert pmod.poly == [1, 1, 1]
+        helpers.compare_float_2d_arrays(pmod.param, [
+            [14.0, 34.0, 38.0, 10.0, 16.0],
+            [6.0, 14.0, 19.0, 15.0, 17.0],
+            [1, 1, 0, 0, 0]
+            ])
+
+    @staticmethod
+    @mark.pipeline_test
+    def test_execution():
+        """ Test the execution of a PipelineTeam98BT and compare results """
+        helpers.test_pipeline_evaluation('98BT')

tests/test_data/pipelines/phasediff_1.json

@@ -0,0 +1 @@
+{"EchoTime1": 0.00492, "EchoTime2": 0.00738, "IntendedFor": ["func/sub-001_task-MGT_run-01_bold.nii.gz", "func/sub-001_task-MGT_run-02_bold.nii.gz", "func/sub-001_task-MGT_run-03_bold.nii.gz", "func/sub-001_task-MGT_run-04_bold.nii.gz"]}

tests/test_data/pipelines/phasediff_2.json

@@ -0,0 +1 @@
+{"EchoTime1": 0.00738, "EchoTime2": 0.00492, "IntendedFor": ["func/sub-001_task-MGT_run-01_bold.nii.gz", "func/sub-001_task-MGT_run-02_bold.nii.gz", "func/sub-001_task-MGT_run-03_bold.nii.gz", "func/sub-001_task-MGT_run-04_bold.nii.gz"]}

tests/test_data/pipelines/team_98BT/parameters_file.tsv

@@ -0,0 +1,4 @@
+"CSF	WhiteMatter	GlobalSignal	stdDVARS	non-stdDVARS	vx-wisestdDVARS	FramewiseDisplacement	tCompCor00	tCompCor01	tCompCor02	tCompCor03	tCompCor04	tCompCor05	aCompCor00	aCompCor01	aCompCor02	aCompCor03	aCompCor04	aCompCor05	Cosine00	Cosine01	Cosine02	Cosine03	Cosine04	Cosine05	NonSteadyStateOutlier00	X	Y	Z	RotX	RotY	RotZ"	261.948061726888
+"6551.281999999999	6476.4653	9874.576	n/a	n/a	n/a	n/a	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	1.0	0.0	0.0	0.0	0.0	-0.0	0.0"	241.43458658854166
+"6484.7285	6473.4890000000005	9830.212	1.09046686	52.78273392	1.05943739	0.13527900930999998	0.0263099209	-0.0673065879	0.0934882554	-0.0079328884	0.0338007737	-0.011491083999999999	-0.042411347099999996	0.027736422900000002	0.0453303087	-0.07022609490000001	0.0963618709	-0.0200867957	0.0665186088	0.0665174038	0.0665153954	0.0665125838	0.0665089688	0.06650455059999999	0.0	-0.00996895	-0.0313444	-3.00931e-06	0.00132687	-0.000384193	-0.00016819"	246.2539803059896
+"6441.5337	6485.7256	9821.212	1.07520139	52.04382706	1.03821933	0.12437666391	-0.0404820317	0.034150583	0.13661184210000002	0.0745358691	-0.0054829985999999995	-0.0217322686	0.046214115199999996	0.005774624	-0.043909359800000006	-0.075619539	0.17546891539999998	-0.0345256763	0.0665153954	0.06650455059999999	0.06648647719999999	0.0664611772	0.0664286533	0.0663889091	0.0	-2.56954e-05	-0.00923735	0.0549667	0.000997278	-0.00019745	-0.000398988"	246.21754170735676