From 25891232871c1642dccd549129a0afc484e974e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= <117362283+bclenet@users.noreply.github.com> Date: Thu, 21 Sep 2023 11:04:55 +0200 Subject: [PATCH 1/8] 2T6S pipeline reproduction (#75) * Dummy change to test 2T6S pipeline on CI with rectified results data * [REFAC] 2T6S pipeline * [BUG] cleaning connections * [BUG] cleaning connections * [TEST] init a test for conftest.py * [BUG] inside unit_tests workflow * [TEST] testing the conftest module * Issue with parameters dir creation * Bug with makedirs import * Workaround to change hypotheses 5&8 files * Change correlation threshold values --- narps_open/pipelines/team_2T6S.py | 134 ++++++++++-------- .../utils/configuration/testing_config.toml | 2 +- 2 files changed, 77 insertions(+), 59 deletions(-) diff --git a/narps_open/pipelines/team_2T6S.py b/narps_open/pipelines/team_2T6S.py index 42aa344f..6c42201a 100755 --- a/narps_open/pipelines/team_2T6S.py +++ b/narps_open/pipelines/team_2T6S.py @@ -1,7 +1,7 @@ #!/usr/bin/python # coding: utf-8 -""" Write the work of NARPS' team 2T6S using Nipype """ +""" Write the work of NARPS team 2T6S using Nipype """ from os.path import join from itertools import product @@ -78,8 +78,10 @@ def get_subject_infos(event_files, runs): duration[val].append(float(info[4])) # durations for trial (rpz by RT) else: # trial with no response : duration of 4 s duration[val].append(float(4)) - weights_gain[val_gain].append(float(info[2])) # weights gain for trial_run1 - weights_loss[val_loss].append(-1.0 * float(info[3])) # weights loss for trial_run1 + # weights gain for trial_run1 + weights_gain[val_gain].append(float(info[2])) + # weights loss for trial_run1 + weights_loss[val_loss].append(-1.0 * float(info[3])) # Bunching is done per run, i.e. trial_run1, trial_run2, etc. # But names must not have '_run1' etc because we concatenate runs @@ -141,11 +143,11 @@ def get_parameters_file(filepaths, subject_id, working_dir): Return : - parameters_file : paths to new files containing only desired parameters. """ - from os import mkdir + from os import makedirs from os.path import join, isdir - import pandas as pd - import numpy as np + from pandas import read_csv, DataFrame + from numpy import array, transpose # Handle the case where filepaths is a single path (str) if not isinstance(filepaths, list): @@ -154,21 +156,20 @@ def get_parameters_file(filepaths, subject_id, working_dir): # Create the parameters files parameters_file = [] for file_id, file in enumerate(filepaths): - data_frame = pd.read_csv(file, sep = '\t', header=0) + data_frame = read_csv(file, sep = '\t', header=0) # Extract parameters we want to use for the model - temp_list = np.array([ + temp_list = array([ data_frame['X'], data_frame['Y'], data_frame['Z'], data_frame['RotX'], data_frame['RotY'], data_frame['RotZ']]) - retained_parameters = pd.DataFrame(np.transpose(temp_list)) + retained_parameters = DataFrame(transpose(temp_list)) # Write parameters to a parameters file # TODO : warning !!! filepaths must be ordered (1,2,3,4) for the following code to work - new_path =join(working_dir, 'parameters_file', + new_path = join(working_dir, 'parameters_file', f'parameters_file_sub-{subject_id}_run-{str(file_id + 1).zfill(2)}.tsv') - if not isdir(join(working_dir, 'parameters_file')): - mkdir(join(working_dir, 'parameters_file')) + makedirs(join(working_dir, 'parameters_file'), exist_ok = True) with open(new_path, 'w') as writer: writer.write(retained_parameters.to_csv( @@ -187,11 +188,11 @@ def remove_gunzip_files(_, subject_id, working_dir): Parameters: - _: Node input only used for triggering the Node - - subject_id: str, TODO - - working_id: str, TODO + - subject_id: str, subject id from which to remove the unzipped file + - working_dir: str, path to the working directory """ - from shutil import rmtree from os.path import join + from shutil import rmtree try: rmtree(join(working_dir, 'l1_analysis', f'_subject_id_{subject_id}', 'gunzip_func')) @@ -209,11 +210,11 @@ def remove_smoothed_files(_, subject_id, working_dir): Parameters: - _: Node input only used for triggering the Node - - subject_id: str, TODO - - working_id: str, TODO + - subject_id: str, subject id from which to remove the smoothed file + - working_dir: str, path to the working directory """ - from shutil import rmtree from os.path import join + from shutil import rmtree try: rmtree(join(working_dir, 'l1_analysis', f'_subject_id_{subject_id}', 'smooth')) @@ -231,11 +232,7 @@ def get_subject_level_analysis(self): """ # Infosource Node - To iterate on subjects infosource = Node(IdentityInterface( - fields = ['subject_id', 'dataset_dir', 'results_dir', 'working_dir', 'run_list'], - dataset_dir = self.directories.dataset_dir, - results_dir = self.directories.results_dir, - working_dir = self.directories.working_dir, - run_list = self.run_list), + fields = ['subject_id']), name = 'infosource') infosource.iterables = [('subject_id', self.subject_list)] @@ -275,6 +272,7 @@ def get_subject_level_analysis(self): input_names = ['event_files', 'runs'], output_names = ['subject_info']), name = 'subject_infos') + subject_infos.inputs.runs = self.run_list # SpecifyModel - generates SPM-specific Model specify_model = Node(SpecifySPMModel( @@ -332,16 +330,13 @@ def get_subject_level_analysis(self): l1_analysis = Workflow(base_dir = self.directories.working_dir, name = 'l1_analysis') l1_analysis.connect([ (infosource, selectfiles, [('subject_id', 'subject_id')]), - (infosource, subject_infos, [('run_list', 'runs')]), (infosource, remove_gunzip_files, [('subject_id', 'subject_id')]), (infosource, remove_smoothed_files, [('subject_id', 'subject_id')]), + (infosource, parameters, [('subject_id', 'subject_id')]), (subject_infos, specify_model, [('subject_info', 'subject_info')]), (contrasts, contrast_estimate, [('contrasts', 'contrasts')]), (selectfiles, parameters, [('param', 'filepaths')]), (selectfiles, subject_infos, [('event', 'event_files')]), - (infosource, parameters, [ - ('subject_id', 'subject_id'), - ('working_dir', 'working_dir')]), (selectfiles, gunzip_func, [('func', 'in_file')]), (gunzip_func, smooth, [('out_file', 'in_files')]), (smooth, specify_model, [('smoothed_files', 'functional_runs')]), @@ -401,8 +396,10 @@ def get_subset_contrasts(file_list, subject_list, participants_file): Returns : - equal_indifference_id : a list of subject ids in the equalIndifference group - equal_range_id : a list of subject ids in the equalRange group - - equal_indifference_files : a subset of file_list corresponding to subjects in the equalIndifference group - - equal_range_files : a subset of file_list corresponding to subjects in the equalRange group + - equal_indifference_files : a subset of file_list corresponding to + subjects in the equalIndifference group + - equal_range_files : a subset of file_list corresponding to + subjects in the equalRange group """ equal_indifference_id = [] equal_range_id = [] @@ -454,8 +451,7 @@ def get_group_level_analysis_sub_workflow(self, method): # Infosource - iterate over the list of contrasts infosource_groupanalysis = Node( IdentityInterface( - fields = ['contrast_id', 'subjects'], - subjects = self.subject_list), + fields = ['contrast_id', 'subjects']), name = 'infosource_groupanalysis') infosource_groupanalysis.iterables = [('contrast_id', self.contrast_list)] @@ -469,7 +465,7 @@ def get_group_level_analysis_sub_workflow(self, method): } selectfiles_groupanalysis = Node(SelectFiles( - templates, base_directory=self.directories.results_dir, force_list= True), + templates, base_directory = self.directories.results_dir, force_list = True), name = 'selectfiles_groupanalysis') # Datasink - save important files @@ -481,14 +477,14 @@ def get_group_level_analysis_sub_workflow(self, method): # Function node get_subset_contrasts - select subset of contrasts sub_contrasts = Node(Function( function = self.get_subset_contrasts, - input_names = ['file_list', 'method', 'subject_list', 'participants_file'], + input_names = ['file_list', 'subject_list', 'participants_file'], output_names = [ 'equalIndifference_id', 'equalRange_id', 'equalIndifference_files', 'equalRange_files']), name = 'sub_contrasts') - sub_contrasts.inputs.method = method + sub_contrasts.inputs.subject_list = self.subject_list # Estimate model estimate_model = Node(EstimateModel( @@ -513,8 +509,6 @@ def get_group_level_analysis_sub_workflow(self, method): l2_analysis.connect([ (infosource_groupanalysis, selectfiles_groupanalysis, [ ('contrast_id', 'contrast_id')]), - (infosource_groupanalysis, sub_contrasts, [ - ('subjects', 'subject_list')]), (selectfiles_groupanalysis, sub_contrasts, [ ('contrast', 'file_list'), ('participants', 'participants_file')]), @@ -618,29 +612,53 @@ def get_group_level_outputs(self): return return_list def get_hypotheses_outputs(self): - """ Return all hypotheses output file names. - Note that hypotheses 5 to 8 correspond to the maps given by the team in their results ; - but they are not fully consistent with the hypotheses definitions as expected by NARPS. - """ + """ Return all hypotheses output file names. """ nb_sub = len(self.subject_list) files = [ - join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_0002', '_threshold0', 'spmT_0001_thr.nii'), - join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_0002', 'spmT_0001.nii'), - join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_0002', '_threshold0', 'spmT_0001_thr.nii'), - join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_0002', 'spmT_0001.nii'), - join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_0002', '_threshold0', 'spmT_0001_thr.nii'), - join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_0002', 'spmT_0001.nii'), - join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_0002', '_threshold0', 'spmT_0001_thr.nii'), - join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_0002', 'spmT_0001.nii'), - join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_0003', '_threshold1', 'spmT_0002_thr.nii'), - join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_0003', 'spmT_0002.nii'), - join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_0003', '_threshold1', 'spmT_0001_thr.nii'), - join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_0003', 'spmT_0001.nii'), - join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_0003', '_threshold0', 'spmT_0001_thr.nii'), - join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_contrast_id_0003', 'spmT_0001.nii'), - join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_0003', '_threshold0', 'spmT_0002_thr.nii'), - join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_contrast_id_0003', 'spmT_0002.nii'), - join(f'l2_analysis_groupComp_nsub_{nb_sub}', '_contrast_id_0003', '_threshold0', 'spmT_0001_thr.nii'), - join(f'l2_analysis_groupComp_nsub_{nb_sub}', '_contrast_id_0003', 'spmT_0001.nii') + # Hypothesis 1 + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_0002', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_0002', 'spmT_0001.nii'), + # Hypothesis 2 + join(f'l2_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_0002', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_0002', 'spmT_0001.nii'), + # Hypothesis 3 + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_0002', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_0002', 'spmT_0001.nii'), + # Hypothesis 4 + join(f'l2_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_0002', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_0002', 'spmT_0001.nii'), + # Hypothesis 5 + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_0003', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_0003', 'spmT_0001.nii'), + # Hypothesis 6 + join(f'l2_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_0003', '_threshold1', 'spmT_0002_thr.nii'), + join(f'l2_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_0003', 'spmT_0002.nii'), + # Hypothesis 7 + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_0003', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', + '_contrast_id_0003', 'spmT_0001.nii'), + # Hypothesis 8 + join(f'l2_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_0003', '_threshold1', 'spmT_0002_thr.nii'), + join(f'l2_analysis_equalRange_nsub_{nb_sub}', + '_contrast_id_0003', 'spmT_0002.nii'), + # Hypothesis 9 + join(f'l2_analysis_groupComp_nsub_{nb_sub}', + '_contrast_id_0003', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_groupComp_nsub_{nb_sub}', + '_contrast_id_0003', 'spmT_0001.nii') ] return [join(self.directories.output_dir, f) for f in files] diff --git a/narps_open/utils/configuration/testing_config.toml b/narps_open/utils/configuration/testing_config.toml index ec0ab8d1..b1fb28ba 100644 --- a/narps_open/utils/configuration/testing_config.toml +++ b/narps_open/utils/configuration/testing_config.toml @@ -19,4 +19,4 @@ neurovault_naming = true # true if results files are saved using the neurovault [testing] [testing.pipelines] -correlation_thresholds = [0.30, 0.70, 0.80, 0.85, 0.93] # Correlation between reproduced hypotheses files and results, respectively for [20, 40, 60, 80, 108] subjects. +correlation_thresholds = [0.30, 0.70, 0.79, 0.85, 0.93] # Correlation between reproduced hypotheses files and results, respectively for [20, 40, 60, 80, 108] subjects. From 2fa3a0dbb4fd9184dbe03d7fde965bbb29dfca98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= <117362283+bclenet@users.noreply.github.com> Date: Tue, 26 Sep 2023 08:53:13 +0200 Subject: [PATCH 2/8] Q6O0 reproduction (#68) * [Q6O0] refactoring pipeline * [Q6O0] updating list of implemented pipelines * [TEST] change implemented pipeline test * [CI] code quality change * [CI] code quality change * [CI] record pylint logs in case of failure only * [CI] install whole package before linting * [CI] adding higher timeouts to self-hosted jobs * [BUG] adding inputs to subject info node * [BUG] changes in pipeline workflow * [BUG] unnecessary commented line in pipeline * [REFAC] remove distance as unused in model design * [TEST] init a test for conftest.py * [BUG] inside unit_tests workflow * [TEST] testing the conftest module * Issue with parameters dir creation * Bug with makedirs import * [TEST] sorting results before computing correlations --- .github/workflows/code_quality.yml | 13 +- .github/workflows/pipeline_tests.yml | 2 +- .github/workflows/test_changes.yml | 2 +- narps_open/pipelines/__init__.py | 2 +- narps_open/pipelines/team_Q6O0.py | 1351 ++++++++++++++------------ tests/conftest.py | 2 +- tests/pipelines/test_pipelines.py | 8 +- tests/pipelines/test_team_Q6O0.py | 68 ++ 8 files changed, 798 insertions(+), 650 deletions(-) create mode 100644 tests/pipelines/test_team_Q6O0.py diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index 2804e813..a9248671 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -8,9 +8,11 @@ on: push: paths: - '**.py' + - '.github/workflows/code_quality.yml' pull_request: paths: - '**.py' + - '.github/workflows/code_quality.yml' # Jobs that define the workflow jobs: @@ -33,22 +35,23 @@ jobs: - uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-pylint + key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} restore-keys: | - ${{ runner.os }}-pip-pylint + ${{ runner.os }}-pip- - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pylint + pip install .[tests] - name: Analyse the code with pylint run: | - pylint --exit-zero narps_open > pylint_report_narps_open.txt - pylint --exit-zero tests > pylint_report_tests.txt + pylint --fail-under 8 --ignore-paths narps_open/pipelines/ narps_open > pylint_report_narps_open.txt + pylint --fail-under 8 tests > pylint_report_tests.txt - name: Archive pylint results uses: actions/upload-artifact@v3 + if: failure() # Only if previous step failed with: name: pylint-reports-python path: | diff --git a/.github/workflows/pipeline_tests.yml b/.github/workflows/pipeline_tests.yml index 66e91a91..132a7662 100644 --- a/.github/workflows/pipeline_tests.yml +++ b/.github/workflows/pipeline_tests.yml @@ -47,7 +47,7 @@ jobs: echo "tests=$test_files" >> $GITHUB_OUTPUT echo "teams=$teams" >> $GITHUB_OUTPUT - # A job to identify and run the tests + # A job to run the tests pytest: needs: identify-tests runs-on: self-hosted diff --git a/.github/workflows/test_changes.yml b/.github/workflows/test_changes.yml index 8c582197..e0608011 100644 --- a/.github/workflows/test_changes.yml +++ b/.github/workflows/test_changes.yml @@ -38,7 +38,7 @@ jobs: echo $test_files echo "tests=$test_files" >> $GITHUB_OUTPUT - # A job to list the tests to be run + # A job to run the tests pytest: needs: identify-tests runs-on: self-hosted diff --git a/narps_open/pipelines/__init__.py b/narps_open/pipelines/__init__.py index c3834fb6..6c5239ca 100644 --- a/narps_open/pipelines/__init__.py +++ b/narps_open/pipelines/__init__.py @@ -62,7 +62,7 @@ 'O6R6': None, 'P5F3': None, 'Q58J': None, - 'Q6O0': None, + 'Q6O0': 'PipelineTeamQ6O0', 'R42Q': None, 'R5K7': None, 'R7D1': None, diff --git a/narps_open/pipelines/team_Q6O0.py b/narps_open/pipelines/team_Q6O0.py index 5eb0462a..69cacc3c 100755 --- a/narps_open/pipelines/team_Q6O0.py +++ b/narps_open/pipelines/team_Q6O0.py @@ -1,643 +1,720 @@ -from nipype.interfaces.spm import (Smooth, OneSampleTTestDesign, EstimateModel, EstimateContrast, Level1Design, - TwoSampleTTestDesign) -from nipype.interfaces.spm import Threshold as Analysis_Threshold # to distinguish with FSL Threshold interface -from nipype.algorithms.modelgen import SpecifySPMModel +#!/usr/bin/python +# coding: utf-8 + +""" Write the work of NARPS' team Q6O0 using Nipype """ + +from os.path import join +from itertools import product + +from nipype import Workflow, Node, MapNode from nipype.interfaces.utility import IdentityInterface, Function from nipype.interfaces.io import SelectFiles, DataSink +from nipype.interfaces.spm import ( + Smooth, + Level1Design, OneSampleTTestDesign, TwoSampleTTestDesign, + EstimateModel, EstimateContrast, Threshold + ) +from nipype.algorithms.modelgen import SpecifySPMModel from nipype.algorithms.misc import Gunzip -from nipype import Workflow, Node, MapNode -from nipype.interfaces.base import Bunch - -from os.path import join as opj -import os -import json - -def get_subject_infos_gain(event_files): - ''' - Create Bunchs for specifySPMModel. - Here, the team wanted to concatenate runs and used RT (response time) for duration except for NoResponse trials - for which the duration was set to 4. - Gain and loss amounts were used as parametric regressors. - - Parameters : - - event_files : list of files containing events information for each run - - Returns : - - subject_info : list of Bunch for 1st level analysis. - ''' - from os.path import join as opj - from nipype.interfaces.base import Bunch - - cond_names = ['trial'] - onset = {} - duration = {} - weights_gain = {} - weights_loss = {} - weights_distance = {} - runs = ['01', '02', '03', '04'] - - for r in range(len(runs)): # Loop over number of runs. - onset.update({s + '_run' + str(r+1) : [] for s in cond_names}) # creates dictionary items with empty lists - duration.update({s + '_run' + str(r+1) : [] for s in cond_names}) - weights_gain.update({'gain_run' + str(r+1) : []}) - weights_loss.update({'loss_run' + str(r+1) : []}) - weights_distance.update({'distance_run' + str(r+1) : []}) - - # subject_id = '001' - # file = sub-001_func_sub-001_task-MGT_run-01_events.tsv - for r, f_events in enumerate(event_files): - with open(f_events, 'rt') as f: - next(f) # skip the header - - for line in f: - info = line.strip().split() - - for cond in cond_names: - val = cond + '_run' + str(r+1) # trial_run1 - val_gain = 'gain_run' + str(r+1) # gain_run1 - val_loss = 'loss_run' + str(r+1) # loss_run1 - val_distance = 'distance_run' + str(r+1) - onset[val].append(float(info[0])) # onsets for trial_run1 - duration[val].append(float(4)) # durations for trial : 4 - weights_gain[val_gain].append(float(info[2])) # weights gain for trial_run1 - weights_loss[val_loss].append(float(info[3])) # weights loss for trial_run1 - weights_distance[val_distance].append(abs(0.5*(float(info[2])) - float(info[3]))) - - # Bunching is done per run, i.e. trial_run1, trial_run2, etc. - # But names must not have '_run1' etc because we concatenate runs - subject_info = [] - for r in range(len(runs)): - - cond = [c + '_run' + str(r+1) for c in cond_names] - gain = 'gain_run' + str(r+1) - loss = 'loss_run' + str(r+1) - distance = 'distance_run' + str(r+1) - - subject_info.insert(r, - Bunch(conditions=cond_names, - onsets=[onset[k] for k in cond], - durations=[duration[k] for k in cond], - amplitudes=None, - tmod=None, - pmod=[Bunch(name=['loss', 'gain'], - poly=[1, 1], - param=[weights_loss[loss], - weights_gain[gain]])], - regressor_names=None, - regressors=None)) - - return subject_info - -def get_subject_infos_loss(event_files): - ''' - Create Bunchs for specifySPMModel. - Here, the team wanted to concatenate runs and used RT (response time) for duration except for NoResponse trials - for which the duration was set to 4. - Gain and loss amounts were used as parametric regressors. - - Parameters : - - event_files : list of files containing events information for each run - - Returns : - - subject_info : list of Bunch for 1st level analysis. - ''' - from os.path import join as opj - from nipype.interfaces.base import Bunch - - cond_names = ['trial'] - onset = {} - duration = {} - weights_gain = {} - weights_loss = {} - weights_distance = {} - runs = ['01', '02', '03', '04'] - - for r in range(len(runs)): # Loop over number of runs. - onset.update({s + '_run' + str(r+1) : [] for s in cond_names}) # creates dictionary items with empty lists - duration.update({s + '_run' + str(r+1) : [] for s in cond_names}) - weights_gain.update({'gain_run' + str(r+1) : []}) - weights_loss.update({'loss_run' + str(r+1) : []}) - weights_distance.update({'distance_run' + str(r+1) : []}) - - # subject_id = '001' - # file = sub-001_func_sub-001_task-MGT_run-01_events.tsv - for r, f_events in enumerate(event_files): - with open(f_events, 'rt') as f: - next(f) # skip the header - - for line in f: - info = line.strip().split() - - for cond in cond_names: - val = cond + '_run' + str(r+1) # trial_run1 - val_gain = 'gain_run' + str(r+1) # gain_run1 - val_loss = 'loss_run' + str(r+1) # loss_run1 - val_distance = 'distance_run' + str(r+1) - onset[val].append(float(info[0])) # onsets for trial_run1 - duration[val].append(float(4)) # durations for trial : 4 - weights_gain[val_gain].append(float(info[2])) # weights gain for trial_run1 - weights_loss[val_loss].append(float(info[3])) # weights loss for trial_run1 - weights_distance[val_distance].append(abs(0.5*(float(info[2])) - float(info[3]))) - - # Bunching is done per run, i.e. trial_run1, trial_run2, etc. - # But names must not have '_run1' etc because we concatenate runs - subject_info = [] - for r in range(len(runs)): - - cond = [c + '_run' + str(r+1) for c in cond_names] - gain = 'gain_run' + str(r+1) - loss = 'loss_run' + str(r+1) - distance = 'distance_run' + str(r+1) - - subject_info.insert(r, - Bunch(conditions=cond_names, - onsets=[onset[k] for k in cond], - durations=[duration[k] for k in cond], - amplitudes=None, - tmod=None, - pmod=[Bunch(name=['gain', 'loss'], - poly=[1, 1], - param=[weights_gain[gain], - weights_loss[loss]])], - regressor_names=None, - regressors=None)) - - return subject_info - -def get_contrasts_gain(subject_id): - ''' - Create the list of tuples that represents contrasts. - Each contrast is in the form : - (Name,Stat,[list of condition names],[weights on those conditions]) - - Parameters: - - subject_id: str, ID of the subject - - Returns: - - contrasts: list of tuples, list of contrasts to analyze - ''' - # list of condition names - conditions = ['trialxgain^1'] - - # create contrasts - pos_effect_gain = ('pos_effect_of_gain', 'T', conditions, [1]) - - neg_effect_gain = ('neg_effect_of_gain', 'T', conditions, [-1]) - - # contrast list - contrasts = [pos_effect_gain] - - return contrasts - -def get_contrasts_loss(subject_id): - ''' - Create the list of tuples that represents contrasts. - Each contrast is in the form : - (Name,Stat,[list of condition names],[weights on those conditions]) - - Parameters: - - subject_id: str, ID of the subject - - Returns: - - contrasts: list of tuples, list of contrasts to analyze - ''' - # list of condition names - conditions = ['trialxloss^1'] - - # create contrasts - pos_effect_loss = ('pos_effect_of_loss', 'T', conditions, [1]) - - neg_effect_loss = ('neg_effect_of_loss', 'T', conditions, [-1]) - - # contrast list - contrasts = [pos_effect_loss] - - return contrasts - -def get_parameters_file(filepaths, subject_id, result_dir, working_dir): - ''' - Create new tsv files with only desired parameters per subject per run. - The six motion parameters, the 5 aCompCor parameters, the global white matter and - cerebral spinal fluid signals were included as nuisance regressors/ - - Parameters : - - filepaths : paths to subject parameters file (i.e. one per run) - - subject_id : subject for whom the 1st level analysis is made - - result_dir: str, directory where results will be stored - - working_dir: str, name of the sub-directory for intermediate results - - Return : - - parameters_file : paths to new files containing only desired parameters. - ''' - import pandas as pd - import numpy as np - from os.path import join as opj - import os - - if not isinstance(filepaths, list): - filepaths = [filepaths] - parameters_file = [] - - for i, file in enumerate(filepaths): - df = pd.read_csv(file, sep = '\t', header=0) - temp_list = np.array([df['X'], df['Y'], df['Z'], - df['RotX'], df['RotY'], df['RotZ'], df['aCompCor00'], - df['aCompCor01'], df['aCompCor02'], df['aCompCor03'], - df['aCompCor04'], df['aCompCor05'], df['WhiteMatter'], - df['CSF']]) # Parameters we want to use for the model - retained_parameters = pd.DataFrame(np.transpose(temp_list)) - new_path =opj(result_dir, working_dir, 'parameters_file', - f"parameters_file_sub-{subject_id}_run0{str(i+1)}.tsv") - if not os.path.isdir(opj(result_dir, working_dir, 'parameters_file')): - os.mkdir(opj(result_dir, working_dir, 'parameters_file')) - writer = open(new_path, "w") - writer.write(retained_parameters.to_csv(sep = '\t', index = False, header = False, na_rep = '0.0')) - writer.close() - - parameters_file.append(new_path) - - return parameters_file - -def rm_gunzip_files(files, subject_id, result_dir, working_dir): - import shutil - from os.path import join as opj - - gunzip_dir = opj(result_dir, working_dir, 'l1_analysis', f"_subject_id_{subject_id}", 'gunzip_func') - - try: - shutil.rmtree(gunzip_dir) - except OSError as e: - print(e) - else: - print("The directory is deleted successfully") - - return files - -def rm_smoothed_files(files, subject_id, result_dir, working_dir): - import shutil - from os.path import join as opj - - smooth_dir = opj(result_dir, working_dir, 'l1_analysis', f"_subject_id_{subject_id}", 'smooth') - - try: - shutil.rmtree(smooth_dir) - except OSError as e: - print(e) - else: - print("The directory is deleted successfully") - - return files - - -def get_l1_analysis(subject_list, TR, fwhm, run_list, exp_dir, result_dir, working_dir, output_dir): - """ - Returns the first level analysis workflow. - - Parameters: - - exp_dir: str, directory where raw data are stored - - result_dir: str, directory where results will be stored - - working_dir: str, name of the sub-directory for intermediate results - - output_dir: str, name of the sub-directory for final results - - subject_list: list of str, list of subject for which you want to do the analysis - - run_list: list of str, list of runs for which you want to do the analysis - - fwhm: float, fwhm for smoothing step - - TR: float, time repetition used during acquisition - - Returns: - - l1_analysis : Nipype WorkFlow - """ - # Infosource Node - To iterate on subjects - infosource = Node(IdentityInterface(fields = ['subject_id']), name = 'infosource') - infosource.iterables = [('subject_id', subject_list)] - - # Templates to select files node - func_file = opj('derivatives', 'fmriprep', 'sub-{subject_id}', 'func', - 'sub-{subject_id}_task-MGT_run-*_bold_space-MNI152NLin2009cAsym_preproc.nii.gz') - - event_file = opj('sub-{subject_id}', 'func', - 'sub-{subject_id}_task-MGT_run-*_events.tsv') - - param_file = opj('derivatives', 'fmriprep', 'sub-{subject_id}', 'func', - 'sub-{subject_id}_task-MGT_run-*_bold_confounds.tsv') - - template = {'param' : param_file, 'event' : event_file, 'func' : func_file} - - # SelectFiles node - to select necessary files - selectfiles = Node(SelectFiles(template, base_directory=exp_dir), name = 'selectfiles') - - # DataSink Node - store the wanted results in the wanted repository - datasink = Node(DataSink(base_directory=result_dir, container=output_dir), name='datasink') - - # GUNZIP NODE : SPM do not use .nii.gz files - gunzip_func = MapNode(Gunzip(), name = 'gunzip_func', iterfield = ['in_file']) - - ## Smoothing node - smooth = Node(Smooth(fwhm = fwhm), name = 'smooth') - - # Get Subject Info - get subject specific condition information - subject_infos_gain = Node(Function(input_names=['event_files'], - output_names=['subject_info'], - function=get_subject_infos_gain), - name='subject_infos_gain') - - subject_infos_loss = Node(Function(input_names=['event_files'], - output_names=['subject_info'], - function=get_subject_infos_loss), - name='subject_infos_loss') - - # Node parameters to get parameters files - parameters = Node(Function(function=get_parameters_file, - input_names=['filepaths', 'subject_id', 'result_dir', 'working_dir'], - output_names=['parameters_file']), - name='parameters') - - parameters.inputs.result_dir = result_dir - parameters.inputs.working_dir = working_dir - - # SpecifyModel - Generates SPM-specific Model - specify_model_gain = Node(SpecifySPMModel(concatenate_runs = True, input_units = 'secs', output_units = 'secs', - time_repetition = TR, high_pass_filter_cutoff = 128), - name='specify_model_gain') - - specify_model_loss = Node(SpecifySPMModel(concatenate_runs = True, input_units = 'secs', output_units = 'secs', - time_repetition = TR, high_pass_filter_cutoff = 128), - name='specify_model_loss') - - # Level1Design - Generates an SPM design matrix - l1_design_gain = Node(Level1Design(bases = {'hrf': {'derivs': [0, 0]}}, timing_units = 'secs', - interscan_interval = TR, model_serial_correlations = 'AR(1)'), name='l1_design_gain') - - l1_design_loss = Node(Level1Design(bases = {'hrf': {'derivs': [0, 0]}}, timing_units = 'secs', - interscan_interval = TR, model_serial_correlations = 'AR(1)'), name='l1_design_loss') - - # EstimateModel - estimate the parameters of the model - l1_estimate_gain = Node(EstimateModel(estimation_method={'Classical': 1}), - name="l1_estimate_gain") - - l1_estimate_loss = Node(EstimateModel(estimation_method={'Classical': 1}), - name="l1_estimate_loss") - - # Node contrasts to get contrasts - contrasts_gain = Node(Function(function=get_contrasts_gain, - input_names=['subject_id'], - output_names=['contrasts']), - name='contrasts_gain') - - contrasts_loss = Node(Function(function=get_contrasts_loss, - input_names=['subject_id'], - output_names=['contrasts']), - name='contrasts_loss') - - # EstimateContrast - estimates contrasts - contrast_estimate_gain = Node(EstimateContrast(), name="contrast_estimate_gain") - - contrast_estimate_loss = Node(EstimateContrast(), name="contrast_estimate_loss") - - remove_gunzip_files = Node(Function(input_names = ['files', 'subject_id', 'result_dir', 'working_dir'], - output_names = ['files'], - function = rm_gunzip_files), name = 'remove_gunzip_files') - - remove_gunzip_files.inputs.result_dir = result_dir - remove_gunzip_files.inputs.working_dir = working_dir - - remove_smoothed_files = Node(Function(input_names = ['files', 'subject_id', 'result_dir', 'working_dir'], - output_names = ['files'], - function = rm_smoothed_files), name = 'remove_smoothed_files') - - remove_smoothed_files.inputs.result_dir = result_dir - remove_smoothed_files.inputs.working_dir = working_dir - - # Create l1 analysis workflow and connect its nodes - l1_analysis = Workflow(base_dir = opj(result_dir, working_dir), name = "l1_analysis") - - l1_analysis.connect([(infosource, selectfiles, [('subject_id', 'subject_id')]), - (selectfiles, subject_infos_gain, [('event','event_files')]), - (selectfiles, subject_infos_loss, [('event','event_files')]), - (selectfiles, parameters, [('param', 'filepaths')]), - (infosource, parameters, [('subject_id', 'subject_id')]), - (infosource, contrasts_gain, [('subject_id', 'subject_id')]), - (infosource, contrasts_loss, [('subject_id', 'subject_id')]), - (infosource, remove_gunzip_files, [('subject_id', 'subject_id')]), - (infosource, remove_smoothed_files, [('subject_id', 'subject_id')]), - (subject_infos_gain, specify_model_gain, [('subject_info', 'subject_info')]), - (subject_infos_loss, specify_model_loss, [('subject_info', 'subject_info')]), - (contrasts_gain, contrast_estimate_gain, [('contrasts', 'contrasts')]), - (contrasts_loss, contrast_estimate_loss, [('contrasts', 'contrasts')]), - (selectfiles, gunzip_func, [('func', 'in_file')]), - (gunzip_func, smooth, [('out_file', 'in_files')]), - (smooth, remove_gunzip_files, [('smoothed_files', 'files')]), - (remove_gunzip_files, specify_model_gain, [('files', 'functional_runs')]), - (remove_gunzip_files, specify_model_loss, [('files', 'functional_runs')]), - (parameters, specify_model_gain, [('parameters_file', 'realignment_parameters')]), - (parameters, specify_model_loss, [('parameters_file', 'realignment_parameters')]), - (specify_model_gain, l1_design_gain, [('session_info', 'session_info')]), - (specify_model_loss, l1_design_loss, [('session_info', 'session_info')]), - (l1_design_gain, l1_estimate_gain, [('spm_mat_file', 'spm_mat_file')]), - (l1_design_loss, l1_estimate_loss, [('spm_mat_file', 'spm_mat_file')]), - (l1_estimate_gain, contrast_estimate_gain, [('spm_mat_file', 'spm_mat_file'), - ('beta_images', 'beta_images'), - ('residual_image', 'residual_image')]), - (l1_estimate_loss, contrast_estimate_loss, [('spm_mat_file', 'spm_mat_file'), - ('beta_images', 'beta_images'), - ('residual_image', 'residual_image')]), - (contrast_estimate_gain, datasink, [('con_images', 'l1_analysis_gain.@con_images'), - ('spmT_images', 'l1_analysis_gain.@spmT_images'), - ('spm_mat_file', 'l1_analysis_gain.@spm_mat_file')]), - (contrast_estimate_loss, datasink, [('con_images', 'l1_analysis_loss.@con_images'), - ('spmT_images', 'l1_analysis_loss.@spmT_images'), - ('spm_mat_file', 'l1_analysis_loss.@spm_mat_file')]), - (contrast_estimate_gain, remove_smoothed_files, [('spmT_images', 'files')]) - ]) - - return l1_analysis - -def get_subset_contrasts(file_list, method, subject_list, participants_file): - ''' - Parameters : - - file_list : original file list selected by selectfiles node - - subject_list : list of subject IDs that are in the wanted group for the analysis - - participants_file: str, file containing participants characteristics - - method: str, one of "equalRange", "equalIndifference" or "groupComp" - - This function return the file list containing only the files belonging to subject in the wanted group. - ''' - equalIndifference_id = [] - equalRange_id = [] - equalIndifference_files = [] - equalRange_files = [] - - with open(participants_file, 'rt') as f: - next(f) # skip the header - - for line in f: + +from narps_open.pipelines import Pipeline + +class PipelineTeamQ6O0(Pipeline): + """ A class that defines the pipeline of team Q6O0. """ + + def __init__(self): + super().__init__() + self.fwhm = 8.0 + self.team_id = 'Q6O0' + self.contrast_list = ['0001'] + self.model_list = ['gain', 'loss'] + + def get_preprocessing(self): + """ No preprocessing has been done by team Q6O0 """ + return None + + def get_run_level_analysis(self): + """ No run level analysis has been done by team Q6O0 """ + return None + + # @staticmethod # Starting python 3.10, staticmethod should be used here + # Otherwise it produces a TypeError: 'staticmethod' object is not callable + def get_subject_infos(event_files, runs, model): + """ Create Bunchs for specifySPMModel. + Here, the team wanted to concatenate runs and used RT (response time) + for duration except for NoResponse trials for which the duration was set to 4. + Gain and loss amounts were used as parametric regressors. + + Parameters : + - event_files : list of files containing events information for each run + - runs: list of str, list of runs to use + - model: str, either 'gain' or 'loss'. + + Returns : + - subject_info : list of Bunch for 1st level analysis. + """ + from nipype.interfaces.base import Bunch + + condition_names = ['trial'] + onset = {} + duration = {} + weights_gain = {} + weights_loss = {} + + for run_id in range(len(runs)): # Loop over number of runs. + # creates dictionary items with empty lists + onset.update({s + '_run' + str(run_id + 1) : [] for s in condition_names}) + duration.update({s + '_run' + str(run_id + 1) : [] for s in condition_names}) + weights_gain.update({'gain_run' + str(run_id + 1) : []}) + weights_loss.update({'loss_run' + str(run_id + 1) : []}) + + for run_id, event_file in enumerate(event_files): + with open(event_file, 'rt') as file: + next(file) # skip the header + + for line in file: + info = line.strip().split() + + for condition in condition_names: + val = condition + '_run' + str(run_id + 1) # trial_run1 + val_gain = 'gain_run' + str(run_id + 1) # gain_run1 + val_loss = 'loss_run' + str(run_id + 1) # loss_run1 + onset[val].append(float(info[0])) # onsets for trial_run1 + duration[val].append(float(4)) # durations for trial : 4 + weights_gain[val_gain].append(float(info[2])) # weights gain for trial_run1 + weights_loss[val_loss].append(float(info[3])) # weights loss for trial_run1 + + # Bunching is done per run, i.e. trial_run1, trial_run2, etc. + # But names must not have '_run1' etc because we concatenate runs + subject_info = [] + for run_id in range(len(runs)): + + conditions = [c + '_run' + str(run_id + 1) for c in condition_names] + gain = 'gain_run' + str(run_id + 1) + loss = 'loss_run' + str(run_id + 1) + + if model == 'gain': + parametric_modulation_bunch = Bunch( + name = ['loss', 'gain'], + poly = [1, 1], + param = [weights_loss[loss], weights_gain[gain]] + ) + elif model == 'loss': + parametric_modulation_bunch = Bunch( + name = ['gain', 'loss'], + poly = [1, 1], + param = [weights_gain[gain], weights_loss[loss]] + ) + else: + raise AttributeError('Model must be gain or loss.') + + subject_info.insert( + run_id, + Bunch( + conditions = condition_names, + onsets=[onset[k] for k in conditions], + durations = [duration[k] for k in conditions], + amplitudes = None, + tmod = None, + pmod = [parametric_modulation_bunch], + regressor_names = None, + regressors = None) + ) + + return subject_info + + def get_contrasts_gain(subject_id): + """ + Create the list of tuples that represents contrasts. + Each contrast is in the form : + (Name,Stat,[list of condition names],[weights on those conditions]) + + Parameters: + - subject_id: str, ID of the subject + + Returns: + - contrasts: list of tuples, list of contrasts to analyze + """ + # List of condition names + conditions = ['trialxgain^1'] + + # Create contrasts + positive_effect_gain = ('positive_effect_gain', 'T', conditions, [1]) + + # Return contrast list + return [positive_effect_gain] + + def get_contrasts_loss(subject_id): + """ + Create the list of tuples that represents contrasts. + Each contrast is in the form : + (Name,Stat,[list of condition names],[weights on those conditions]) + + Parameters: + - subject_id: str, ID of the subject + + Returns: + - contrasts: list of tuples, list of contrasts to analyze + """ + # List of condition names + conditions = ['trialxloss^1'] + + # Create contrasts + positive_effect_loss = ('positive_effect_loss', 'T', conditions, [1]) + + # Return contrast list + return [positive_effect_loss] + + def get_parameters_file(filepaths, subject_id, working_dir): + """ + Create new tsv files with only desired parameters per subject per run. + The six motion parameters, the 5 aCompCor parameters, the global white matter and + cerebral spinal fluid signals were included as nuisance regressors/ + + Parameters : + - filepaths : paths to subject parameters file (i.e. one per run) + - subject_id : subject for whom the 1st level analysis is made + - working_dir: str, name of the sub-directory for intermediate results + + Return : + - parameters_file : paths to new files containing only desired parameters. + """ + from os import makedirs + from os.path import join, isdir + + import pandas as pd + import numpy as np + + # Handle the case where filepaths is a single path (str) + if not isinstance(filepaths, list): + filepaths = [filepaths] + + # Create the parameters files + parameters_file = [] + for file_id, file in enumerate(filepaths): + data_frame = pd.read_csv(file, sep = '\t', header=0) + + # Extract parameters we want to use for the model + temp_list = np.array([ + data_frame['X'], data_frame['Y'], data_frame['Z'], + data_frame['RotX'], data_frame['RotY'], data_frame['RotZ'], + data_frame['aCompCor00'], data_frame['aCompCor01'], data_frame['aCompCor02'], + data_frame['aCompCor03'], data_frame['aCompCor04'], data_frame['aCompCor05'], + data_frame['WhiteMatter'], data_frame['CSF']]) + retained_parameters = pd.DataFrame(np.transpose(temp_list)) + + # Write parameters to a parameters file + # TODO : warning !!! filepaths must be ordered (1,2,3,4) for the following code to work + new_path =join(working_dir, 'parameters_file', + f'parameters_file_sub-{subject_id}_run-{str(file_id + 1).zfill(2)}.tsv') + + makedirs(join(working_dir, 'parameters_file'), exist_ok = True) + + with open(new_path, 'w') as writer: + writer.write(retained_parameters.to_csv( + sep = '\t', index = False, header = False, na_rep = '0.0')) + + parameters_file.append(new_path) + + return parameters_file + + def remove_gunzip_files(_, subject_id, working_dir): + """ + This method is used in a Function node to fully remove + the files generated by the gunzip node, once they aren't needed anymore. + + Parameters: + - _: Node input only used for triggering the Node + - subject_id: str, TODO + - working_id: str, TODO + """ + from shutil import rmtree + from os.path import join + + try: + rmtree(join(working_dir, 'l1_analysis', f'_subject_id_{subject_id}', 'gunzip_func')) + except OSError as error: + print(error) + else: + print('The directory is deleted successfully') + + def remove_smoothed_files(_, subject_id, working_dir): + """ + This method is used in a Function node to fully remove + the files generated by the smoothing node, once they aren't needed anymore. + + Parameters: + - _: Node input only used for triggering the Node + - subject_id: str, TODO + - working_id: str, TODO + """ + from shutil import rmtree + from os.path import join + + try: + rmtree(join(working_dir, 'l1_analysis', f'_subject_id_{subject_id}', 'smooth')) + except OSError as error: + print(error) + else: + print('The directory is deleted successfully') + + def get_subject_level_analysis(self): + """ + Create the subject level analysis workflow. + + Returns: + - l1_analysis : nipype.WorkFlow + """ + # Infosource Node - To iterate on subjects + infosource = Node(IdentityInterface( fields = ['subject_id']), + name = 'infosource') + infosource.iterables = [('subject_id', self.subject_list)] + + # Templates to select files node + template = { + 'param' : join('derivatives', 'fmriprep', 'sub-{subject_id}', 'func', + 'sub-{subject_id}_task-MGT_run-*_bold_confounds.tsv'), + 'event' : join('sub-{subject_id}', 'func', + 'sub-{subject_id}_task-MGT_run-*_events.tsv'), + 'func' : join('derivatives', 'fmriprep', 'sub-{subject_id}', 'func', + 'sub-{subject_id}_task-MGT_run-*_bold_space-MNI152NLin2009cAsym_preproc.nii.gz') + } + + # SelectFiles - to select necessary files + selectfiles = Node(SelectFiles(template, base_directory = self.directories.dataset_dir), + name = 'selectfiles') + + # DataSink - store the wanted results in the wanted repository + datasink = Node(DataSink(base_directory = self.directories.output_dir), + name='datasink') + + # Gunzip - gunzip files because SPM do not use .nii.gz files + gunzip_func = MapNode(Gunzip(), name = 'gunzip_func', iterfield = ['in_file']) + + # Smooth - smoothing node + smooth = Node(Smooth(fwhm = self.fwhm), + name = 'smooth') + + # Function node get_subject_infos - get subject specific condition information + subject_infos_gain = Node(Function( + function = self.get_subject_infos, + input_names = ['event_files', 'runs', 'model'], + output_names=['subject_info']), + name='subject_infos_gain') + subject_infos_gain.inputs.runs = self.run_list + subject_infos_gain.inputs.model = 'gain' + + subject_infos_loss = Node(Function( + function = self.get_subject_infos, + input_names = ['event_files', 'runs', 'model'], + output_names = ['subject_info']), + name='subject_infos_loss') + subject_infos_loss.inputs.runs = self.run_list + subject_infos_loss.inputs.model = 'loss' + + # Function node get_parameters_file - get parameters files + parameters = Node(Function( + function = self.get_parameters_file, + input_names = ['filepaths', 'subject_id', 'working_dir'], + output_names = ['parameters_file']), + name = 'parameters') + parameters.inputs.working_dir = self.directories.working_dir + + # SpecifyModel - Generates SPM-specific Model + specify_model_gain = Node(SpecifySPMModel( + concatenate_runs = True, input_units = 'secs', output_units = 'secs', + time_repetition = self.tr, high_pass_filter_cutoff = 128), + name='specify_model_gain') + + specify_model_loss = Node(SpecifySPMModel( + concatenate_runs = True, input_units = 'secs', output_units = 'secs', + time_repetition = self.tr, high_pass_filter_cutoff = 128), + name='specify_model_loss') + + # Level1Design - Generates an SPM design matrix + l1_design_gain = Node(Level1Design( + bases = {'hrf': {'derivs': [0, 0]}}, timing_units = 'secs', + interscan_interval = self.tr, model_serial_correlations = 'AR(1)'), + name='l1_design_gain') + + l1_design_loss = Node(Level1Design( + bases = {'hrf': {'derivs': [0, 0]}}, timing_units = 'secs', + interscan_interval = self.tr, model_serial_correlations = 'AR(1)'), + name='l1_design_loss') + + # EstimateModel - estimate the parameters of the model + l1_estimate_gain = Node(EstimateModel( + estimation_method = {'Classical': 1}), + name = 'l1_estimate_gain') + + l1_estimate_loss = Node(EstimateModel( + estimation_method = {'Classical': 1}), + name = 'l1_estimate_loss') + + # Function nodes get_contrasts_* - get the contrasts + contrasts_gain = Node(Function( + function = self.get_contrasts_gain, + input_names = ['subject_id'], + output_names = ['contrasts']), + name = 'contrasts_gain') + + contrasts_loss = Node(Function( + function = self.get_contrasts_loss, + input_names = ['subject_id'], + output_names = ['contrasts']), + name = 'contrasts_loss') + + # EstimateContrast - estimates contrasts + contrast_estimate_gain = Node(EstimateContrast(), + name = 'contrast_estimate_gain') + + contrast_estimate_loss = Node(EstimateContrast(), + name = 'contrast_estimate_loss') + + # Function node remove_gunzip_files - remove output of the gunzip node + remove_gunzip_files = Node(Function( + function = self.remove_gunzip_files, + input_names = ['_', 'subject_id', 'working_dir'], + output_names = []), + name = 'remove_gunzip_files') + remove_gunzip_files.inputs.working_dir = self.directories.working_dir + + # Function node remove_smoothed_files - remove output of the smooth node + remove_smoothed_files = Node(Function( + function = self.remove_smoothed_files, + input_names = ['_', 'subject_id', 'working_dir'], + output_names = []), + name = 'remove_smoothed_files') + remove_smoothed_files.inputs.working_dir = self.directories.working_dir + + # Create l1 analysis workflow and connect its nodes + l1_analysis = Workflow(base_dir = self.directories.working_dir, name = 'l1_analysis') + l1_analysis.connect([ + (infosource, selectfiles, [('subject_id', 'subject_id')]), + (selectfiles, subject_infos_gain, [('event','event_files')]), + (selectfiles, subject_infos_loss, [('event','event_files')]), + (selectfiles, parameters, [('param', 'filepaths')]), + (infosource, parameters, [('subject_id', 'subject_id')]), + (infosource, contrasts_gain, [('subject_id', 'subject_id')]), + (infosource, contrasts_loss, [('subject_id', 'subject_id')]), + (infosource, remove_gunzip_files, [('subject_id', 'subject_id')]), + (infosource, remove_smoothed_files, [('subject_id', 'subject_id')]), + (subject_infos_gain, specify_model_gain, [('subject_info', 'subject_info')]), + (subject_infos_loss, specify_model_loss, [('subject_info', 'subject_info')]), + (contrasts_gain, contrast_estimate_gain, [('contrasts', 'contrasts')]), + (contrasts_loss, contrast_estimate_loss, [('contrasts', 'contrasts')]), + (selectfiles, gunzip_func, [('func', 'in_file')]), + (gunzip_func, smooth, [('out_file', 'in_files')]), + (smooth, remove_gunzip_files, [('smoothed_files', '_')]), + (smooth, specify_model_gain, [('smoothed_files', 'functional_runs')]), + (smooth, specify_model_loss, [('smoothed_files', 'functional_runs')]), + (parameters, specify_model_gain, [('parameters_file', 'realignment_parameters')]), + (parameters, specify_model_loss, [('parameters_file', 'realignment_parameters')]), + (specify_model_gain, l1_design_gain, [('session_info', 'session_info')]), + (specify_model_loss, l1_design_loss, [('session_info', 'session_info')]), + (l1_design_gain, l1_estimate_gain, [('spm_mat_file', 'spm_mat_file')]), + (l1_design_loss, l1_estimate_loss, [('spm_mat_file', 'spm_mat_file')]), + (l1_estimate_gain, contrast_estimate_gain, [ + ('spm_mat_file', 'spm_mat_file'), + ('beta_images', 'beta_images'), + ('residual_image', 'residual_image')]), + (l1_estimate_loss, contrast_estimate_loss, [ + ('spm_mat_file', 'spm_mat_file'), + ('beta_images', 'beta_images'), + ('residual_image', 'residual_image')]), + (contrast_estimate_gain, datasink, [ + ('con_images', 'l1_analysis_gain.@con_images'), + ('spmT_images', 'l1_analysis_gain.@spmT_images'), + ('spm_mat_file', 'l1_analysis_gain.@spm_mat_file')]), + (contrast_estimate_loss, datasink, [ + ('con_images', 'l1_analysis_loss.@con_images'), + ('spmT_images', 'l1_analysis_loss.@spmT_images'), + ('spm_mat_file', 'l1_analysis_loss.@spm_mat_file')]), + (contrast_estimate_gain, remove_smoothed_files, [('spmT_images', '_')]) + ]) + + return l1_analysis + + def get_subject_level_outputs(self): + """ Return the names of the files the subject level analysis is supposed to generate. """ + + # Generate a list of parameter sets for further templates formatting + parameters = { + 'contrast_id': self.contrast_list, + 'model_type': self.model_list, + 'subject_id': self.subject_list + } + # Combining all possibilities + # Here we use a list because the itertools.product is an iterator objects that + # is meant for a single-use iteration only. + parameter_sets = list(product(*parameters.values())) + + # Contrat maps + contrast_map_template = join( + self.directories.output_dir, + 'l1_analysis_{model_type}', '_subject_id_{subject_id}', 'con_{contrast_id}.nii' + ) + + # SPM.mat file + mat_file_template = join( + self.directories.output_dir, + 'l1_analysis_{model_type}', '_subject_id_{subject_id}', 'SPM.mat' + ) + + # spmT maps + spmt_file_template = join( + self.directories.output_dir, + 'l1_analysis_{model_type}', '_subject_id_{subject_id}', 'spmT_{contrast_id}.nii' + ) + + # Formatting templates and returning it as a list of files + output_files = [contrast_map_template.format(**dict(zip(parameters.keys(), parameter_values)))\ + for parameter_values in parameter_sets] + output_files += [mat_file_template.format(**dict(zip(parameters.keys(), parameter_values)))\ + for parameter_values in parameter_sets] + output_files += [spmt_file_template.format(**dict(zip(parameters.keys(), parameter_values)))\ + for parameter_values in parameter_sets] + + return output_files + + def get_subset_contrasts(file_list, subject_list, participants_file): + """ + Parameters : + - file_list : original file list selected by selectfiles node + - subject_list : list of subject IDs that are in the wanted group for the analysis + - participants_file: str, file containing participants characteristics + + Returns: + - The file list containing only the files belonging to subject in the wanted group. + """ + equal_indifference_id = [] + equal_range_id = [] + equal_indifference_files = [] + equal_range_files = [] + + with open(participants_file, 'rt') as file: + next(file) # skip the header + for line in file: info = line.strip().split() - if info[0][-3:] in subject_list and info[1] == "equalIndifference": - equalIndifference_id.append(info[0][-3:]) + equal_indifference_id.append(info[0][-3:]) elif info[0][-3:] in subject_list and info[1] == "equalRange": - equalRange_id.append(info[0][-3:]) - - for file in file_list: - sub_id = file.split('/') - if sub_id[-2][-3:] in equalIndifference_id: - equalIndifference_files.append(file) - elif sub_id[-2][-3:] in equalRange_id: - equalRange_files.append(file) - - return equalIndifference_id, equalRange_id, equalIndifference_files, equalRange_files - - -def get_l2_analysis(subject_list, n_sub, model_list, method, exp_dir, result_dir, working_dir, output_dir): - """ - Returns the 2nd level of analysis workflow. - - Parameters: - - exp_dir: str, directory where raw data are stored - - result_dir: str, directory where results will be stored - - working_dir: str, name of the sub-directory for intermediate results - - output_dir: str, name of the sub-directory for final results - - subject_list: list of str, list of subject for which you want to do the preprocessing - - model_list: list of str, list of models to use for the analysis - - contrast_list: list of str, list of contrasts to analyze - - n_sub: float, number of subjects used to do the analysis - - method: one of "equalRange", "equalIndifference" or "groupComp" - - Returns: - - l2_analysis: Nipype WorkFlow - """ - # Infosource - a function free node to iterate over the list of subject names - infosource_groupanalysis = Node(IdentityInterface(fields=['subjects', 'model_type'], - subjects = subject_list), - name="infosource_groupanalysis") - - infosource_groupanalysis.iterables = [('model_type', model_list)] - - # SelectFiles - contrast_file = opj(result_dir, output_dir, "l1_analysis_{model_type}", "_subject_id_*", "con_0001.nii") - - participants_file = opj(exp_dir, 'participants.tsv') - - templates = {'contrast' : contrast_file, 'participants' : participants_file} - - selectfiles_groupanalysis = Node(SelectFiles(templates, base_directory=result_dir, force_list= True), - name="selectfiles_groupanalysis") - - # Datasink node : to save important files - datasink_groupanalysis = Node(DataSink(base_directory = result_dir, container = output_dir), - name = 'datasink_groupanalysis') - - # Node to select subset of contrasts - sub_contrasts = Node(Function(input_names = ['file_list', 'method', 'subject_list', 'participants_file'], - output_names = ['equalIndifference_id', 'equalRange_id', 'equalIndifference_files', 'equalRange_files'], - function = get_subset_contrasts), - name = 'sub_contrasts') - - sub_contrasts.inputs.method = method - - ## Estimate model - estimate_model = Node(EstimateModel(estimation_method={'Classical':1}), name = "estimate_model") - - ## Estimate contrasts - estimate_contrast = Node(EstimateContrast(group_contrast=True), - name = "estimate_contrast") - - ## Create thresholded maps - threshold = MapNode(Analysis_Threshold(use_fwe_correction=True, - height_threshold_type='p-value', - force_activation = False), name = "threshold", iterfield = ["stat_image", "contrast_index"]) - - l2_analysis = Workflow(base_dir = opj(result_dir, working_dir), name = 'l2_analysis') - - l2_analysis.connect([(infosource_groupanalysis, selectfiles_groupanalysis, [('model_type', 'model_type')]), - (infosource_groupanalysis, sub_contrasts, [('subjects', 'subject_list')]), - (selectfiles_groupanalysis, sub_contrasts, [('contrast', 'file_list'), ('participants', 'participants_file')]), - (estimate_model, estimate_contrast, [('spm_mat_file', 'spm_mat_file'), - ('residual_image', 'residual_image'), - ('beta_images', 'beta_images')]), - (estimate_contrast, threshold, [('spm_mat_file', 'spm_mat_file'), - ('spmT_images', 'stat_image')]), - (estimate_model, datasink_groupanalysis, [('mask_image', f"l2_analysis_{method}_nsub_{n_sub}.@mask")]), - (estimate_contrast, datasink_groupanalysis, [('spm_mat_file', f"l2_analysis_{method}_nsub_{n_sub}.@spm_mat"), - ('spmT_images', f"l2_analysis_{method}_nsub_{n_sub}.@T"), - ('con_images', f"l2_analysis_{method}_nsub_{n_sub}.@con")]), - (threshold, datasink_groupanalysis, [('thresholded_map', f"l2_analysis_{method}_nsub_{n_sub}.@thresh")])]) - - if method=='equalRange' or method=='equalIndifference': - contrasts = [('Group', 'T', ['mean'], [1]), ('Group', 'T', ['mean'], [-1])] - ## Specify design matrix - one_sample_t_test_design = Node(OneSampleTTestDesign(), name = "one_sample_t_test_design") - - l2_analysis.connect([(sub_contrasts, one_sample_t_test_design, [(f"{method}_files", 'in_files')]), - (one_sample_t_test_design, estimate_model, [('spm_mat_file', 'spm_mat_file')])]) - - threshold.inputs.contrast_index = [1, 2] - threshold.synchronize = True - - elif method == 'groupComp': - contrasts = [('Eq range vs Eq indiff in loss', 'T', ['Group_{1}', 'Group_{2}'], [1, -1])] - # Node for the design matrix - two_sample_t_test_design = Node(TwoSampleTTestDesign(), name = 'two_sample_t_test_design') - - l2_analysis.connect([(sub_contrasts, two_sample_t_test_design, [('equalRange_files', "group1_files"), - ('equalIndifference_files', 'group2_files')]), - (two_sample_t_test_design, estimate_model, [("spm_mat_file", "spm_mat_file")])]) - - threshold.inputs.contrast_index = [1] - threshold.synchronize = True - - estimate_contrast.inputs.contrasts = contrasts - - return l2_analysis - - -def reorganize_results(result_dir, output_dir, n_sub, team_ID): - """ - Reorganize the results to analyze them. - - Parameters: - - result_dir: str, directory where results will be stored - - output_dir: str, name of the sub-directory for final results - - n_sub: float, number of subject used for the analysis - - team_ID: str, ID of the team to reorganize results - - """ - from os.path import join as opj - import os - import shutil - import gzip - - h1 = opj(result_dir, output_dir, f"l2_analysis_equalIndifference_nsub_{n_sub}", '_model_type_gain') - h2 = opj(result_dir, output_dir, f"l2_analysis_equalRange_nsub_{n_sub}", '_model_type_gain') - h3 = opj(result_dir, output_dir, f"l2_analysis_equalIndifference_nsub_{n_sub}", '_model_type_gain') - h4 = opj(result_dir, output_dir, f"l2_analysis_equalRange_nsub_{n_sub}", '_model_type_gain') - h5 = opj(result_dir, output_dir, f"l2_analysis_equalIndifference_nsub_{n_sub}", '_model_type_loss') - h6 = opj(result_dir, output_dir, f"l2_analysis_equalRange_nsub_{n_sub}", '_model_type_loss') - h7 = opj(result_dir, output_dir, f"l2_analysis_equalIndifference_nsub_{n_sub}", '_model_type_loss') - h8 = opj(result_dir, output_dir, f"l2_analysis_equalRange_nsub_{n_sub}", '_model_type_loss') - h9 = opj(result_dir, output_dir, f"l2_analysis_groupComp_nsub_{n_sub}", '_model_type_loss') - - h = [h1, h2, h3, h4, h5, h6, h7, h8, h9] - - repro_unthresh = [opj(filename, "spmT_0002.nii") if i in [4, 5] else opj(filename, "spmT_0001.nii") for i, filename in enumerate(h)] - - repro_thresh = [opj(filename, "_threshold1", - "spmT_0002_thr.nii") if i in [4, 5] else opj(filename, "_threshold0", "spmT_0001_thr.nii") for i, filename in enumerate(h)] - - if not os.path.isdir(opj(result_dir, "NARPS-reproduction")): - os.mkdir(opj(result_dir, "NARPS-reproduction")) - - for i, filename in enumerate(repro_unthresh): - f_in = filename - f_out = opj(result_dir, "NARPS-reproduction", f"team_{team_ID}_nsub_{n_sub}_hypo{i+1}_unthresholded.nii") - shutil.copyfile(f_in, f_out) - - for i, filename in enumerate(repro_thresh): - f_in = filename - f_out = opj(result_dir, "NARPS-reproduction", f"team_{team_ID}_nsub_{n_sub}_hypo{i+1}_thresholded.nii") - shutil.copyfile(f_in, f_out) - - print(f"Results files of team {team_ID} reorganized.") - - - - + equal_range_id.append(info[0][-3:]) + + for file in file_list: + sub_id = file.split('/') + if sub_id[-2][-3:] in equal_indifference_id: + equal_indifference_files.append(file) + elif sub_id[-2][-3:] in equal_range_id: + equal_range_files.append(file) + + return equal_indifference_id, equal_range_id, equal_indifference_files, equal_range_files + + def get_group_level_analysis(self): + """ + Return all workflows for the group level analysis. + + Returns; + - a list of nipype.WorkFlow + """ + + methods = ['equalRange', 'equalIndifference', 'groupComp'] + return [self.get_group_level_analysis_sub_workflow(method) for method in methods] + + def get_group_level_analysis_sub_workflow(self, method): + """ + Return a workflow for the group level analysis. + + Parameters: + - method: one of 'equalRange', 'equalIndifference' or 'groupComp' + + Returns: + - l2_analysis: nipype.WorkFlow + """ + # Compute the number of participants used to do the analysis + nb_subjects = len(self.subject_list) + + # Infosource - iterate over the list of contrasts + infosource_groupanalysis = Node( + IdentityInterface( + fields=['subjects', 'model_type'], + subjects = self.subject_list), + name='infosource_groupanalysis') + infosource_groupanalysis.iterables = [('model_type', self.model_list)] + + # SelectFiles + templates = { + 'contrast' : join(self.directories.output_dir, + 'l1_analysis_{model_type}', '_subject_id_*', 'con_0001.nii'), + 'participants' : join(self.directories.dataset_dir, 'participants.tsv') + } + + selectfiles_groupanalysis = Node(SelectFiles( + templates, + base_directory = self.directories.results_dir, + force_list= True), + name="selectfiles_groupanalysis") + + # Datasink - save important files + datasink_groupanalysis = Node(DataSink( + base_directory = str(self.directories.output_dir) + ), + name = 'datasink_groupanalysis') + + # Function node get_subset_contrasts - select subset of contrasts + sub_contrasts = Node(Function( + function = self.get_subset_contrasts, + input_names = ['file_list', 'subject_list', 'participants_file'], + output_names = [ + 'equalIndifference_id', + 'equalRange_id', + 'equalIndifference_files', + 'equalRange_files']), + name = 'sub_contrasts') + + # Estimate model + estimate_model = Node(EstimateModel( + estimation_method={'Classical':1}), + name = "estimate_model") + + # Estimate contrasts + estimate_contrast = Node(EstimateContrast( + group_contrast=True), + name = "estimate_contrast") + + # Create thresholded maps + threshold = MapNode(Threshold( + use_fwe_correction = True, + height_threshold_type = 'p-value', + force_activation = False), + name = "threshold", iterfield = ['stat_image', 'contrast_index']) + + l2_analysis = Workflow( + base_dir = self.directories.working_dir, + name = f'l2_analysis_{method}_nsub_{nb_subjects}') + + l2_analysis.connect([ + (infosource_groupanalysis, selectfiles_groupanalysis, [ + ('model_type', 'model_type')]), + (infosource_groupanalysis, sub_contrasts, [ + ('subjects', 'subject_list')]), + (selectfiles_groupanalysis, sub_contrasts, [ + ('contrast', 'file_list'), + ('participants', 'participants_file')]), + (estimate_model, estimate_contrast, [ + ('spm_mat_file', 'spm_mat_file'), + ('residual_image', 'residual_image'), + ('beta_images', 'beta_images')]), + (estimate_contrast, threshold, [ + ('spm_mat_file', 'spm_mat_file'), + ('spmT_images', 'stat_image')]), + (estimate_model, datasink_groupanalysis, [ + ('mask_image', f"l2_analysis_{method}_nsub_{nb_subjects}.@mask")]), + (estimate_contrast, datasink_groupanalysis, [ + ('spm_mat_file', f"l2_analysis_{method}_nsub_{nb_subjects}.@spm_mat"), + ('spmT_images', f"l2_analysis_{method}_nsub_{nb_subjects}.@T"), + ('con_images', f"l2_analysis_{method}_nsub_{nb_subjects}.@con")]), + (threshold, datasink_groupanalysis, [ + ('thresholded_map', f"l2_analysis_{method}_nsub_{nb_subjects}.@thresh")])]) + + if method in ('equalRange', 'equalIndifference'): + contrasts = [('Group', 'T', ['mean'], [1]), ('Group', 'T', ['mean'], [-1])] + + # Specify design matrix + one_sample_t_test_design = Node(OneSampleTTestDesign(), + name = 'one_sample_t_test_design') + + l2_analysis.connect([ + (sub_contrasts, one_sample_t_test_design, [(f"{method}_files", 'in_files')]), + (one_sample_t_test_design, estimate_model, [('spm_mat_file', 'spm_mat_file')])]) + + threshold.inputs.contrast_index = [1, 2] + threshold.synchronize = True + + elif method == 'groupComp': + contrasts = [( + 'Eq range vs Eq indiff in loss', 'T', ['Group_{1}', 'Group_{2}'], [1, -1])] + + # Specify design matrix + two_sample_t_test_design = Node(TwoSampleTTestDesign(), + name = 'two_sample_t_test_design') + + l2_analysis.connect([ + (sub_contrasts, two_sample_t_test_design, [ + ('equalRange_files', "group1_files"), + ('equalIndifference_files', 'group2_files')]), + (two_sample_t_test_design, estimate_model, [("spm_mat_file", "spm_mat_file")])]) + + threshold.inputs.contrast_index = [1] + threshold.synchronize = True + + estimate_contrast.inputs.contrasts = contrasts + + return l2_analysis + + def get_group_level_outputs(self): + """ Return all names for the files the group level analysis is supposed to generate. """ + + # Handle equalRange and equalIndifference + parameters = { + 'model_type': self.model_list, + 'method': ['equalRange', 'equalIndifference'], + 'file': [ + 'con_0001.nii', 'con_0002.nii', 'mask.nii', 'SPM.mat', + 'spmT_0001.nii', 'spmT_0002.nii', + join('_threshold0', 'spmT_0001_thr.nii'), join('_threshold1', 'spmT_0002_thr.nii') + ], + 'nb_subjects': [str(len(self.subject_list))] + } + parameter_sets = product(*parameters.values()) + template = join( + self.directories.output_dir, + 'l2_analysis_{method}_nsub_{nb_subjects}', + '_model_type_{model_type}', + '{file}' + ) + + return_list = [template.format(**dict(zip(parameters.keys(), parameter_values)))\ + for parameter_values in parameter_sets] + + # Handle groupComp + parameters = { + 'model_type': ['loss'], + 'method': ['groupComp'], + 'file': [ + 'con_0001.nii', 'mask.nii', 'SPM.mat', 'spmT_0001.nii', + join('_threshold0', 'spmT_0001_thr.nii') + ], + 'nb_subjects' : [str(len(self.subject_list))] + } + parameter_sets = product(*parameters.values()) + + return_list += [template.format(**dict(zip(parameters.keys(), parameter_values)))\ + for parameter_values in parameter_sets] + + return return_list + + def get_hypotheses_outputs(self): + """ Return all hypotheses output file names. """ + nb_sub = len(self.subject_list) + files = [ + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_gain', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_gain', 'spmT_0001.nii'), + join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_gain', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_gain', 'spmT_0001.nii'), + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_gain', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_gain', 'spmT_0001.nii'), + join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_gain', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_gain', 'spmT_0001.nii'), + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_loss', '_threshold1', 'spmT_0002_thr.nii'), + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_loss', 'spmT_0002.nii'), + join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_loss', '_threshold1', 'spmT_0002_thr.nii'), + join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_loss', 'spmT_0002.nii'), + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_loss', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_equalIndifference_nsub_{nb_sub}', '_model_type_loss', 'spmT_0001.nii'), + join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_loss', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_equalRange_nsub_{nb_sub}', '_model_type_loss', 'spmT_0001.nii'), + join(f'l2_analysis_groupComp_nsub_{nb_sub}', '_model_type_loss', '_threshold0', 'spmT_0001_thr.nii'), + join(f'l2_analysis_groupComp_nsub_{nb_sub}', '_model_type_loss', 'spmT_0001.nii') + ] + return [join(self.directories.output_dir, f) for f in files] diff --git a/tests/conftest.py b/tests/conftest.py index e1530e48..7c57c1f9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -92,7 +92,7 @@ def test_pipeline_execution( # Retrieve the paths to the results files collection = ResultsCollection(team_id) - results_files = [join(collection.directory, f) for f in collection.files.keys()] + results_files = [join(collection.directory, f) for f in sorted(collection.files.keys())] results_files = [results_files[i] for i in indices] # Compute the correlation coefficients diff --git a/tests/pipelines/test_pipelines.py b/tests/pipelines/test_pipelines.py index 9016aeb7..c38cf36a 100644 --- a/tests/pipelines/test_pipelines.py +++ b/tests/pipelines/test_pipelines.py @@ -135,8 +135,8 @@ class TestUtils: @mark.unit_test def test_utils(): """ Test the utils methods of PipelineRunner """ - # 1 - Get number of not implemented pipelines - assert len(get_not_implemented_pipelines()) == 69 + # 1 - Get not implemented pipelines + assert '1K0E' in get_not_implemented_pipelines() - # 2 - Get number of implemented pipelines - assert len(get_implemented_pipelines()) == 1 + # 2 - Get implemented pipelines + assert '2T6S' in get_implemented_pipelines() diff --git a/tests/pipelines/test_team_Q6O0.py b/tests/pipelines/test_team_Q6O0.py new file mode 100644 index 00000000..639f609e --- /dev/null +++ b/tests/pipelines/test_team_Q6O0.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +# coding: utf-8 + +""" Tests of the 'narps_open.pipelines.team_Q6O0' module. + +Launch this test with PyTest + +Usage: +====== + pytest -q test_team_Q6O0.py + pytest -q test_team_Q6O0.py -k +""" + +from pytest import helpers, mark +from nipype import Workflow + +from narps_open.pipelines.team_Q6O0 import PipelineTeamQ6O0 + +class TestPipelinesTeamQ6O0: + """ A class that contains all the unit tests for the PipelineTeamQ6O0 class.""" + + @staticmethod + @mark.unit_test + def test_create(): + """ Test the creation of a PipelineTeamQ6O0 object """ + + pipeline = PipelineTeamQ6O0() + + # 1 - check the parameters + assert pipeline.fwhm == 8.0 + assert pipeline.team_id == 'Q6O0' + + # 2 - check workflows + assert pipeline.get_preprocessing() is None + assert pipeline.get_run_level_analysis() is None + assert isinstance(pipeline.get_subject_level_analysis(), Workflow) + + group_level = pipeline.get_group_level_analysis() + assert len(group_level) == 3 + for sub_workflow in group_level: + assert isinstance(sub_workflow, Workflow) + + @staticmethod + @mark.unit_test + def test_outputs(): + """ Test the expected outputs of a PipelineTeamQ6O0 object """ + pipeline = PipelineTeamQ6O0() + # 1 - 1 subject outputs + pipeline.subject_list = ['001'] + assert len(pipeline.get_preprocessing_outputs()) == 0 + assert len(pipeline.get_run_level_outputs()) == 0 + assert len(pipeline.get_subject_level_outputs()) == 6 + assert len(pipeline.get_group_level_outputs()) == 37 + assert len(pipeline.get_hypotheses_outputs()) == 18 + + # 2 - 4 subjects outputs + pipeline.subject_list = ['001', '002', '003', '004'] + assert len(pipeline.get_preprocessing_outputs()) == 0 + assert len(pipeline.get_run_level_outputs()) == 0 + assert len(pipeline.get_subject_level_outputs()) == 24 + assert len(pipeline.get_group_level_outputs()) == 37 + assert len(pipeline.get_hypotheses_outputs()) == 18 + + @staticmethod + @mark.pipeline_test + def test_execution(): + """ Test the execution of a PipelineTeamQ6O0 and compare results """ + helpers.test_pipeline_evaluation('Q6O0') From a3e0a5f77a37cb55258cce53b294bd406c1c3fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= <117362283+bclenet@users.noreply.github.com> Date: Wed, 27 Sep 2023 13:41:23 +0200 Subject: [PATCH 3/8] Fixing the `narps_open.utils.status` module (#109) * [BUG] inside unit_tests workflow * Browsing all issues pages from Github API * Get all pages of GitHub issues * [TEST] Updating test for status module * [TEST] fetch several issues --- docs/status.md | 2 +- narps_open/utils/status.py | 18 ++++-- tests/utils/test_status.py | 123 +++++++++++++++++++++++-------------- 3 files changed, 92 insertions(+), 51 deletions(-) diff --git a/docs/status.md b/docs/status.md index f323cc8f..7fde8239 100644 --- a/docs/status.md +++ b/docs/status.md @@ -36,7 +36,7 @@ print(pipeline_info['status']) report.markdown() # Returns a string containing the markdown ``` -You can also use the command-line tool as so. Option `-t` is for the team id, option `-d` allows to print only one of the sub parts of the description among : `general`, `exclusions`, `preprocessing`, `analysis`, and `categorized_for_analysis`. +You can also use the command-line tool as so. ```bash python narps_open/utils/status -h diff --git a/narps_open/utils/status.py b/narps_open/utils/status.py index cc4eb8a7..fef7708d 100644 --- a/narps_open/utils/status.py +++ b/narps_open/utils/status.py @@ -18,10 +18,20 @@ def get_opened_issues(): """ Return a list of opened issues and pull requests for the NARPS Open Pipelines project """ request_url = 'https://api.github.com/repos/Inria-Empenn/narps_open_pipelines/issues' - response = get(request_url, timeout = 2) - response.raise_for_status() - - return response.json() + request_url += '?page={page_number}?per_page=100' + + issues = [] + page = True # Will later be replaced by a table + page_number = 1 # According to the doc, first page is not page 0 + # https://docs.github.com/en/rest/issues/issues#list-repository-issues + while bool(page) is True : # Test if the page is empty + response = get(request_url.format(page_number = str(page_number)), timeout = 2) + response.raise_for_status() + page = response.json() + issues += page + page_number += 1 + + return issues def get_teams_with_pipeline_files(): """ Return a set of teams having a file for their pipeline in the repository """ diff --git a/tests/utils/test_status.py b/tests/utils/test_status.py index 0e98ef83..6c16279b 100644 --- a/tests/utils/test_status.py +++ b/tests/utils/test_status.py @@ -34,38 +34,62 @@ def mock_api_issue(mocker): which is actually imported as `get` inside the `narps_open.utils.status` module. Hence, we patch the `narps_open.utils.status.get` method. """ - response = Response() - response.status_code = 200 - def json_func(): - return [ - { - "html_url": "url_issue_2", - "number": 2, - "title" : "Issue for pipeline UK24", - "body" : "Nothing to add here." - }, - { - "html_url": "url_pull_3", - "number": 3, - "title" : "Pull request for pipeline 2T6S", - "pull_request" : {}, - "body" : "Work has been done." - }, - { - "html_url": "url_issue_4", - "number": 4, - "title" : None, - "body" : "This is a malformed issue about C88N." - }, - { - "html_url": "url_issue_5", - "number": 5, - "title" : "Issue about 2T6S", - "body" : "Something about 2T6S." - } - ] - response.json = json_func - mocker.patch('narps_open.utils.status.get', return_value = response) + + # Create a method to mock requests.get + def mocked_requests_get(url, params=None, **kwargs): + + response = Response() + response.status_code = 200 + def json_func_page_0(): + return [ + { + "html_url": "url_issue_2", + "number": 2, + "title" : "Issue for pipeline UK24", + "body" : "Nothing to add here." + }, + { + "html_url": "url_pull_3", + "number": 3, + "title" : "Pull request for pipeline 2T6S", + "pull_request" : {}, + "body" : "Work has been done." + } + ] + + json_func_page_1 = json_func_page_0 + + def json_func_page_2(): + return [ + { + "html_url": "url_issue_4", + "number": 4, + "title" : None, + "body" : "This is a malformed issue about C88N." + }, + { + "html_url": "url_issue_5", + "number": 5, + "title" : "Issue about 2T6S", + "body" : "Something about 2T6S." + } + ] + + def json_func_page_3(): + return [] + + if '?page=1' in url: + response.json = json_func_page_1 + elif '?page=2' in url: + response.json = json_func_page_2 + elif '?page=3' in url: + response.json = json_func_page_3 + else: + response.json = json_func_page_0 + + return response + + mocker.patch('narps_open.utils.status.get', side_effect = mocked_requests_get) mocker.patch( 'narps_open.utils.status.get_teams_with_pipeline_files', return_value = ['2T6S', 'UK24', 'Q6O0'] @@ -94,8 +118,6 @@ def test_get_issues(mocker): which is actually imported as `get` inside the `narps_open.utils.status` module. Hence, we patch the `narps_open.utils.status.get` method. """ - get_opened_issues() - # Create a mock API response for 404 error response = Response() response.status_code = 404 @@ -114,18 +136,27 @@ def json_func(): assert len(get_opened_issues()) == 0 # Create a mock API response for the general usecase - response = Response() - response.status_code = 200 - def json_func(): - return [ - { - "html_url": "urls", - "number": 2, - } - ] - response.json = json_func - - mocker.patch('narps_open.utils.status.get', return_value = response) + def mocked_requests_get(url, params=None, **kwargs): + response = Response() + response.status_code = 200 + def json_func_page_1(): + return [ + { + "html_url": "urls", + "number": 2, + } + ] + def json_func_page_2(): + return [] + + if '?page=2' in url: + response.json = json_func_page_2 + else: + response.json = json_func_page_1 + + return response + + mocker.patch('narps_open.utils.status.get', side_effect = mocked_requests_get) issues = get_opened_issues() assert len(issues) == 1 assert issues[0]['html_url'] == 'urls' From fda1eb3912fdc021ad8c2eb535311e40299d4800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= <117362283+bclenet@users.noreply.github.com> Date: Fri, 29 Sep 2023 10:21:20 +0200 Subject: [PATCH 4/8] Fixing the `narps_open.utils.status` module (2) (#113) * [BUG] inside unit_tests workflow * Browsing all issues pages from Github API * Get all pages of GitHub issues * [TEST] Updating test for status module * [TEST] fetch several issues * Dealing with single page of issues --- narps_open/utils/status.py | 14 ++- tests/utils/test_status.py | 233 +++++++++++++++++++++++-------------- 2 files changed, 156 insertions(+), 91 deletions(-) diff --git a/narps_open/utils/status.py b/narps_open/utils/status.py index fef7708d..96d4265f 100644 --- a/narps_open/utils/status.py +++ b/narps_open/utils/status.py @@ -17,8 +17,16 @@ def get_opened_issues(): """ Return a list of opened issues and pull requests for the NARPS Open Pipelines project """ + + # First get the number of issues of the project + request_url = 'https://api.github.com/repos/Inria-Empenn/narps_open_pipelines' + response = get(request_url, timeout = 2) + response.raise_for_status() + nb_issues = response.json()['open_issues'] + + # Get all opened issues request_url = 'https://api.github.com/repos/Inria-Empenn/narps_open_pipelines/issues' - request_url += '?page={page_number}?per_page=100' + request_url += '?page={page_number}?per_page=30' issues = [] page = True # Will later be replaced by a table @@ -31,6 +39,10 @@ def get_opened_issues(): issues += page page_number += 1 + # Leave if there is only one page (in this case, the `page` query parameter has no effect) + if nb_issues < 30: + break + return issues def get_teams_with_pipeline_files(): diff --git a/tests/utils/test_status.py b/tests/utils/test_status.py index 6c16279b..42f9f584 100644 --- a/tests/utils/test_status.py +++ b/tests/utils/test_status.py @@ -25,6 +25,118 @@ PipelineStatusReport ) +mocked_issues_4 = [ + { + "html_url": "url_issue_2", + "number": 2, + "title" : "Issue for pipeline UK24", + "body" : "Nothing to add here." + }, + { + "html_url": "url_pull_3", + "number": 3, + "title" : "Pull request for pipeline 2T6S", + "pull_request" : {}, + "body" : "Work has been done." + }, + { + "html_url": "url_issue_4", + "number": 4, + "title" : None, + "body" : "This is a malformed issue about C88N." + }, + { + "html_url": "url_issue_5", + "number": 5, + "title" : "Issue about 2T6S", + "body" : "Something about 2T6S." + } +] + +mocked_issues_40_1 = [ + { + "html_url": "url_issue_55", + "number": i, + "title" : "Issue for pipeline UK24", + "body" : "Nothing to add here." + } for i in range(0,30) +] + +mocked_issues_40_2 = [ + { + "html_url": "url_issue_55", + "number": i, + "title" : "Issue for pipeline UK24", + "body" : "Nothing to add here." + } for i in range(0,10) +] + +mocked_repo_info_0 = { + "open_issues": 0 +} + +mocked_repo_info_4 = { + "open_issues": 4 +} + +mocked_repo_info_40 = { + "open_issues": 40 +} + +def mocked_requests_get_0(url, params=None, **kwargs): + """ Create a method to mock requests.get in case there are no issues """ + + response = Response() + response.status_code = 200 + + if 'issues' not in url: + def mocked_json(): + return mocked_repo_info_0 + else: + def mocked_json(): + return [] + + response.json = mocked_json + return response + +def mocked_requests_get_4(url, params=None, **kwargs): + """ Create a method to mock requests.get in case there are less than 30 issues """ + + response = Response() + response.status_code = 200 + + if 'issues' not in url: + def mocked_json(): + return mocked_repo_info_4 + else: + def mocked_json(): + return mocked_issues_4 + + response.json = mocked_json + return response + +def mocked_requests_get_40(url, params=None, **kwargs): + """ Create a method to mock requests.get in case there are more than 30 issues """ + + response = Response() + response.status_code = 200 + + if 'issues' not in url: + def mocked_json(): + return mocked_repo_info_40 + elif '?page=1' in url: + def mocked_json(): + return mocked_issues_40_1 + elif '?page=2' in url: + def mocked_json(): + return mocked_issues_40_2 + else: + def mocked_json(): + return [] + + response.json = mocked_json + return response + @fixture def mock_api_issue(mocker): """ Create a mock GitHub API response for successful query on open issues @@ -35,61 +147,6 @@ def mock_api_issue(mocker): Hence, we patch the `narps_open.utils.status.get` method. """ - # Create a method to mock requests.get - def mocked_requests_get(url, params=None, **kwargs): - - response = Response() - response.status_code = 200 - def json_func_page_0(): - return [ - { - "html_url": "url_issue_2", - "number": 2, - "title" : "Issue for pipeline UK24", - "body" : "Nothing to add here." - }, - { - "html_url": "url_pull_3", - "number": 3, - "title" : "Pull request for pipeline 2T6S", - "pull_request" : {}, - "body" : "Work has been done." - } - ] - - json_func_page_1 = json_func_page_0 - - def json_func_page_2(): - return [ - { - "html_url": "url_issue_4", - "number": 4, - "title" : None, - "body" : "This is a malformed issue about C88N." - }, - { - "html_url": "url_issue_5", - "number": 5, - "title" : "Issue about 2T6S", - "body" : "Something about 2T6S." - } - ] - - def json_func_page_3(): - return [] - - if '?page=1' in url: - response.json = json_func_page_1 - elif '?page=2' in url: - response.json = json_func_page_2 - elif '?page=3' in url: - response.json = json_func_page_3 - else: - response.json = json_func_page_0 - - return response - - mocker.patch('narps_open.utils.status.get', side_effect = mocked_requests_get) mocker.patch( 'narps_open.utils.status.get_teams_with_pipeline_files', return_value = ['2T6S', 'UK24', 'Q6O0'] @@ -118,7 +175,7 @@ def test_get_issues(mocker): which is actually imported as `get` inside the `narps_open.utils.status` module. Hence, we patch the `narps_open.utils.status.get` method. """ - # Create a mock API response for 404 error + # Behavior for 404 error response = Response() response.status_code = 404 @@ -126,41 +183,28 @@ def test_get_issues(mocker): with raises(HTTPError): get_opened_issues() - # Create a mock API response for the no issues case - response = Response() - response.status_code = 200 - def json_func(): - return [] - response.json = json_func - mocker.patch('narps_open.utils.status.get', return_value = response) + # No issues case + mocker.patch('narps_open.utils.status.get', side_effect = mocked_requests_get_0) assert len(get_opened_issues()) == 0 - # Create a mock API response for the general usecase - def mocked_requests_get(url, params=None, **kwargs): - response = Response() - response.status_code = 200 - def json_func_page_1(): - return [ - { - "html_url": "urls", - "number": 2, - } - ] - def json_func_page_2(): - return [] - - if '?page=2' in url: - response.json = json_func_page_2 - else: - response.json = json_func_page_1 - - return response - - mocker.patch('narps_open.utils.status.get', side_effect = mocked_requests_get) + # General usecase 4 issues + mocker.patch('narps_open.utils.status.get', side_effect = mocked_requests_get_4) + issues = get_opened_issues() - assert len(issues) == 1 - assert issues[0]['html_url'] == 'urls' + assert len(issues) == 4 + assert issues[0]['html_url'] == 'url_issue_2' assert issues[0]['number'] == 2 + assert issues[0]['title'] == 'Issue for pipeline UK24' + assert issues[0]['body'] == 'Nothing to add here.' + + # General usecase 40 issues + mocker.patch('narps_open.utils.status.get', side_effect = mocked_requests_get_40) + issues = get_opened_issues() + assert len(issues) == 40 + assert issues[0]['html_url'] == 'url_issue_55' + assert issues[0]['number'] == 0 + assert issues[0]['title'] == 'Issue for pipeline UK24' + assert issues[0]['body'] == 'Nothing to add here.' @staticmethod @mark.unit_test @@ -175,9 +219,12 @@ def test_get_teams(): @staticmethod @mark.unit_test - def test_generate(mock_api_issue): + def test_generate(mock_api_issue, mocker): """ Test generating a PipelineStatusReport """ + # Mock requests.get + mocker.patch('narps_open.utils.status.get', side_effect = mocked_requests_get_4) + # Test the generation report = PipelineStatusReport() report.generate() @@ -223,9 +270,12 @@ def test_generate(mock_api_issue): @staticmethod @mark.unit_test - def test_markdown(mock_api_issue): + def test_markdown(mock_api_issue, mocker): """ Test writing a PipelineStatusReport as Markdown """ + # Mock requests.get + mocker.patch('narps_open.utils.status.get', side_effect = mocked_requests_get_4) + # Generate markdown from report report = PipelineStatusReport() report.generate() @@ -241,9 +291,12 @@ def test_markdown(mock_api_issue): @staticmethod @mark.unit_test - def test_str(mock_api_issue): + def test_str(mock_api_issue, mocker): """ Test writing a PipelineStatusReport as JSON """ + # Mock requests.get + mocker.patch('narps_open.utils.status.get', side_effect = mocked_requests_get_4) + # Generate report report = PipelineStatusReport() report.generate() From f9feceaad126f6c30a4e775cb6faa8cc40e5b3f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= <117362283+bclenet@users.noreply.github.com> Date: Fri, 29 Sep 2023 10:37:24 +0200 Subject: [PATCH 5/8] Markdown export of team descriptions (#112) * [BUG] inside unit_tests workflow * Description markdown export option * [TEST] testing json and markdown export for team description * [DOC] update of team description --- docs/description.md | 45 +++++++- narps_open/data/description/__init__.py | 33 ++++++ narps_open/data/description/__main__.py | 36 ++++--- tests/data/test_description.py | 36 +++++++ tests/data/test_results.py | 3 +- .../data/description/test_markdown.md | 98 ++++++++++++++++++ .../test_data/data/description/test_str.json | 57 ++++++++++ .../results/team_2T6S/hypo1_unthresh.nii.gz | Bin .../results/team_2T6S/hypo2_unthresh.nii.gz | Bin .../results/team_2T6S/hypo3_unthresh.nii.gz | Bin .../results/team_2T6S/hypo4_unthresh.nii.gz | Bin .../results/team_2T6S/hypo5_unthresh.nii.gz | Bin .../results/team_2T6S/hypo6_unthresh.nii.gz | Bin .../results/team_2T6S/hypo7_unthresh.nii.gz | Bin .../results/team_2T6S/hypo8_unthresh.nii.gz | Bin .../results/team_2T6S/hypo9_unthresh.nii.gz | Bin 16 files changed, 292 insertions(+), 16 deletions(-) create mode 100644 tests/test_data/data/description/test_markdown.md create mode 100644 tests/test_data/data/description/test_str.json rename tests/test_data/{ => data}/results/team_2T6S/hypo1_unthresh.nii.gz (100%) rename tests/test_data/{ => data}/results/team_2T6S/hypo2_unthresh.nii.gz (100%) rename tests/test_data/{ => data}/results/team_2T6S/hypo3_unthresh.nii.gz (100%) rename tests/test_data/{ => data}/results/team_2T6S/hypo4_unthresh.nii.gz (100%) rename tests/test_data/{ => data}/results/team_2T6S/hypo5_unthresh.nii.gz (100%) rename tests/test_data/{ => data}/results/team_2T6S/hypo6_unthresh.nii.gz (100%) rename tests/test_data/{ => data}/results/team_2T6S/hypo7_unthresh.nii.gz (100%) rename tests/test_data/{ => data}/results/team_2T6S/hypo8_unthresh.nii.gz (100%) rename tests/test_data/{ => data}/results/team_2T6S/hypo9_unthresh.nii.gz (100%) diff --git a/docs/description.md b/docs/description.md index 0a769229..f0475117 100644 --- a/docs/description.md +++ b/docs/description.md @@ -8,7 +8,7 @@ The file `narps_open/data/description/analysis_pipelines_derived_descriptions.ts The class `TeamDescription` of module `narps_open.data.description` acts as a parser for these two files. -You can also use the command-line tool as so. Option `-t` is for the team id, option `-d` allows to print only one of the sub parts of the description among : `general`, `exclusions`, `preprocessing`, `analysis`, and `categorized_for_analysis`. +You can use the command-line tool as so. Option `-t` is for the team id, option `-d` allows to print only one of the sub parts of the description among : `general`, `exclusions`, `preprocessing`, `analysis`, and `categorized_for_analysis`. Options `--json` and `--md` allow to choose the export format you prefer between JSON and Markdown. ```bash python narps_open/data/description -h @@ -21,8 +21,25 @@ python narps_open/data/description -h # -t TEAM, --team TEAM the team ID # -d {general,exclusions,preprocessing,analysis,categorized_for_analysis,derived}, --dictionary {general,exclusions,preprocessing,analysis,categorized_for_analysis,derived} # the sub dictionary of team description +# --json output team description as JSON +# --md output team description as Markdown -python narps_open/data/description -t 2T6S -d general +python narps_open/data/description -t 2T6S --json +# { +# "general.teamID": "2T6S", +# "general.NV_collection_link": "https://neurovault.org/collections/4881/", +# "general.results_comments": "NA", +# "general.preregistered": "No", +# "general.link_preregistration_form": "We did not pre-register our analysis.", +# "general.regions_definition": "We employed the pre-hypothesized brain regions (vmPFC, vSTR, and amygdala) from Barta, McGuire, and Kable (2010, Neuroimage). Specific MNI coordinates are:\nvmPFC: x = 2, y = 46, z = -8\nleft vSTR: x = -12, y = 12, z = -6, right vSTR = x = 12, y = 10, z = -6\n(right) Amygdala: x = 24, y = -4, z = -18", +# "general.softwares": "SPM12 , \nfmriprep 1.1.4", +# "exclusions.n_participants": "108", +# "exclusions.exclusions_details": "We did not exclude any participant in the analysis", +# "preprocessing.used_fmriprep_data": "Yes", +# "preprocessing.preprocessing_order": "We used the provided preprocessed data by fMRIPprep 1.1.4 (Esteban, Markiewicz, et al. (2018); Esteban, Blair, et al. (2018); RRID:SCR_016216), which is based on Nipype 1.1.1 (Gorgolewski et al. (2011); Gorgolewski et al. (2018); RRID:SCR_002502) and we additionally conducted a spatial smoothing using the provided preprocessed data set and SPM12. Here, we attach the preprocessing steps described in the provided data set. \nAnatomical data preprocessing\nThe T1-weighted (T1w) image was corrected for intensity non-uniformity (INU) using N4BiasFieldCorrection (Tustison et al. 2010, ANTs 2.2.0), and used as T1w-reference throughout the workflow. The T1w-reference was then skull-stripped using antsBrainExtraction.sh (ANTs 2.2.0), using OASIS as target template. Brain surfaces we +# ... + +python narps_open/data/description -t 2T6S -d general --json # { # "teamID": "2T6S", # "NV_collection_link": "https://neurovault.org/collections/4881/", @@ -33,6 +50,30 @@ python narps_open/data/description -t 2T6S -d general # "softwares": "SPM12 , \nfmriprep 1.1.4", # "general_comments": "NA" # } + +python narps_open/data/description -t 2T6S --md +# # NARPS team description : 2T6S +# ## General +# * `teamID` : 2T6S +# * `NV_collection_link` : https://neurovault.org/collections/4881/ +# * `results_comments` : NA +# * `preregistered` : No +# * `link_preregistration_form` : We did not pre-register our analysis. +# * `regions_definition` : We employed the pre-hypothesized brain regions (vmPFC, vSTR, and amygdala) from Barta, McGuire, and Kable (2010, Neuroimage). Specific MNI coordinates are: +# vmPFC: x = 2, y = 46, z = -8 +# left vSTR: x = -12, y = 12, z = -6, right vSTR = x = 12, y = 10, z = -6 +# (right) Amygdala: x = 24, y = -4, z = -18 +# * `softwares` : SPM12 , +# fmriprep 1.1.4 +# * `general_comments` : NA +# ## Exclusions +# * `n_participants` : 108 +# * `exclusions_details` : We did not exclude any participant in the analysis +# ## Preprocessing +# * `used_fmriprep_data` : Yes +# * `preprocessing_order` : We used the provided preprocessed data by fMRIPprep 1.1.4 (Esteban, Markiewicz, et al. (2018); Esteban, Blair, et al. (2018); RRID:SCR_016216), which is based on Nipype 1.1.1 (Gorgolewski et al. (2011); Gorgolewski et al. (2018); RRID:SCR_002502) and we additionally conducted a spatial smoothing using the provided preprocessed data set and SPM12. Here, we attach the preprocessing steps described in the provided data set. +# Anatomical data preprocessing +# ... ``` Of course the `narps_open.data.description` module is accessible programmatically, here is an example on how to use it: diff --git a/narps_open/data/description/__init__.py b/narps_open/data/description/__init__.py index 0637f6c7..908ebfe7 100644 --- a/narps_open/data/description/__init__.py +++ b/narps_open/data/description/__init__.py @@ -5,6 +5,7 @@ from os.path import join from csv import DictReader +from json import dumps from importlib_resources import files class TeamDescription(dict): @@ -25,6 +26,9 @@ def __init__(self, team_id): self.team_id = team_id self._load() + def __str__(self): + return dumps(self, indent = 4) + @property def general(self) -> dict: """ Getter for the sub dictionary general """ @@ -55,6 +59,35 @@ def derived(self) -> dict: """ Getter for the sub dictionary containing derived team description """ return self._get_sub_dict('derived') + def markdown(self): + """ Return the team description as a string formatted in markdown """ + return_string = f'# NARPS team description : {self.team_id}\n' + + dictionaries = [ + self.general, + self.exclusions, + self.preprocessing, + self.analysis, + self.categorized_for_analysis, + self.derived + ] + + names = [ + 'General', + 'Exclusions', + 'Preprocessing', + 'Analysis', + 'Categorized for analysis', + 'Derived' + ] + + for dictionary, name in zip(dictionaries, names): + return_string += f'## {name}\n' + for key in dictionary: + return_string += f'* `{key}` : {dictionary[key]}\n' + + return return_string + def _get_sub_dict(self, key_first_part:str) -> dict: """ Return a sub-dictionary of self, with keys that contain key_first_part. The first part of the keys are removed, e.g.: diff --git a/narps_open/data/description/__main__.py b/narps_open/data/description/__main__.py index 226249c5..49daad22 100644 --- a/narps_open/data/description/__main__.py +++ b/narps_open/data/description/__main__.py @@ -22,22 +22,32 @@ 'derived' ], help='the sub dictionary of team description') +formats = parser.add_mutually_exclusive_group(required = False) +formats.add_argument('--json', action='store_true', help='output team description as JSON') +formats.add_argument('--md', action='store_true', help='output team description as Markdown') arguments = parser.parse_args() # Initialize a TeamDescription information = TeamDescription(team_id = arguments.team) -if arguments.dictionary == 'general': - print(dumps(information.general, indent = 4)) -elif arguments.dictionary == 'exclusions': - print(dumps(information.exclusions, indent = 4)) -elif arguments.dictionary == 'preprocessing': - print(dumps(information.preprocessing, indent = 4)) -elif arguments.dictionary == 'analysis': - print(dumps(information.analysis, indent = 4)) -elif arguments.dictionary == 'categorized_for_analysis': - print(dumps(information.categorized_for_analysis, indent = 4)) -elif arguments.dictionary == 'derived': - print(dumps(information.derived, indent = 4)) +# Output description +if arguments.md and arguments.dictionary is not None: + print('Sub dictionaries cannot be exported as Markdown yet.') + print('Print the whole description instead.') +elif arguments.md: + print(information.markdown()) else: - print(dumps(information, indent = 4)) + if arguments.dictionary == 'general': + print(dumps(information.general, indent = 4)) + elif arguments.dictionary == 'exclusions': + print(dumps(information.exclusions, indent = 4)) + elif arguments.dictionary == 'preprocessing': + print(dumps(information.preprocessing, indent = 4)) + elif arguments.dictionary == 'analysis': + print(dumps(information.analysis, indent = 4)) + elif arguments.dictionary == 'categorized_for_analysis': + print(dumps(information.categorized_for_analysis, indent = 4)) + elif arguments.dictionary == 'derived': + print(dumps(information.derived, indent = 4)) + else: + print(dumps(information, indent = 4)) diff --git a/tests/data/test_description.py b/tests/data/test_description.py index 3bdc7c2c..03f95d4b 100644 --- a/tests/data/test_description.py +++ b/tests/data/test_description.py @@ -11,8 +11,11 @@ pytest -q test_description.py -k """ +from os.path import join + from pytest import raises, mark +from narps_open.utils.configuration import Configuration from narps_open.data.description import TeamDescription class TestUtilsDescription: @@ -86,3 +89,36 @@ def test_arguments_properties(): assert description['general.softwares'] == 'FSL 5.0.11, MRIQC, FMRIPREP' assert isinstance(description.general, dict) assert description.general['softwares'] == 'FSL 5.0.11, MRIQC, FMRIPREP' + + @staticmethod + @mark.unit_test + def test_markdown(): + """ Test writing a TeamDescription as Markdown """ + + # Generate markdown from description + description = TeamDescription('9Q6R') + markdown = description.markdown() + + # Compare markdown with test file + test_file_path = join( + Configuration()['directories']['test_data'], + 'data', 'description', 'test_markdown.md' + ) + with open(test_file_path, 'r', encoding = 'utf-8') as file: + assert markdown == file.read() + + @staticmethod + @mark.unit_test + def test_str(): + """ Test writing a TeamDescription as JSON """ + + # Generate report + description = TeamDescription('9Q6R') + + # Compare string version of the description with test file + test_file_path = join( + Configuration()['directories']['test_data'], + 'data', 'description', 'test_str.json' + ) + with open(test_file_path, 'r', encoding = 'utf-8') as file: + assert str(description) == file.read() diff --git a/tests/data/test_results.py b/tests/data/test_results.py index 46d2d1f5..2465eb7d 100644 --- a/tests/data/test_results.py +++ b/tests/data/test_results.py @@ -103,7 +103,8 @@ def test_rectify(): """ Test the rectify method """ # Get raw data - orig_directory = join(Configuration()['directories']['test_data'], 'results', 'team_2T6S') + orig_directory = join( + Configuration()['directories']['test_data'], 'data', 'results', 'team_2T6S') # Create test data test_directory = join(Configuration()['directories']['test_runs'], 'results_team_2T6S') diff --git a/tests/test_data/data/description/test_markdown.md b/tests/test_data/data/description/test_markdown.md new file mode 100644 index 00000000..080d397c --- /dev/null +++ b/tests/test_data/data/description/test_markdown.md @@ -0,0 +1,98 @@ +# NARPS team description : 9Q6R +## General +* `teamID` : 9Q6R +* `NV_collection_link` : https://neurovault.org/collections/4765/ +* `results_comments` : Note: Amygdala wasn't recruited for hypothesis tests 7-9, but the extended salience network was recruited in all contrasts (e.g. aINS, ACC). Based on looking at the unthresholded maps, hypotheses 8 and 9 would've been confirmed at lower cluster thresholds (i.e. z≥2.3 rather than z≥3.1). +* `preregistered` : No +* `link_preregistration_form` : NA +* `regions_definition` : Harvard-Oxford probabilistic cortical and subcortical atlases (Frontal Median Cortex, L+R Amyg, and L+R Accum for vmPFC, amyg, and VS, respectively). Also used Neurosynth to generate a mask based on the search term "ventral striatum" (height threshold at z>12, and cluster-extent at > 400mm^3) +* `softwares` : FSL 5.0.11, MRIQC, FMRIPREP +* `general_comments` : NA +## Exclusions +* `n_participants` : 104 +* `exclusions_details` : N=104 (54 eq_indiff, 50 eq_range). Excluded sub-018, sub-030, sub-088, and sub-100. High motion during function runs: All four participants had at least one run where > 50% of the TRs contained FD > 0.2mm. 18, 30, and 100 in particular were constant movers (all 4 runs > 50% TRS > 0.2 mm FD) +## Preprocessing +* `used_fmriprep_data` : No +* `preprocessing_order` : - MRIQC and FMRIPREP run on a local HPC +- FSL used for mass univariate analyses, avoiding re-registration using this approach: https://www.youtube.com/watch?time_continue=7&v=U3tG7JMEf7M +* `brain_extraction` : Freesurfer (i.e. part of fmriprep default pipeline) +* `segmentation` : Freesurfer +* `slice_time_correction` : Not performed +* `motion_correction` : Framewise displacement, and six standard motion regressors (x, y, z, rotx, rotx, and rotz) within subjects.; generated via MRIQC +* `motion` : 6 +* `gradient_distortion_correction` : NA +* `intra_subject_coreg` : bbregister, flirt, default FMRIPREP +* `distortion_correction` : Fieldmap-less distortion correction within fmriprep pipeline (--use-syn-sdc) +* `inter_subject_reg` : ANTs, multiscale nonlinear mutual-information default within FMRIPREP pipeline. +* `intensity_correction` : Default fMRIPREP INU correction +* `intensity_normalization` : Default fMRIPREP INU normalization +* `noise_removal` : None +* `volume_censoring` : None +* `spatial_smoothing` : 5mm FWHM +* `preprocessing_comments` : NA +## Analysis +* `data_submitted_to_model` : 453 total volumes, 104 participants (54 eq_indiff, 50 eq_range) +* `spatial_region_modeled` : Whole-Brain +* `independent_vars_first_level` : Event-related design predictors: +- Modeled duration = 4 +- EVs (3): Mean-centered Gain, Mean-Centered Loss, Events (constant) +Block design: +- baseline not explicitly modeled +HRF: +- FMRIB's Linear Optimal Basis Sets +Movement regressors: +- FD, six parameters (x, y, z, RotX, RotY, RotZ) +* `RT_modeling` : none +* `movement_modeling` : 1 +* `independent_vars_higher_level` : EVs (2): eq_indiff, eq_range +Contrasts in the group-level design matrix: +1 --> mean (1, 1) +2 --> eq_indiff (1, 0) +3 --> eq_range (0, 1) +4 --> indiff_gr_range (1, -1) +5 --> range_gr_indiff (-1, 1) +* `model_type` : Mass Univariate +* `model_settings` : First model: individual runs; +Second model: higher-level analysis on lower-level FEAT directories in a fixed effects model at the participant-level; +Third model: higher-level analysis on 3D COPE images from *.feat directories within second model *.gfeat; FLAME 1 (FMRIB's Local Analysis of Mixed Effects), with a cluster threshold of z≥3.1 +* `inference_contrast_effect` : First-Level A (Run-level; not listed: linear basis functions, FSL FLOBs): +Model EVs (3): gain, loss, event +- COPE1: Pos Gain (1, 0, 0) +- COPE4: Neg Gain (-1, 0, 0) +- COPE7: Pos Loss (0, 1, 0) +- COPE10: Neg Loss (0, -1, 0) +- COPE13: Events (0, 0, 1) +Confound EVs (7): Framewise Displacement, x, y, z, RotX, RotY, RotZ. Generated in MRIQC. + +First-Level B (Participant-level): +- All COPEs from the runs modeled in a high-level FEAT fixed effect model + +Second-Level (Group-level): +- Separate high-level FLAME 1 models run on COPE1, COPE4, COPE7, and COPE10. Hypotheses 1-4 answered using the COPE1 model, Hypotheses 5-6 answered using the COPE10 model, and Hypotheses 7-9 answered using the COPE7 model. +Model EVs (2): eq_indiff, eq_range +- mean (1, 1) +- eq_indiff (1, 0) +- eq_range (0, 1) +- indiff_gr_range (1, -1) +- range_gr_indiff (-1, 1) +* `search_region` : Whole brain +* `statistic_type` : Cluster size +* `pval_computation` : Standard parametric inference +* `multiple_testing_correction` : GRF_theory based FEW correction at z≥3.1 in FSL +* `comments_analysis` : NA +## Categorized for analysis +* `region_definition_vmpfc` : atlas HOA +* `region_definition_striatum` : atlas HOA, neurosynth +* `region_definition_amygdala` : atlas HOA +* `analysis_SW` : FSL +* `analysis_SW_with_version` : FSL 5.0.11 +* `smoothing_coef` : 5 +* `testing` : parametric +* `testing_thresh` : p<0.001 +* `correction_method` : GRTFWE cluster +* `correction_thresh_` : p<0.05 +## Derived +* `n_participants` : 104 +* `excluded_participants` : 018, 030, 088, 100 +* `func_fwhm` : 5 +* `con_fwhm` : diff --git a/tests/test_data/data/description/test_str.json b/tests/test_data/data/description/test_str.json new file mode 100644 index 00000000..0d27767e --- /dev/null +++ b/tests/test_data/data/description/test_str.json @@ -0,0 +1,57 @@ +{ + "general.teamID": "9Q6R", + "general.NV_collection_link": "https://neurovault.org/collections/4765/", + "general.results_comments": "Note: Amygdala wasn't recruited for hypothesis tests 7-9, but the extended salience network was recruited in all contrasts (e.g. aINS, ACC). Based on looking at the unthresholded maps, hypotheses 8 and 9 would've been confirmed at lower cluster thresholds (i.e. z\u22652.3 rather than z\u22653.1).", + "general.preregistered": "No", + "general.link_preregistration_form": "NA", + "general.regions_definition": "Harvard-Oxford probabilistic cortical and subcortical atlases (Frontal Median Cortex, L+R Amyg, and L+R Accum for vmPFC, amyg, and VS, respectively). Also used Neurosynth to generate a mask based on the search term \"ventral striatum\" (height threshold at z>12, and cluster-extent at > 400mm^3)", + "general.softwares": "FSL 5.0.11, MRIQC, FMRIPREP", + "exclusions.n_participants": "104", + "exclusions.exclusions_details": "N=104 (54 eq_indiff, 50 eq_range). Excluded sub-018, sub-030, sub-088, and sub-100. High motion during function runs: All four participants had at least one run where > 50% of the TRs contained FD > 0.2mm. 18, 30, and 100 in particular were constant movers (all 4 runs > 50% TRS > 0.2 mm FD) ", + "preprocessing.used_fmriprep_data": "No", + "preprocessing.preprocessing_order": " - MRIQC and FMRIPREP run on a local HPC\n- FSL used for mass univariate analyses, avoiding re-registration using this approach: https://www.youtube.com/watch?time_continue=7&v=U3tG7JMEf7M", + "preprocessing.brain_extraction": "Freesurfer (i.e. part of fmriprep default pipeline)", + "preprocessing.segmentation": "Freesurfer", + "preprocessing.slice_time_correction": "Not performed", + "preprocessing.motion_correction": "Framewise displacement, and six standard motion regressors (x, y, z, rotx, rotx, and rotz) within subjects.; generated via MRIQC ", + "preprocessing.motion": "6", + "preprocessing.gradient_distortion_correction": "NA", + "preprocessing.intra_subject_coreg": "bbregister, flirt, default FMRIPREP", + "preprocessing.distortion_correction": "Fieldmap-less distortion correction within fmriprep pipeline (--use-syn-sdc)", + "preprocessing.inter_subject_reg": "ANTs, multiscale nonlinear mutual-information default within FMRIPREP pipeline.", + "preprocessing.intensity_correction": "Default fMRIPREP INU correction", + "preprocessing.intensity_normalization": "Default fMRIPREP INU normalization", + "preprocessing.noise_removal": "None", + "preprocessing.volume_censoring": "None", + "preprocessing.spatial_smoothing": "5mm FWHM", + "preprocessing.preprocessing_comments": "NA", + "analysis.data_submitted_to_model": "453 total volumes, 104 participants (54 eq_indiff, 50 eq_range)", + "analysis.spatial_region_modeled": "Whole-Brain", + "analysis.independent_vars_first_level": "Event-related design predictors:\n- Modeled duration = 4\n- EVs (3): Mean-centered Gain, Mean-Centered Loss, Events (constant)\nBlock design:\n- baseline not explicitly modeled\nHRF:\n- FMRIB's Linear Optimal Basis Sets\nMovement regressors:\n- FD, six parameters (x, y, z, RotX, RotY, RotZ)", + "analysis.RT_modeling": "none", + "analysis.movement_modeling": "1", + "analysis.independent_vars_higher_level": "EVs (2): eq_indiff, eq_range\nContrasts in the group-level design matrix:\n1 --> mean (1, 1)\n2 --> eq_indiff (1, 0)\n3 --> eq_range (0, 1)\n4 --> indiff_gr_range (1, -1)\n5 --> range_gr_indiff (-1, 1)", + "analysis.model_type": "Mass Univariate", + "analysis.model_settings": "First model: individual runs; \nSecond model: higher-level analysis on lower-level FEAT directories in a fixed effects model at the participant-level; \nThird model: higher-level analysis on 3D COPE images from *.feat directories within second model *.gfeat; FLAME 1 (FMRIB's Local Analysis of Mixed Effects), with a cluster threshold of z\u22653.1", + "analysis.inference_contrast_effect": "First-Level A (Run-level; not listed: linear basis functions, FSL FLOBs):\nModel EVs (3): gain, loss, event\n- COPE1: Pos Gain (1, 0, 0)\n- COPE4: Neg Gain (-1, 0, 0)\n- COPE7: Pos Loss (0, 1, 0)\n- COPE10: Neg Loss (0, -1, 0)\n- COPE13: Events (0, 0, 1)\nConfound EVs (7): Framewise Displacement, x, y, z, RotX, RotY, RotZ. Generated in MRIQC.\n\nFirst-Level B (Participant-level):\n- All COPEs from the runs modeled in a high-level FEAT fixed effect model\n\nSecond-Level (Group-level):\n- Separate high-level FLAME 1 models run on COPE1, COPE4, COPE7, and COPE10. Hypotheses 1-4 answered using the COPE1 model, Hypotheses 5-6 answered using the COPE10 model, and Hypotheses 7-9 answered using the COPE7 model.\nModel EVs (2): eq_indiff, eq_range\n- mean (1, 1)\n- eq_indiff (1, 0)\n- eq_range (0, 1)\n- indiff_gr_range (1, -1)\n- range_gr_indiff (-1, 1) ", + "analysis.search_region": "Whole brain", + "analysis.statistic_type": "Cluster size", + "analysis.pval_computation": "Standard parametric inference", + "analysis.multiple_testing_correction": "GRF_theory based FEW correction at z\u22653.1 in FSL", + "analysis.comments_analysis": "NA", + "general.general_comments": "NA", + "categorized_for_analysis.region_definition_vmpfc": "atlas HOA", + "categorized_for_analysis.region_definition_striatum": "atlas HOA, neurosynth", + "categorized_for_analysis.region_definition_amygdala": "atlas HOA", + "categorized_for_analysis.analysis_SW": "FSL", + "categorized_for_analysis.analysis_SW_with_version": "FSL 5.0.11", + "categorized_for_analysis.smoothing_coef": "5", + "categorized_for_analysis.testing": "parametric", + "categorized_for_analysis.testing_thresh": "p<0.001", + "categorized_for_analysis.correction_method": "GRTFWE cluster", + "categorized_for_analysis.correction_thresh_": "p<0.05", + "derived.n_participants": "104", + "derived.excluded_participants": "018, 030, 088, 100", + "derived.func_fwhm": "5", + "derived.con_fwhm": "" +} \ No newline at end of file diff --git a/tests/test_data/results/team_2T6S/hypo1_unthresh.nii.gz b/tests/test_data/data/results/team_2T6S/hypo1_unthresh.nii.gz similarity index 100% rename from tests/test_data/results/team_2T6S/hypo1_unthresh.nii.gz rename to tests/test_data/data/results/team_2T6S/hypo1_unthresh.nii.gz diff --git a/tests/test_data/results/team_2T6S/hypo2_unthresh.nii.gz b/tests/test_data/data/results/team_2T6S/hypo2_unthresh.nii.gz similarity index 100% rename from tests/test_data/results/team_2T6S/hypo2_unthresh.nii.gz rename to tests/test_data/data/results/team_2T6S/hypo2_unthresh.nii.gz diff --git a/tests/test_data/results/team_2T6S/hypo3_unthresh.nii.gz b/tests/test_data/data/results/team_2T6S/hypo3_unthresh.nii.gz similarity index 100% rename from tests/test_data/results/team_2T6S/hypo3_unthresh.nii.gz rename to tests/test_data/data/results/team_2T6S/hypo3_unthresh.nii.gz diff --git a/tests/test_data/results/team_2T6S/hypo4_unthresh.nii.gz b/tests/test_data/data/results/team_2T6S/hypo4_unthresh.nii.gz similarity index 100% rename from tests/test_data/results/team_2T6S/hypo4_unthresh.nii.gz rename to tests/test_data/data/results/team_2T6S/hypo4_unthresh.nii.gz diff --git a/tests/test_data/results/team_2T6S/hypo5_unthresh.nii.gz b/tests/test_data/data/results/team_2T6S/hypo5_unthresh.nii.gz similarity index 100% rename from tests/test_data/results/team_2T6S/hypo5_unthresh.nii.gz rename to tests/test_data/data/results/team_2T6S/hypo5_unthresh.nii.gz diff --git a/tests/test_data/results/team_2T6S/hypo6_unthresh.nii.gz b/tests/test_data/data/results/team_2T6S/hypo6_unthresh.nii.gz similarity index 100% rename from tests/test_data/results/team_2T6S/hypo6_unthresh.nii.gz rename to tests/test_data/data/results/team_2T6S/hypo6_unthresh.nii.gz diff --git a/tests/test_data/results/team_2T6S/hypo7_unthresh.nii.gz b/tests/test_data/data/results/team_2T6S/hypo7_unthresh.nii.gz similarity index 100% rename from tests/test_data/results/team_2T6S/hypo7_unthresh.nii.gz rename to tests/test_data/data/results/team_2T6S/hypo7_unthresh.nii.gz diff --git a/tests/test_data/results/team_2T6S/hypo8_unthresh.nii.gz b/tests/test_data/data/results/team_2T6S/hypo8_unthresh.nii.gz similarity index 100% rename from tests/test_data/results/team_2T6S/hypo8_unthresh.nii.gz rename to tests/test_data/data/results/team_2T6S/hypo8_unthresh.nii.gz diff --git a/tests/test_data/results/team_2T6S/hypo9_unthresh.nii.gz b/tests/test_data/data/results/team_2T6S/hypo9_unthresh.nii.gz similarity index 100% rename from tests/test_data/results/team_2T6S/hypo9_unthresh.nii.gz rename to tests/test_data/data/results/team_2T6S/hypo9_unthresh.nii.gz From 2836b1026290a041438e812a960b33144f90760a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= <117362283+bclenet@users.noreply.github.com> Date: Fri, 29 Sep 2023 11:45:13 +0200 Subject: [PATCH 6/8] Fixing the `narps_open.utils.status` module (3) (#114) * [BUG] inside unit_tests workflow * Browsing all issues pages from Github API * Get all pages of GitHub issues * [TEST] Updating test for status module * [TEST] fetch several issues * Dealing with single page of issues * Removine per_page query parameter * [TEST] adjusting tests --- narps_open/utils/status.py | 6 +----- tests/utils/test_status.py | 6 ++++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/narps_open/utils/status.py b/narps_open/utils/status.py index 96d4265f..6dced5bc 100644 --- a/narps_open/utils/status.py +++ b/narps_open/utils/status.py @@ -26,7 +26,7 @@ def get_opened_issues(): # Get all opened issues request_url = 'https://api.github.com/repos/Inria-Empenn/narps_open_pipelines/issues' - request_url += '?page={page_number}?per_page=30' + request_url += '?page={page_number}' issues = [] page = True # Will later be replaced by a table @@ -39,10 +39,6 @@ def get_opened_issues(): issues += page page_number += 1 - # Leave if there is only one page (in this case, the `page` query parameter has no effect) - if nb_issues < 30: - break - return issues def get_teams_with_pipeline_files(): diff --git a/tests/utils/test_status.py b/tests/utils/test_status.py index 42f9f584..17170b71 100644 --- a/tests/utils/test_status.py +++ b/tests/utils/test_status.py @@ -108,9 +108,12 @@ def mocked_requests_get_4(url, params=None, **kwargs): if 'issues' not in url: def mocked_json(): return mocked_repo_info_4 - else: + elif '?page=1' in url: def mocked_json(): return mocked_issues_4 + else: + def mocked_json(): + return [] response.json = mocked_json return response @@ -189,7 +192,6 @@ def test_get_issues(mocker): # General usecase 4 issues mocker.patch('narps_open.utils.status.get', side_effect = mocked_requests_get_4) - issues = get_opened_issues() assert len(issues) == 4 assert issues[0]['html_url'] == 'url_issue_2' From a838c46a1fcb8fff1e93bd72c19c00ed0c4eb2d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= <117362283+bclenet@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:24:02 +0200 Subject: [PATCH 7/8] Task information (#117) * [BUG] inside unit_tests workflow * [ENH] A class to parse task data * [TEST] making tests pass * [DOC] update for narps_open.data --- docs/data.md | 36 +++++++++++++++ narps_open/data/task.py | 27 +++++++++++ tests/data/test_task.py | 57 ++++++++++++++++++++++++ tests/test_data/data/task/task-info.json | 12 +++++ 4 files changed, 132 insertions(+) create mode 100644 narps_open/data/task.py create mode 100644 tests/data/test_task.py create mode 100644 tests/test_data/data/task/task-info.json diff --git a/docs/data.md b/docs/data.md index 1e6b4fc3..e2e84da1 100644 --- a/docs/data.md +++ b/docs/data.md @@ -94,3 +94,39 @@ python narps_open/utils/results -r -t 2T6S C88N L1A8 The collections are also available [here](https://zenodo.org/record/3528329/) as one release on Zenodo that you can download. Each team results collection is kept in the `data/results/orig` directory, in a folder using the pattern `_` (e.g.: `4881_2T6S` for the 2T6S team). + +## Access NARPS data + +Inside `narps_open.data`, several modules allow to parse data from the NARPS file, so it's easier to use it inside the Narps Open Pipelines project. These are : + +### `narps_open.data.description` +Get textual description of the pipelines, as written by the teams (see [docs/description.md](/docs/description.md)). + +### `narps_open.data.results` +Get the result collections, as described earlier in this file. + +### `narps_open.data.participants` +Get the participants data (parses the `data/original/ds001734/participants.tsv` file) as well as participants subsets to perform analyses on lower numbers of images. + +### `narps_open.data.task` +Get information about the task (parses the `data/original/ds001734/task-MGT_bold.json` file). Here is an example how to use it : + +```python +from narps_open.data.task import TaskInformation + +task_info = TaskInformation() # task_info is a dict + +# All available keys +print(task_info.keys()) +# dict_keys(['TaskName', 'Manufacturer', 'ManufacturersModelName', 'MagneticFieldStrength', 'RepetitionTime', 'EchoTime', 'FlipAngle', 'MultibandAccelerationFactor', 'EffectiveEchoSpacing', 'SliceTiming', 'BandwidthPerPixelPhaseEncode', 'PhaseEncodingDirection', 'TaskDescription', 'CogAtlasID', 'NumberOfSlices', 'AcquisitionTime', 'TotalReadoutTime']) + +# Original data +print(task_info['TaskName']) +print(task_info['Manufacturer']) +print(task_info['RepetitionTime']) # And so on ... + +# Derived data +print(task_info['NumberOfSlices']) +print(task_info['AcquisitionTime']) +print(task_info['TotalReadoutTime']) +``` diff --git a/narps_open/data/task.py b/narps_open/data/task.py new file mode 100644 index 00000000..f3e86803 --- /dev/null +++ b/narps_open/data/task.py @@ -0,0 +1,27 @@ +#!/usr/bin/python +# coding: utf-8 + +""" A mdoule to parse task data from NARPS for the narps_open package """ + +from os.path import join +from json import load + +from narps_open.utils.configuration import Configuration +from narps_open.utils.singleton import SingletonMeta + +class TaskInformation(dict, metaclass=SingletonMeta): + """ This class allows to access information about the task performed in NARPS """ + + task_information_file = join(Configuration()['directories']['dataset'], 'task-MGT_bold.json') + + def __init__(self): + super().__init__() + + # Load information from the task-MGT_bold.json file + with open(self.task_information_file, 'rb') as file: + self.update(load(file)) + + # Compute derived information + self['NumberOfSlices'] = len(self['SliceTiming']) + self['AcquisitionTime'] = self['RepetitionTime'] / self['NumberOfSlices'] + self['TotalReadoutTime'] = self['NumberOfSlices'] * self['EffectiveEchoSpacing'] diff --git a/tests/data/test_task.py b/tests/data/test_task.py new file mode 100644 index 00000000..8b6860dd --- /dev/null +++ b/tests/data/test_task.py @@ -0,0 +1,57 @@ +#!/usr/bin/python +# coding: utf-8 + +""" Tests of the 'narps_open.data.task' module. + +Launch this test with PyTest + +Usage: +====== + pytest -q test_task.py + pytest -q test_task.py -k +""" +from os.path import join + +from pytest import mark, fixture + +from narps_open.utils.configuration import Configuration +import narps_open.data.task as task + +@fixture(scope='function', autouse=True) +def mock_task_data(mocker): + """ Patch the json.load method to mock task data """ + mocker.patch.object( + task.TaskInformation, 'task_information_file', + join(Configuration()['directories']['test_data'], 'data', 'task', 'task-info.json') + ) + +class TestTaskInformation: + """ A class that contains all the unit tests for the TaskInformation class.""" + + @staticmethod + @mark.unit_test + def test_accessing(): + """ Check that task information is reachable """ + + assert task.TaskInformation()['RepetitionTime'] == 1 + assert len(task.TaskInformation()['SliceTiming']) == 6 + + @staticmethod + @mark.unit_test + def test_singleton(): + """ Check that TaskInformation is a singleton. """ + + obj1 = task.TaskInformation() + obj2 = task.TaskInformation() + + assert id(obj1) == id(obj2) + + @staticmethod + @mark.unit_test + def test_derived(): + """ Test the derived values of a TaskInformation object """ + + task_info = task.TaskInformation() + assert task_info['NumberOfSlices'] == 6 + assert task_info['AcquisitionTime'] == 1 / 6 + assert task_info['TotalReadoutTime'] == 12 diff --git a/tests/test_data/data/task/task-info.json b/tests/test_data/data/task/task-info.json new file mode 100644 index 00000000..7927183d --- /dev/null +++ b/tests/test_data/data/task/task-info.json @@ -0,0 +1,12 @@ +{ + "RepetitionTime": 1, + "EffectiveEchoSpacing": 2, + "SliceTiming": [ + 0, + 0.4375, + 0.875, + 0.3125, + 0.75, + 0.1875 + ] +} \ No newline at end of file From 5b375cce9ebe89ea1ba19de2f9f59ed7fad6a5bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= <117362283+bclenet@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:25:11 +0200 Subject: [PATCH 8/8] Credits section + install documentation (#91) * [BUG] inside unit_tests workflow * Adding names of contributors * [DOC] simplifying the install doc * [DOC] simplifying the install doc * [DOC] simplifying the environment doc * [DOC] simplifying the environment doc * [DOC] typo correction --- INSTALL.md | 91 +++++++++++++++++----------------- README.md | 4 +- docs/environment.md | 116 +++++++++++++++++++------------------------- docs/running.md | 58 +++++++++++----------- 4 files changed, 127 insertions(+), 142 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index f1589893..b6142cc0 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,82 +1,85 @@ # How to install NARPS Open Pipelines ? -## 1 - Get the code +## 1 - Fork the repository -First, [fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo) the repository, so you have your own working copy of it. +[Fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo) the repository, so you have your own working copy of it. -Then, you have two options to [clone](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) the project : +## 2 - Clone the code -### Option 1: Using DataLad (recommended) +First, install [Datalad](https://www.datalad.org/). This will allow you to access the NARPS data easily, as it is included in the repository as [datalad subdatasets](http://handbook.datalad.org/en/latest/basics/101-106-nesting.html). -Cloning the fork using [Datalad](https://www.datalad.org/) will allow you to get the code as well as "links" to the data, because the NARPS data is bundled in this repository as [datalad subdatasets](http://handbook.datalad.org/en/latest/basics/101-106-nesting.html). +Then, [clone](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) the project : ```bash +# Replace YOUR_GITHUB_USERNAME in the following command. datalad install --recursive https://github.com/YOUR_GITHUB_USERNAME/narps_open_pipelines.git ``` -### Option 2: Using Git +> [!WARNING] +> It is still possible to clone the fork using [git](https://git-scm.com/) ; but by doing this, you will only get the code. +> ```bash +> # Replace YOUR_GITHUB_USERNAME in the following command. +> git clone https://github.com/YOUR_GITHUB_USERNAME/narps_open_pipelines.git +> ``` -Cloning the fork using [git](https://git-scm.com/) ; by doing this, you will only get the code. +## 3 - Get the data -```bash -git clone https://github.com/YOUR_GITHUB_USERNAME/narps_open_pipelines.git -``` - -## 2 - Get the data +Now that you cloned the repository using Datalad, you are able to get the data : -Ignore this step if you used DataLad (option 1) in the previous step. - -Otherwise, there are several ways to get the data. +```bash +# Move inside the root directory of the repository. +cd narps_open_pipelines -## 3 - Set up the environment +# Select the data you want to download. Here is an example to get data of the first 4 subjects. +datalad get data/original/ds001734/sub-00[1-4] -J 12 +datalad get data/original/ds001734/derivatives/fmriprep/sub-00[1-4] -J 12 +``` -The Narps Open Pipelines project is build upon several dependencies, such as [Nipype](https://nipype.readthedocs.io/en/latest/) but also the original software packages used by the pipelines (SPM, FSL, AFNI...). +> [!NOTE] +> For further information and alternatives on how to get the data, see the corresponding documentation page [docs/data.md](docs/data.md). -To facilitate this step, we created a Docker container based on [Neurodocker](https://github.com/ReproNim/neurodocker) that contains the necessary Python packages and software. To install the Docker image, two options are available. +## 4 - Set up the environment -### Option 1: Using Dockerhub +[Install Docker](https://docs.docker.com/engine/install/) then pull the Docker image : ```bash docker pull elodiegermani/open_pipeline:latest ``` -The image should install itself. Once it's done you can check the image is available on your system: +Once it's done you can check the image is available on your system : ```bash docker images + REPOSITORY TAG IMAGE ID CREATED SIZE docker.io/elodiegermani/open_pipeline latest 0f3c74d28406 9 months ago 22.7 GB ``` -### Option 2: Using a Dockerfile +> [!NOTE] +> Feel free to read this documentation page [docs/environment.md](docs/environment.md) to get further information about this environment. + +## 5 - Run the project + +Start a Docker container from the Docker image : + +```bash +# Replace PATH_TO_THE_REPOSITORY in the following command (e.g.: with /home/user/dev/narps_open_pipelines/) +docker run -it -v PATH_TO_THE_REPOSITORY:/home/neuro/code/ elodiegermani/open_pipeline +``` -The Dockerfile used to create the image stored on DockerHub is available at the root of the repository ([Dockerfile](Dockerfile)). But you might want to personalize this Dockerfile. To do so, change the command below that will generate a new Dockerfile: +Install NARPS Open Pipelines inside the container : ```bash -docker run --rm repronim/neurodocker:0.7.0 generate docker \ - --base neurodebian:stretch-non-free --pkg-manager apt \ - --install git \ - --fsl version=6.0.3 \ - --afni version=latest method=binaries install_r=true install_r_pkgs=true install_python2=true install_python3=true \ - --spm12 version=r7771 method=binaries \ - --user=neuro \ - --workdir /home \ - --miniconda create_env=neuro \ - conda_install="python=3.8 traits jupyter nilearn graphviz nipype scikit-image" \ - pip_install="matplotlib" \ - activate=True \ - --env LD_LIBRARY_PATH="/opt/miniconda-latest/envs/neuro:$LD_LIBRARY_PATH" \ - --run-bash "source activate neuro" \ - --user=root \ - --run 'chmod 777 -Rf /home' \ - --run 'chown -R neuro /home' \ - --user=neuro \ - --run 'mkdir -p ~/.jupyter && echo c.NotebookApp.ip = \"0.0.0.0\" > ~/.jupyter/jupyter_notebook_config.py' > Dockerfile +source activate neuro +cd /home/neuro/code/ +pip install . ``` -When you are satisfied with your Dockerfile, just build the image: +Finally, you are able to run pipelines : ```bash -docker build --tag [name_of_the_image] - < Dockerfile +python narps_open/runner.py + usage: runner.py [-h] -t TEAM (-r RSUBJECTS | -s SUBJECTS [SUBJECTS ...] | -n NSUBJECTS) [-g | -f] [-c] ``` -When the image is built, follow the instructions in [docs/environment.md](docs/environment.md) to start the environment from it. +> [!NOTE] +> For further information, read this documentation page [docs/running.md](docs/running.md). diff --git a/README.md b/README.md index 7ad3172c..20125d83 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,6 @@ This project is developed in the Empenn team by Boris Clenet, Elodie Germani, Je In addition, this project was presented and received contributions during the following events: - OHBM Brainhack 2022 (June 2022): Elodie Germani, Arshitha Basavaraj, Trang Cao, Rémi Gau, Anna Menacher, Camille Maumet. - - e-ReproNim FENS NENS Cluster Brainhack: - - OHBM Brainhack 2023 (July 2023): + - e-ReproNim FENS NENS Cluster Brainhack (June 2023) : Liz Bushby, Boris Clénet, Michael Dayan, Aimee Westbrook. + - OHBM Brainhack 2023 (July 2023): Arshitha Basavaraj, Boris Clénet, Rémi Gau, Élodie Germani, Yaroslav Halchenko, Camille Maumet, Paul Taylor. - ORIGAMI lab hackathon (Sept 2023): diff --git a/docs/environment.md b/docs/environment.md index 98addd6a..edab9b4d 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -1,100 +1,82 @@ -# Set up the environment to run pipelines +# About the environment of NARPS Open Pipelines -## Run a docker container :whale: +## The Docker container :whale: -Start a container using the command below: +The NARPS Open Pipelines project is build upon several dependencies, such as [Nipype](https://nipype.readthedocs.io/en/latest/) but also the original software packages used by the pipelines (SPM, FSL, AFNI...). Therefore, we created a Docker container based on [Neurodocker](https://github.com/ReproNim/neurodocker) that contains software dependencies. -```bash -docker run -ti \ - -p 8888:8888 \ - elodiegermani/open_pipeline -``` - -On this command line, you need to add volumes to be able to link with your local files (original dataset and git repository). If you stored the original dataset in `data/original`, just make a volume with the `narps_open_pipelines` directory: - -```bash -docker run -ti \ - -p 8888:8888 \ - -v /users/egermani/Documents/narps_open_pipelines:/home/ \ - elodiegermani/open_pipeline -``` - -If it is in another directory, make a second volume with the path to your dataset: - -```bash -docker run -ti \ - -p 8888:8888 \ - -v /Users/egermani/Documents/narps_open_pipelines:/home/ \ - -v /Users/egermani/Documents/data/NARPS/:/data/ \ - elodiegermani/open_pipeline -``` - -After that, your container will be launched! - -## Other useful docker commands - -### START A CONTAINER - -```bash -docker start [name_of_the_container] -``` - -### VERIFY A CONTAINER IS IN THE LIST - -```bash -docker ps -``` - -### EXECUTE BASH OR ATTACH YOUR CONTAINER +The simplest way to start the container using the command below : ```bash -docker exec -ti [name_of_the_container] bash +docker run -it elodiegermani/open_pipeline ``` -**OR** +From this command line, you need to add volumes to be able to link with your local files (code repository). ```bash -docker attach [name_of_the_container] -``` +# Replace PATH_TO_THE_REPOSITORY in the following command (e.g.: with /home/user/dev/narps_open_pipelines/) +docker run -it \ + -v PATH_TO_THE_REPOSITORY:/home/neuro/code/ \ + elodiegermani/open_pipeline +``` -## Useful commands inside the container +## Use Jupyter with the container -### ACTIVATE CONDA ENVIRONMENT +If you wish to use [Jupyter](https://jupyter.org/) to run the code, a port forwarding is needed : ```bash -source activate neuro -``` +docker run -it \ + -v PATH_TO_THE_REPOSITORY:/home/neuro/code/ \ + -p 8888:8888 \ + elodiegermani/open_pipeline +``` -### LAUNCH JUPYTER NOTEBOOK +Then, from inside the container : ```bash jupyter notebook --port=8888 --no-browser --ip=0.0.0.0 ``` -## If you did not use your container for a while +You can now access Jupyter using the address provided by the command line. -Verify it still runs : +> [!NOTE] +> Find useful information on the [Docker documentation page](https://docs.docker.com/get-started/). Here is a [cheat sheet with Docker commands](https://docs.docker.com/get-started/docker_cheatsheet.pdf) -```bash -docker ps -l -``` +## Create a custom Docker image -If your container is in the list, run : +The `elodiegermani/open_pipeline` Docker image is based on [Neurodocker](https://github.com/ReproNim/neurodocker). It was created using the following command line : ```bash -docker start [name_of_the_container] +docker run --rm repronim/neurodocker:0.7.0 generate docker \ + --base neurodebian:stretch-non-free --pkg-manager apt \ + --install git \ + --fsl version=6.0.3 \ + --afni version=latest method=binaries install_r=true install_r_pkgs=true install_python2=true install_python3=true \ + --spm12 version=r7771 method=binaries \ + --user=neuro \ + --workdir /home \ + --miniconda create_env=neuro \ + conda_install="python=3.8 traits jupyter nilearn graphviz nipype scikit-image" \ + pip_install="matplotlib" \ + activate=True \ + --env LD_LIBRARY_PATH="/opt/miniconda-latest/envs/neuro:$LD_LIBRARY_PATH" \ + --run-bash "source activate neuro" \ + --user=root \ + --run 'chmod 777 -Rf /home' \ + --run 'chown -R neuro /home' \ + --user=neuro \ + --run 'mkdir -p ~/.jupyter && echo c.NotebookApp.ip = \"0.0.0.0\" > ~/.jupyter/jupyter_notebook_config.py' > Dockerfile ``` -Else, relaunch it with : +If you wish to create your own custom environment, make changes to the parameters, and build your custom image from the generated Dockerfile. ```bash -docker run -ti \ - -p 8888:8888 \ - -v /home/egermani:/home \ - [name_of_the_image] +# Replace IMAGE_NAME in the following command +docker build --tag IMAGE_NAME - < Dockerfile ``` -### To use SPM inside the container, use this command at the beginning of your script: +## Good to know + +To use SPM inside the container, use this command at the beginning of your script: ```python from nipype.interfaces import spm diff --git a/docs/running.md b/docs/running.md index 6344c042..eb614eef 100644 --- a/docs/running.md +++ b/docs/running.md @@ -1,6 +1,33 @@ -# :running: How to run NARPS open pipelines ? +# How to run NARPS open pipelines ? :running: -## Using the `PipelineRunner` +## Using the runner application + +The `narps_open.runner` module allows to run pipelines from the command line : + +```bash +python narps_open/runner.py -h + usage: runner.py [-h] -t TEAM (-r RANDOM | -s SUBJECTS [SUBJECTS ...]) [-g | -f] + + Run the pipelines from NARPS. + + options: + -h, --help show this help message and exit + -t TEAM, --team TEAM the team ID + -r RANDOM, --random RANDOM the number of subjects to be randomly selected + -s SUBJECTS [SUBJECTS ...], --subjects SUBJECTS [SUBJECTS ...] a list of subjects + -g, --group run the group level only + -f, --first run the first levels only (preprocessing + subjects + runs) + -c, --check check pipeline outputs (runner is not launched) + +python narps_open/runner.py -t 2T6S -s 001 006 020 100 +python narps_open/runner.py -t 2T6S -r 4 +python narps_open/runner.py -t 2T6S -r 4 -f +python narps_open/runner.py -t 2T6S -r 4 -f -c # Check the output files without launching the runner +``` + +In this usecase, the paths where to store the outputs and to the dataset are picked by the runner from the [configuration](docs/configuration.md). + +## Using the `PipelineRunner` object The class `PipelineRunner` is available from the `narps_open.runner` module. You can use it from inside python code, as follows : @@ -35,30 +62,3 @@ runner.start(True, True) runner.get_missing_first_level_outputs() runner.get_missing_group_level_outputs() ``` - -## Using the runner application - -The `narps_open.runner` module also allows to run pipelines from the command line : - -```bash -python narps_open/runner.py -h - usage: runner.py [-h] -t TEAM (-r RANDOM | -s SUBJECTS [SUBJECTS ...]) [-g | -f] - - Run the pipelines from NARPS. - - options: - -h, --help show this help message and exit - -t TEAM, --team TEAM the team ID - -r RANDOM, --random RANDOM the number of subjects to be randomly selected - -s SUBJECTS [SUBJECTS ...], --subjects SUBJECTS [SUBJECTS ...] a list of subjects - -g, --group run the group level only - -f, --first run the first levels only (preprocessing + subjects + runs) - -c, --check check pipeline outputs (runner is not launched) - -python narps_open/runner.py -t 2T6S -s 001 006 020 100 -python narps_open/runner.py -t 2T6S -r 4 -python narps_open/runner.py -t 2T6S -r 4 -f -python narps_open/runner.py -t 2T6S -r 4 -f -c # Check the output files without launching the runner -``` - -In this usecase, the paths where to store the outputs and to the dataset are picked by the runner from the [configuration](docs/configuration.md).