Merge pull request #315 from cohenlabUNC/release-1.7.3

Release 1.7.3
cohenlabUNC · Feb 23, 2023 · 2568ca3 · 2568ca3
2 parents db89f6c + b266eeb
commit 2568ca3
Show file tree

Hide file tree

Showing 41 changed files with 1,077 additions and 983 deletions.
diff --git a/.gitignore b/.gitignore
@@ -61,6 +61,7 @@ crashlytics-build.properties
 staticfiles
 .env
 TestFiles/
+tests/temp
 tests/artifacts
 deploy.sh
 

diff --git a/README.md b/README.md
@@ -26,7 +26,7 @@ All clpipe commands should now be accessible.
 ### Manual Installation
 
 1. Open a Longleaf terminal session
-2. Switch to python 3.6.6 using `module add python/3.6.6`
+2. Switch to python 3.7 using `module add python/3.7.14`
 3. Install clpipe from GitHub with 
 ```pip3 install --user --upgrade  git+https://github.com/cohenlabUNC/clpipe.git```
 

diff --git a/clpipe/batch_manager.py b/clpipe/batch_manager.py
@@ -27,7 +27,7 @@ def __init__(self, batch_system_config: os.PathLike, output_directory=None,
 
         if os.path.exists(os.path.abspath(batch_system_config)):
             self.logger.debug(f"Using batch config at: {batch_system_config}")
-            with os.open(os.path.abspath(batch_system_config)) as bat_config:
+            with open(os.path.abspath(batch_system_config)) as bat_config:
                 self.config = json.load(bat_config)
         else:
             with resource_stream(

diff --git a/clpipe/bids_conversion.py b/clpipe/bids_conversion.py
@@ -23,10 +23,10 @@
 
 
 def convert2bids(dicom_dir=None, dicom_dir_format=None, bids_dir=None, 
-                 conv_config_file=None, config_file=None, overwrite=None, 
+                 conv_config_file=None, config_file=None, overwrite=False, 
                  clear_cache=False, clear_outputs=False, log_dir=None, subject=None,
                  subjects=None, session=None, 
-                 longitudinal=False, status_cache=None, submit=None, debug=False, 
+                 longitudinal=False, status_cache=None, submit=False, debug=False, 
                  dcm2bids=True, batch=False):
 
     config_parser = ClpipeConfigParser()

diff --git a/clpipe/bids_validator.py b/clpipe/bids_validator.py
@@ -45,23 +45,29 @@ def bids_validate(bids_dir=None, config_file=None, log_dir=None,
         debug=debug)
     batch_manager.update_mem_usage('3000')
 
+    # Validator image moved to BIDSValidationOptions block
+    # If the image isn't there, try looking in the old spot for backwards compatibility
+    #   with clpipe <= v1.7.2
+    try:
+        bids_validator_image = config.config['BIDSValidationOptions']['BIDSValidatorImage']
+    except KeyError:
+        bids_validator_image = config.config['PostProcessingOptions']['BIDSValidatorImage']
+
     singularity_string = SINGULARITY_CMD_TEMPLATE
     if verbose:
         logger.debug("Verbose mode: on")
         singularity_string = singularity_string + ' --verbose'
     if interactive:
         logger.info("Running BIDS validation interactively.")
         os.system(singularity_string.format(
-            validatorInstance=config
-                .config['PostProcessingOptions']['BIDSValidatorImage'],
+            validatorInstance=bids_validator_image,
             bidsDir=bids_dir,
             bindPaths=batch_manager.config['SingularityBindPaths']
         ))
         logger.info("Validation complete.")
     else:
         batch_manager.addjob(Job("BIDSValidator", singularity_string.format(
-            validatorInstance=config
-                .config['PostProcessingOptions']['BIDSValidatorImage'],
+            validatorInstance=bids_validator_image,
             bidsDir=config.config['FMRIPrepOptions']['BIDSDirectory'],
             bindPaths=batch_manager.config['SingularityBindPaths']
         )))

diff --git a/clpipe/cli.py b/clpipe/cli.py
@@ -171,7 +171,7 @@ def project_setup_cli(project_title=None, project_dir=None, source_data=None,
 
 @click.command(CONVERSION_COMMAND_NAME, no_args_is_help=True)
 @click.argument('subjects', nargs=-1, required=False, default=None)
-@click.option('-config_file', '-c', type=CLICK_FILE_TYPE_EXISTS, default=None,
+@click.option('-config_file', '-c', type=CLICK_FILE_TYPE_EXISTS, required=True,
               help=CONFIG_HELP)
 @click.option('-conv_config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, 
               help=CONVERSION_CONFIG_HELP)
@@ -221,7 +221,7 @@ def convert2bids_cli(dicom_dir, dicom_dir_format, bids_dir,
 
 @click.command(VALIDATOR_COMMAND_NAME, no_args_is_help=True)
 @click.argument('bids_dir', type=CLICK_DIR_TYPE_EXISTS, required=False)
-@click.option('-config_file', '-c', type=CLICK_FILE_TYPE_EXISTS, default=None, 
+@click.option('-config_file', '-c', type=CLICK_FILE_TYPE_EXISTS, required=True, 
               help=CONFIG_HELP)
 @click.option('-log_dir', type=CLICK_FILE_TYPE_EXISTS, default=None,
               help=LOG_DIR_HELP)
@@ -248,7 +248,7 @@ def bids_validate_cli(bids_dir, config_file, log_dir, interactive, submit,
 
 @click.command(FMRIPREP_COMMAND_NAME, no_args_is_help=True)
 @click.argument('subjects', nargs=-1, required=False, default=None)
-@click.option('-config_file', '-c', default=None, type=CLICK_FILE_TYPE_EXISTS, 
+@click.option('-config_file', '-c', required=True, type=CLICK_FILE_TYPE_EXISTS, 
               help=CONFIG_HELP)
 @click.option('-bids_dir', '-i', type=CLICK_DIR_TYPE_EXISTS,
               help=BIDS_DIR_HELP)
@@ -295,7 +295,7 @@ def get_fmriprep_reports_cli(config_file, output_name, clear_temp, debug):
 
 @click.command(POSTPROCESS_COMMAND_NAME, no_args_is_help=True)
 @click.argument('subjects', nargs=-1, required=False, default=None)
-@click.option('-config_file', '-c', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, help = 'Use a given configuration file. If left blank, uses the default config file, requiring definition of BIDS, working and output directories.')
+@click.option('-config_file', '-c', type=click.Path(exists=True, dir_okay=False, file_okay=True), required=True, help = 'Use a given configuration file. If left blank, uses the default config file, requiring definition of BIDS, working and output directories.')
 @click.option('-target_dir', '-i', type=click.Path(exists=True, dir_okay=True, file_okay=False), help='Which fmriprep directory to process. If a configuration file is provided with a BIDS directory, this argument is not necessary. Note, must point to the ``fmriprep`` directory, not its parent directory.')
 @click.option('-target_suffix', help= 'Which file suffix to use. If a configuration file is provided with a target suffix, this argument is not necessary. Defaults to "preproc_bold.nii.gz"')
 @click.option('-output_dir', '-o', type=click.Path(dir_okay=True, file_okay=False), help = 'Where to put the postprocessed data. If a configuration file is provided with a output directory, this argument is not necessary.')
@@ -332,8 +332,7 @@ def fmri_postprocess_cli(config_file=None, subjects=None, target_dir=None,
 
 @click.command(POSTPROCESS2_COMMAND_NAME, no_args_is_help=True)
 @click.argument('subjects', nargs=-1, required=False, default=None)
-@click.option('-config_file', '-c', type=CLICK_FILE_TYPE_EXISTS, default=None, 
-              required=True, help=CONFIG_HELP)
+@click.option('-config_file', '-c', type=CLICK_FILE_TYPE_EXISTS, required=True, help=CONFIG_HELP)
 @click.option('-fmriprep_dir', '-i', type=CLICK_DIR_TYPE_EXISTS, 
               help=FMRIPREP_DIR_HELP)
 @click.option('-output_dir', '-o', type=CLICK_DIR_TYPE, default=None, required=False,
@@ -371,11 +370,11 @@ def fmri_postprocess2_cli(subjects, config_file, fmriprep_dir, output_dir,
 
 @click.command(GLM_SETUP_COMMAND_NAME, no_args_is_help=True)
 @click.argument('subjects', nargs=-1, required=False, default=None)
-@click.option('-config_file', '-c', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True,
+@click.option('-config_file', '-c', type=click.Path(exists=True, dir_okay=False, file_okay=True), required=True,
               help='Use a given configuration file.')
-@click.option('-glm_config_file', '-g', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True,
+@click.option('-glm_config_file', '-g', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required=True,
               help='Use a given GLM configuration file.')
-@click.option('-drop_tps', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = False,
+@click.option('-drop_tps', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required=False,
               help='Drop timepoints csv sheet')
 @click.option('-submit', '-s', is_flag=True, default=False, help='Flag to submit commands to the HPC.')
 @click.option('-batch/-single', default=True,
@@ -402,8 +401,7 @@ def glm_setup_cli(subjects, config_file, glm_config_file, submit, batch, debug,
 @click.argument('level')
 @click.argument('model')
 @click.option('-glm_config_file', '-g', type=click.Path(exists=True, dir_okay=False, 
-              file_okay=True), default=None, required = True,
-              help=CONFIG_HELP)
+              file_okay=True), required=True, help=CONFIG_HELP)
 @click.option('-debug', '-d', is_flag=True, 
               help=DEBUG_HELP)
 def glm_prepare_cli(level, model, glm_config_file, debug):
@@ -419,7 +417,7 @@ def glm_prepare_cli(level, model, glm_config_file, debug):
 
 
 @click.command(L1_PREPARE_FSF_COMMAND_NAME, no_args_is_help=True)
-@click.option('-glm_config_file', '-g', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True,
+@click.option('-glm_config_file', '-g', type=click.Path(exists=True, dir_okay=False, file_okay=True), required=True,
               help='Your GLM configuration file.')
 @click.option('-l1_name',  default=None, required = True,
               help='Name for a given L1 model as defined in your GLM configuration file.')
@@ -429,9 +427,9 @@ def glm_l1_preparefsf_cli(glm_config_file, l1_name, debug):
     
     You must create a template .fsf file in FSL's FEAT GUI first.
     """
-    from .glm_l1 import glm_l1_preparefsf
-    glm_l1_preparefsf(
-        glm_config_file=glm_config_file, l1_name=l1_name, debug=debug)
+    from .glm_prepare import glm_prepare
+    glm_prepare(glm_config_file=glm_config_file, level="L1", model=l1_name,
+                debug=debug)
 
 
 @click.command(L2_PREPARE_FSF_COMMAND_NAME, no_args_is_help=True)
@@ -446,9 +444,9 @@ def glm_l2_preparefsf_cli(glm_config_file, l2_name, debug):
     
     You must create a group-level template .fsf file in FSL's FEAT GUI first.
     """
-    from .glm_l2 import glm_l2_preparefsf
-    glm_l2_preparefsf(glm_config_file=glm_config_file, l2_name=l2_name,
-                      debug=debug)
+    from .glm_prepare import glm_prepare
+    glm_prepare(glm_config_file=glm_config_file, level="L2", model=l2_name,
+                debug=debug)
 
 
 @click.command(APPLY_MUMFORD_COMMAND_NAME, no_args_is_help=True)
@@ -473,7 +471,7 @@ def glm_apply_mumford_workaround_cli(glm_config_file, l1_feat_folders_path,
 
     Must provide GLM config file OR a path to your L1 FEAT folders.
     """
-    from .glm_l2 import glm_apply_mumford_workaround
+    from .glm_prepare import glm_apply_mumford_workaround
     if not (glm_config_file or l1_feat_folders_path):
         click.echo(("Error: At least one of either option '-glm_config_file' "
                     "or '-l1_feat_folders_path' required."))
@@ -488,7 +486,7 @@ def glm_apply_mumford_workaround_cli(glm_config_file, l1_feat_folders_path,
 @click.argument('level')
 @click.argument('model')
 @click.option('-glm_config_file', '-g', type=click.Path(exists=True, dir_okay=False, 
-              file_okay=True), default=None, required = True,
+              file_okay=True), required=True,
               help=CONFIG_HELP)
 @click.option('-test_one', is_flag=True,
               help=TEST_ONE_HELP)
@@ -511,9 +509,9 @@ def glm_launch_cli(level, model, glm_config_file, test_one, submit, debug):
 
 @click.command(no_args_is_help=True)
 @click.option('-glm_config_file', '-g', type=click.Path(exists=True, dir_okay=False, 
-              file_okay=True), default=None, required = True,
+              file_okay=True), required=True,
               help=CONFIG_HELP)
-@click.option('-l1_name',  default=None, required = True,
+@click.option('-l1_name', required=True,
               help=L1_MODEL_HELP)
 @click.option('-test_one', is_flag=True,
               help=TEST_ONE_HELP)
@@ -530,9 +528,9 @@ def glm_l1_launch_cli(glm_config_file, l1_name, test_one, submit, debug):
 
 @click.command(no_args_is_help=True)
 @click.option('-glm_config_file', '-g', type=click.Path(exists=True, dir_okay=False, 
-              file_okay=True), default=None, required = True,
+              file_okay=True), required=True,
               help=CONFIG_HELP)
-@click.option('-l2_name',  default=None, required = True,
+@click.option('-l2_name', required=True,
               help=L2_MODEL_HELP)
 @click.option('-test_one', is_flag=True,
               help=TEST_ONE_HELP)
@@ -550,7 +548,7 @@ def glm_l2_launch_cli(glm_config_file, l2_name, test_one, submit, debug):
 
 @click.command(ONSET_EXTRACT_COMMAND_NAME, no_args_is_help=True)
 @click.option('-config_file', '-c', type=click.Path(exists=True, dir_okay=False, file_okay=True), 
-              default=None, required = True,
+              required=True,
               help='Use a given configuration file.')
 @click.option('-glm_config_file', '-g', type=click.Path(exists=True, dir_okay=False, file_okay=True), 
               default=None, required = True,

diff --git a/clpipe/config_json_parser.py b/clpipe/config_json_parser.py
@@ -126,11 +126,13 @@ def setup_glm(self, project_path):
         os.mkdir(os.path.join(project_path, "l2_fsfs"))
         os.mkdir(os.path.join(project_path, "l2_gfeat_folders"))
 
-        glm_config.config_json_dump(project_path, "glm_config.json")
-        shutil.copy(resource_filename('clpipe', 'data/l2_sublist.csv'), os.path.join(project_path, "l2_sublist.csv"))
         glm_config.config['GLMSetupOptions']['LogDirectory'] = os.path.join(project_path, "logs", "glm_setup_logs")
         os.mkdir(os.path.join(project_path, "logs", "glm_setup_logs"))
 
+        glm_config.config_json_dump(project_path, "glm_config.json")
+        shutil.copyfile(resource_filename('clpipe', 'data/l2_sublist.csv'), os.path.join(project_path, "l2_sublist.csv"))
+
+
     def setup_fmriprep_directories(self, bidsDir, workingDir, outputDir, log_dir = None):
         if bidsDir is not None:
             self.config['FMRIPrepOptions']['BIDSDirectory'] = os.path.abspath(bidsDir)
@@ -199,7 +201,9 @@ def setup_dcm2bids(self, dicom_directory, heuristic_file, output_directory, dico
         if not os.path.exists(bids_ignore_path):
             with open(bids_ignore_path, 'w') as bids_ignore_file:
                 # Ignore dcm2bid's auto-generated directory
-                bids_ignore_file.write("tmp_dcm2bids")
+                bids_ignore_file.write("tmp_dcm2bids\n")
+                # Ignore heudiconv's auto-generated scan file
+                bids_ignore_file.write("scans.json\n")
 
     def setup_bids_validation(self, log_dir=None):
         if log_dir is not None:

diff --git a/clpipe/data/defaultConfig.json b/clpipe/data/defaultConfig.json
@@ -4,6 +4,15 @@
 	"ProjectDirectory": "",
 	"EmailAddress": "",
 	"TempDirectory": "",
+	"SourceOptions": {
+		"SourceURL": "",
+		"DropoffDirectory": "",
+		"TempDirectory": "",
+		"CommandLineOpts": "",
+		"TimeUsage": "1:0:0",
+		"MemUsage": "10000",
+		"CoreUsage": "1"
+	},
 	"DICOMToBIDSOptions": {
 		"DICOMDirectory": "",
 		"BIDSDirectory": "",
@@ -15,19 +24,26 @@
         "LogDirectory": ""
 	},
 	"BIDSValidationOptions": {
+		"BIDSValidatorImage": "/proj/hng/singularity_imgs/validator.simg",
 		"LogDirectory": ""
 	},
 	"FMRIPrepOptions": {
 		"BIDSDirectory": "",
 		"WorkingDirectory": "",
 		"OutputDirectory": "",
-		"FMRIPrepPath": "/proj/hng/singularity_imgs/fmriprep_21.0.2.sif",
+		"FMRIPrepPath": "/proj/hng/singularity_imgs/fmriprep_22.1.1.sif",
 		"FreesurferLicensePath": "/proj/hng/singularity_imgs/license.txt",
 		"UseAROMA": false,
 		"CommandLineOpts": "",
 		"TemplateFlowToggle": true,
 		"TemplateFlowPath": "",
-		"TemplateFlowTemplates": ["MNI152NLin2009cAsym", "MNI152NLin6Asym", "OASIS30ANTs", "MNIPediatricAsym", "MNIInfant"],
+		"TemplateFlowTemplates": [
+			"MNI152NLin2009cAsym", 
+			"MNI152NLin6Asym", 
+			"OASIS30ANTs", 
+			"MNIPediatricAsym", 
+			"MNIInfant"
+		],
 		"FMapCleanupROIs": 3,
 		"FMRIPrepMemoryUsage": "50000",
 		"FMRIPrepTimeUsage": "16:0:0",
@@ -44,14 +60,26 @@
 		"ConfoundSuffix": "desc-confounds_regressors.tsv",
 		"DropCSV": "",
 		"Regress": true,
-	    "Confounds": ["trans_x", "trans_y", "trans_z", "rot_x", "rot_y", "rot_z",
-   		 "csf", "white_matter", "global_signal", "a_comp_cor.*"],
-		"ConfoundsQuad": ["trans_x", "trans_y", "trans_z", "rot_x", "rot_y", "rot_z",
-    	 "csf", "white_matter", "global_signal"],
-		"ConfoundsDerive": ["trans_x", "trans_y", "trans_z", "rot_x", "rot_y", "rot_z",
-    	 "csf", "white_matter", "global_signal"],
-		"ConfoundsQuadDerive": ["trans_x", "trans_y", "trans_z", "rot_x", "rot_y", "rot_z",
-    	 "csf", "white_matter", "global_signal"],
+	    "Confounds": [
+			"trans_x", "trans_y", "trans_z", 
+			"rot_x", "rot_y", "rot_z",
+   		 	"csf", "white_matter", "global_signal", "a_comp_cor.*"
+		],
+		"ConfoundsQuad": [
+			"trans_x", "trans_y", "trans_z", 
+			"rot_x", "rot_y", "rot_z",
+    	 	"csf", "white_matter", "global_signal"
+		],
+		"ConfoundsDerive": [
+			"trans_x", "trans_y", "trans_z", 
+			"rot_x", "rot_y", "rot_z",
+    	 	"csf", "white_matter", "global_signal"
+		],
+		"ConfoundsQuadDerive": [
+			"trans_x", "trans_y", "trans_z", 
+			"rot_x", "rot_y", "rot_z",
+    	 	"csf", "white_matter", "global_signal"
+		],
 		"FilteringHighPass": 0.008,
 		"FilteringLowPass": -1,
 		"FilteringOrder": 2,
@@ -64,13 +92,15 @@
 		"ScrubBehind": 1,
 		"ScrubContig": 2,
 		"RespNotchFilter": true,
-		"MotionVars": ["trans_x", "trans_y", "trans_z", "rot_x", "rot_y", "rot_z"],
+		"MotionVars": [
+			"trans_x", "trans_y", "trans_z", 
+			"rot_x", "rot_y", "rot_z"
+		],
 		"RespNotchFilterBand":[0.31,0.43],
 		"PostProcessingMemoryUsage": "20000",
 		"PostProcessingTimeUsage": "8:0:0",
 		"NThreads": "1",
 		"SpectralInterpolationBinSize": 5000,
-		"BIDSValidatorImage": "/proj/hng/singularity_imgs/validator.simg",
         "LogDirectory": ""
 	},
 	"PostProcessingOptions2": {
@@ -116,7 +146,10 @@
 			}
 		},
 		"ConfoundOptions": {
-			"Columns": ["csf", "csf_derivative1", "white_matter", "white_matter_derivative1"],
+			"Columns": [
+				"csf", "csf_derivative1", 
+				"white_matter", "white_matter_derivative1"
+			],
 			"MotionOutliers": {
 				"Include": true,
 				"ScrubVar": "framewise_displacement",