From d3fed64990231385480cf5c6641cac600b3e77c5 Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Thu, 17 Nov 2022 12:11:14 -0500 Subject: [PATCH 01/16] Update version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6210c60a..f3f20a10 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup(name='clpipe', - version='1.7.0', + version='1.7.1', description='clpipe: MRI processing pipeline for high performance clusters', url='https://github.com/cohenlabUNC/clpipe', author='Maintainer: Teague Henry, Maintainer: Will Asciutto, Contributor: Deepak Melwani', From 5ca842858a81d7f700bc62f121f42d8d742258b9 Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Thu, 17 Nov 2022 15:23:43 -0500 Subject: [PATCH 02/16] Move CLI commands back to CLI class. Use function isolated imports to prevent having to load in all packages from all tools. --- clpipe/bids_conversion.py | 81 +------ clpipe/bids_validator.py | 30 --- clpipe/cli.py | 406 +++++++++++++++++++++++++++++++++--- clpipe/config.py | 108 +++++++++- clpipe/fmri_postprocess.py | 33 --- clpipe/fmri_postprocess2.py | 57 +---- clpipe/fmri_preprocess.py | 47 ----- clpipe/fsl_onset_extract.py | 16 -- clpipe/glm_l1.py | 16 -- clpipe/glm_l2.py | 45 +--- clpipe/glm_launch.py | 71 +------ clpipe/glm_setup.py | 25 --- clpipe/outliers_report.py | 18 -- clpipe/project_setup.py | 36 +--- clpipe/status.py | 14 -- 15 files changed, 491 insertions(+), 512 deletions(-) diff --git a/clpipe/bids_conversion.py b/clpipe/bids_conversion.py index 82723616..1f440959 100644 --- a/clpipe/bids_conversion.py +++ b/clpipe/bids_conversion.py @@ -1,24 +1,19 @@ -from distutils.log import debug from pathlib import Path -from .batch_manager import LOGGER_NAME, BatchManager, Job +from .batch_manager import BatchManager, Job from .config_json_parser import ClpipeConfigParser import os import parse import glob import sys import click +import logging from .utils import get_logger, add_file_handler from .status import needs_processing, write_record -from .config import BATCH_HELP, CONFIG_HELP, DEBUG_HELP, LOG_DIR_HELP, SUBMIT_HELP, CLICK_FILE_TYPE, \ - STATUS_CACHE_HELP, CLICK_FILE_TYPE_EXISTS, CLICK_DIR_TYPE_EXISTS # These imports are for the heudiconv converter from pkg_resources import resource_filename -from .error_handler import exception_handler -import logging -COMMAND_NAME = "convert" INFO_COMMAND_NAME = "dicom-info" STEP_NAME = "bids-conversion" BASE_CMD = ("dcm2bids -d {subject_dicom_dir} -o {bids_dir} " @@ -26,77 +21,6 @@ HEUDICONV_BASE_CMD = '''heudiconv --files {subject_dicom_dir} -s {subject} '''\ '''-f {heuristic} -o {output_directory} -b''' -CONVERSION_CONFIG_HELP = ( - "A dcm2bids conversion definition .json file." -) -HEURISTIC_FILE_HELP = ( - "A heudiconv heuristic file to use for the conversion." -) -DICOM_DIR_HELP = "The folder where subject dicoms are located." -DICOM_DIR_FORMAT_HELP = ( - "Format string for how subjects/sessions are organized within the " - "dicom_dir." -) -BIDS_DIR_HELP = "The dicom info output file name." -OVERWRITE_HELP = "Overwrite existing BIDS data?" -SUBJECT_HELP = ( - "Deprecated version of specifying one subject to process - can give an " - "arbitrary number of subjects as arguments now." -) -SESSION_HELP = ( - "A session to convert using the supplied configuration file. Use in " - "combination with -subject to convert single subject/sessions, " - "else leave empty" -) -LONGITUDINAL_HELP = ( - "Convert all subjects/sessions into individual pseudo-subjects. " - "Use if you do not want T1w averaged across sessions during FMRIprep" -) -MODE_HELP = ( - "Specify which converter to use." -) - - -@click.command(COMMAND_NAME) -@click.argument('subjects', nargs=-1, required=False, default=None) -@click.option('-config_file', '-c', type=CLICK_FILE_TYPE_EXISTS, default=None, - help=CONFIG_HELP) -@click.option('-conv_config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, - help=CONVERSION_CONFIG_HELP) -@click.option('-dicom_dir', '-i', type=CLICK_DIR_TYPE_EXISTS, help=DICOM_DIR_HELP) -@click.option('-dicom_dir_format', help=DICOM_DIR_FORMAT_HELP) -@click.option('-BIDS_dir', '-o', type=CLICK_DIR_TYPE_EXISTS, - help=BIDS_DIR_HELP) -@click.option('-overwrite', is_flag=True, default=False, help=OVERWRITE_HELP) -@click.option('-clear_cache', is_flag=True, default=False, help="Clear cached data for given subject.", - hidden=True) -@click.option('-clear_outputs', is_flag=True, default=False, help="Clear all BIDS data for given subject.", - hidden=True) -@click.option('-log_dir', type=CLICK_DIR_TYPE_EXISTS, help=LOG_DIR_HELP) -@click.option('-subject', required=False, help=SUBJECT_HELP) -@click.option('-session', required=False, help=SESSION_HELP) -@click.option('-longitudinal', is_flag=True, default=False, - help=LONGITUDINAL_HELP) -@click.option('-submit', '-s', is_flag=True, default=False, help=SUBMIT_HELP) -@click.option('-batch/-no-batch', is_flag = True, default=True, - help=BATCH_HELP, hidden=True) -@click.option('-debug', '-d', is_flag=True, help=DEBUG_HELP) -@click.option('-dcm2bids/-heudiconv', default=True, help=MODE_HELP) -@click.option('-status_cache', default=None, type=CLICK_FILE_TYPE, - help=STATUS_CACHE_HELP, hidden=True) -def convert2bids_cli(dicom_dir, dicom_dir_format, bids_dir, - conv_config_file, dcm2bids, - config_file, overwrite, clear_cache, clear_outputs, - log_dir, subject, subjects, session, - longitudinal, submit, batch, debug, status_cache): - """Convert DICOM files to BIDS format""" - convert2bids( - dicom_dir=dicom_dir, dicom_dir_format=dicom_dir_format, - bids_dir=bids_dir, conv_config_file=conv_config_file, - config_file=config_file, overwrite=overwrite, clear_cache=clear_cache, clear_outputs=clear_outputs, - log_dir=log_dir, batch=batch, subject=subject, subjects=subjects, session=session, - longitudinal=longitudinal, submit=submit, status_cache=status_cache, debug=debug, dcm2bids=dcm2bids) - def convert2bids(dicom_dir=None, dicom_dir_format=None, bids_dir=None, conv_config_file=None, config_file=None, overwrite=None, @@ -406,7 +330,6 @@ def _get_sub_session_list(dicom_dir, dicom_dir_format, logger, subjects=None, se return sub_sess_list, folders - @click.command(INFO_COMMAND_NAME) @click.option('-config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default = None, help = 'The configuration file for the study, use if you have a custom batch configuration.') @click.option('-subject', required=True, default=None, help = 'A subject that has all scans of interest present.') diff --git a/clpipe/bids_validator.py b/clpipe/bids_validator.py index ae6fbedf..7a7f72c9 100644 --- a/clpipe/bids_validator.py +++ b/clpipe/bids_validator.py @@ -1,44 +1,14 @@ import os import sys -import click from .batch_manager import BatchManager, Job from .config_json_parser import ClpipeConfigParser from .utils import add_file_handler, get_logger -from .config import CONFIG_HELP, LOG_DIR_HELP, SUBMIT_HELP, \ - INTERACTIVE_HELP, DEBUG_HELP, CLICK_FILE_TYPE_EXISTS, CLICK_DIR_TYPE_EXISTS -COMMAND_NAME = "validate" STEP_NAME = "bids-validation" SINGULARITY_CMD_TEMPLATE = ('singularity run --cleanenv -B {bindPaths} ' '{validatorInstance} {bidsDir}') -VERBOSE_HELP = ( - "Creates verbose validator output. Use if you want to see ALL files " - "with errors/warnings." -) - - -@click.command(COMMAND_NAME) -@click.option('-config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, - help=CONFIG_HELP) -@click.argument('bids_dir', type=CLICK_DIR_TYPE_EXISTS, required=False) -@click.option('-log_dir', type=CLICK_FILE_TYPE_EXISTS, default=None, - help=LOG_DIR_HELP) -@click.option('-verbose', is_flag=True, default=False, - help=VERBOSE_HELP) -@click.option('-submit', is_flag=True, help=SUBMIT_HELP) -@click.option('-interactive', is_flag=True, default=False, - help=INTERACTIVE_HELP) -@click.option('-debug', is_flag=True, help=DEBUG_HELP) -def bids_validate_cli(bids_dir, config_file, log_dir, interactive, submit, - verbose, debug): - """Check that the given directory conforms to the BIDS standard""" - - bids_validate( - bids_dir=bids_dir, config_file=config_file, log_dir=log_dir, - interactive=interactive, submit=submit, verbose=verbose, debug=debug) - def bids_validate(bids_dir=None, config_file=None, log_dir=None, interactive=False, submit=False, verbose=False, diff --git a/clpipe/cli.py b/clpipe/cli.py index 1e0434ba..4b349a62 100644 --- a/clpipe/cli.py +++ b/clpipe/cli.py @@ -2,21 +2,7 @@ import sys import pkg_resources -from .config import VERSION_HELP - -from .project_setup import project_setup_cli -from .bids_validator import bids_validate_cli -from .bids_conversion import convert2bids_cli -from .fmri_preprocess import fmriprep_process_cli -from .fmri_postprocess import fmri_postprocess_cli -from .fmri_postprocess2 import fmri_postprocess2_cli -from .glm_setup import glm_setup_cli -from .glm_l1 import glm_l1_preparefsf_cli -from .glm_l2 import glm_l2_preparefsf_cli, glm_apply_mumford_workaround_cli -from .glm_launch import glm_launch_cli -from .fsl_onset_extract import fsl_onset_extract_cli -from .outliers_report import report_outliers_cli -from .status import status_cli +from .config import * DEFAULT_HELP_PRIORITY = 5 @@ -85,23 +71,381 @@ def glm_cli(): def bids_cli(): """BIDS Commands""" +def _add_commands(): + cli.add_command(project_setup_cli, help_priority=1) + cli.add_command(fmriprep_process_cli, help_priority=3) + cli.add_command(fmri_postprocess_cli, help_priority=4) + cli.add_command(fmri_postprocess2_cli, help_priority=4) + cli.add_command(status_cli) -cli.add_command(project_setup_cli, help_priority=1) -cli.add_command(fmriprep_process_cli, help_priority=3) -cli.add_command(fmri_postprocess_cli, help_priority=4) -cli.add_command(fmri_postprocess2_cli, help_priority=4) -cli.add_command(status_cli) + bids_cli.add_command(convert2bids_cli) + bids_cli.add_command(bids_validate_cli) -bids_cli.add_command(convert2bids_cli) -bids_cli.add_command(bids_validate_cli) + glm_cli.add_command(glm_setup_cli, help_priority=1) + glm_cli.add_command(glm_l1_preparefsf_cli, help_priority=3) + glm_cli.add_command(glm_launch_cli, help_priority=4) + glm_cli.add_command(glm_l2_preparefsf_cli, help_priority=6) + glm_cli.add_command(glm_apply_mumford_workaround_cli, help_priority=5) + glm_cli.add_command(fsl_onset_extract_cli, help_priority=2) + glm_cli.add_command(report_outliers_cli, help_priority=7) -glm_cli.add_command(glm_setup_cli, help_priority=1) -glm_cli.add_command(glm_l1_preparefsf_cli, help_priority=3) -glm_cli.add_command(glm_launch_cli, help_priority=4) -glm_cli.add_command(glm_l2_preparefsf_cli, help_priority=6) -glm_cli.add_command(glm_apply_mumford_workaround_cli, help_priority=5) -glm_cli.add_command(fsl_onset_extract_cli, help_priority=2) -glm_cli.add_command(report_outliers_cli, help_priority=7) + cli.add_command(bids_cli, help_priority=2) + cli.add_command(glm_cli) -cli.add_command(bids_cli, help_priority=2) -cli.add_command(glm_cli) + +@click.command(SETUP_COMMAND_NAME) +@click.option('-project_title', required=True, default=None) +@click.option('-project_dir', required=True ,type=CLICK_DIR_TYPE_NOT_EXIST, + default=None, help=PROJECT_DIR_HELP) +@click.option('-source_data', type=CLICK_DIR_TYPE_EXISTS, + help=SOURCE_DATA_HELP) +@click.option('-move_source_data', is_flag=True, default=False, + help=MOVE_SOURCE_DATA_HELP) +@click.option('-symlink_source_data', is_flag=True, default=False, + help=SYM_LINK_HELP) +@click.option('-debug', is_flag=True, help=DEBUG_HELP) +def project_setup_cli(project_title=None, project_dir=None, source_data=None, + move_source_data=None, symlink_source_data=None, log_dir=None, + debug=False): + """Set up a clpipe project""" + from .project_setup import project_setup + project_setup( + project_title=project_title, + project_dir=project_dir, source_data=source_data, + move_source_data=move_source_data, + symlink_source_data=symlink_source_data,debug=debug) + + +@click.command(CONVERSION_COMMAND_NAME) +@click.argument('subjects', nargs=-1, required=False, default=None) +@click.option('-config_file', '-c', type=CLICK_FILE_TYPE_EXISTS, default=None, + help=CONFIG_HELP) +@click.option('-conv_config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, + help=CONVERSION_CONFIG_HELP) +@click.option('-dicom_dir', '-i', type=CLICK_DIR_TYPE_EXISTS, help=DICOM_DIR_HELP) +@click.option('-dicom_dir_format', help=DICOM_DIR_FORMAT_HELP) +@click.option('-BIDS_dir', '-o', type=CLICK_DIR_TYPE_EXISTS, + help=BIDS_DIR_HELP) +@click.option('-overwrite', is_flag=True, default=False, help=OVERWRITE_HELP) +@click.option('-clear_cache', is_flag=True, default=False, help="Clear cached data for given subject.", + hidden=True) +@click.option('-clear_outputs', is_flag=True, default=False, help="Clear all BIDS data for given subject.", + hidden=True) +@click.option('-log_dir', type=CLICK_DIR_TYPE_EXISTS, help=LOG_DIR_HELP) +@click.option('-subject', required=False, help=SUBJECT_HELP) +@click.option('-session', required=False, help=SESSION_HELP) +@click.option('-longitudinal', is_flag=True, default=False, + help=LONGITUDINAL_HELP) +@click.option('-submit', '-s', is_flag=True, default=False, help=SUBMIT_HELP) +@click.option('-batch/-no-batch', is_flag = True, default=True, + help=BATCH_HELP, hidden=True) +@click.option('-debug', '-d', is_flag=True, help=DEBUG_HELP) +@click.option('-dcm2bids/-heudiconv', default=True, help=MODE_HELP) +@click.option('-status_cache', default=None, type=CLICK_FILE_TYPE, + help=STATUS_CACHE_HELP, hidden=True) +def convert2bids_cli(dicom_dir, dicom_dir_format, bids_dir, + conv_config_file, dcm2bids, + config_file, overwrite, clear_cache, clear_outputs, + log_dir, subject, subjects, session, + longitudinal, submit, batch, debug, status_cache): + """Convert DICOM files to BIDS format""" + from .bids_conversion import convert2bids + convert2bids( + dicom_dir=dicom_dir, dicom_dir_format=dicom_dir_format, + bids_dir=bids_dir, conv_config_file=conv_config_file, + config_file=config_file, overwrite=overwrite, clear_cache=clear_cache, clear_outputs=clear_outputs, + log_dir=log_dir, batch=batch, subject=subject, subjects=subjects, session=session, + longitudinal=longitudinal, submit=submit, status_cache=status_cache, debug=debug, dcm2bids=dcm2bids) + + +@click.command(VALIDATOR_COMMAND_NAME) +@click.option('-config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, + help=CONFIG_HELP) +@click.argument('bids_dir', type=CLICK_DIR_TYPE_EXISTS, required=False) +@click.option('-log_dir', type=CLICK_FILE_TYPE_EXISTS, default=None, + help=LOG_DIR_HELP) +@click.option('-verbose', is_flag=True, default=False, + help=VERBOSE_HELP) +@click.option('-submit', is_flag=True, help=SUBMIT_HELP) +@click.option('-interactive', is_flag=True, default=False, + help=INTERACTIVE_HELP) +@click.option('-debug', is_flag=True, help=DEBUG_HELP) +def bids_validate_cli(bids_dir, config_file, log_dir, interactive, submit, + verbose, debug): + """Check that the given directory conforms to the BIDS standard""" + from .bids_validator import bids_validate + bids_validate( + bids_dir=bids_dir, config_file=config_file, log_dir=log_dir, + interactive=interactive, submit=submit, verbose=verbose, debug=debug) + + +@click.command(FMRIPREP_COMMAND_NAME) +@click.argument('subjects', nargs=-1, required=False, default=None) +@click.option('-config_file', default=None, type=CLICK_FILE_TYPE_EXISTS, + help=CONFIG_HELP) +@click.option('-bids_dir', type=CLICK_DIR_TYPE_EXISTS, + help=BIDS_DIR_HELP) +@click.option('-working_dir', type=CLICK_DIR_TYPE, + help=WORKING_DIR_HELP) +@click.option('-output_dir', type=CLICK_DIR_TYPE, + help=FMRIPREP_OUTPUT_DIR_HELP) +@click.option('-log_dir', type=CLICK_DIR_TYPE, help=LOG_DIR_HELP) +@click.option('-submit', is_flag=True, default=False, help=SUBMIT_HELP) +@click.option('-debug', is_flag=True, help=DEBUG_HELP) +@click.option('-status_cache', default=None, type=CLICK_FILE_TYPE, + help=STATUS_CACHE_HELP, hidden=True) +def fmriprep_process_cli(bids_dir, working_dir, output_dir, config_file, + subjects, log_dir, submit, debug, status_cache): + """Submit BIDS-formatted images to fMRIPrep""" + from .fmri_preprocess import fmriprep_process + fmriprep_process( + bids_dir=bids_dir, working_dir=working_dir, + output_dir=output_dir, config_file=config_file, + subjects=subjects, log_dir=log_dir, submit=submit, debug=debug, + status_cache=status_cache) + + +@click.command(POSTPROCESS_COMMAND_NAME) +@click.argument('subjects', nargs=-1, required=False, default=None) +@click.option('-config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, help = 'Use a given configuration file. If left blank, uses the default config file, requiring definition of BIDS, working and output directories.') +@click.option('-target_dir', type=click.Path(exists=True, dir_okay=True, file_okay=False), help='Which fmriprep directory to process. If a configuration file is provided with a BIDS directory, this argument is not necessary. Note, must point to the ``fmriprep`` directory, not its parent directory.') +@click.option('-target_suffix', help= 'Which file suffix to use. If a configuration file is provided with a target suffix, this argument is not necessary. Defaults to "preproc_bold.nii.gz"') +@click.option('-output_dir', type=click.Path(dir_okay=True, file_okay=False), help = 'Where to put the postprocessed data. If a configuration file is provided with a output directory, this argument is not necessary.') +@click.option('-output_suffix', help = 'What suffix to append to the postprocessed files. If a configuration file is provided with a output suffix, this argument is not necessary.') +@click.option('-task', help = 'Which task to postprocess. If left blank, defaults to all tasks.') +@click.option('-TR', help = 'The TR of the scans. If a config file is not provided, this option is required. If a config file is provided, this information is found from the sidecar jsons.') +@click.option('-processing_stream', help = 'Optional processing stream selector.') +@click.option('-log_dir', type=click.Path(dir_okay=True, file_okay=False), help = 'Where to put HPC output files. If not specified, defaults to /batchOutput.') +@click.option('-beta_series', is_flag = True, default = False, help = "Flag to activate beta-series correlation correlation. ADVANCED METHOD, refer to the documentation.") +@click.option('-submit', is_flag = True, default=False, help = 'Flag to submit commands to the HPC.') +@click.option('-batch/-single', default=True, help = 'Submit to batch, or run in current session. Mainly used internally.') +@click.option('-debug', is_flag = True, default=False, help = 'Print detailed processing information and traceback for errors.') +def fmri_postprocess_cli(config_file=None, subjects=None, target_dir=None, + target_suffix=None, output_dir=None, + output_suffix=None, log_dir=None, + submit=False, batch=True, task=None, tr=None, + processing_stream = None, debug = False, + beta_series = False): + """Additional preprocessing for connectivity analysis""" + from .fmri_postprocess import fmri_postprocess + fmri_postprocess( + config_file=config_file, subjects=subjects, target_dir=target_dir, + target_suffix=target_suffix, output_dir=output_dir, + output_suffix=output_suffix, log_dir=log_dir, submit=submit, + batch=batch, task=task, tr=tr, processing_stream=processing_stream, + debug=debug, beta_series=beta_series) + + +@click.command(POSTPROCESS2_COMMAND_NAME) +@click.argument('subjects', nargs=-1, required=False, default=None) +@click.option('-config_file', '-c', type=CLICK_FILE_TYPE_EXISTS, default=None, + required=True, help=CONFIG_HELP) +@click.option('-fmriprep_dir', '-i', type=CLICK_DIR_TYPE_EXISTS, + help=FMRIPREP_DIR_HELP) +@click.option('-output_dir', '-o', type=CLICK_DIR_TYPE, default=None, required=False, + help=OUTPUT_DIR_HELP) +@click.option('-processing_stream', '-p', default=DEFAULT_PROCESSING_STREAM_NAME, +required=False, help=PROCESSING_STREAM_HELP) +@click.option('-log_dir', '-l', type=CLICK_DIR_TYPE_EXISTS, default=None, + required=False, help=LOG_DIR_HELP) +@click.option('-index_dir', type=CLICK_DIR_TYPE, default=None, required=False, + help=INDEX_HELP) +@click.option('-refresh_index', '-r', is_flag=True, default=False, required=False, + help=REFRESH_INDEX_HELP) +@click.option('-batch/-no-batch', is_flag = True, default=True, + help=BATCH_HELP) +@click.option('-cache/-no-cache', is_flag=True, default=True) +@click.option('-submit', '-s', is_flag = True, default=False, help=SUBMIT_HELP) +@click.option('-debug', '-d', is_flag = True, default=False, help=DEBUG_HELP) +def fmri_postprocess2_cli(subjects, config_file, fmriprep_dir, output_dir, + processing_stream, batch, submit, log_dir, index_dir, + refresh_index, debug, cache): + """Additional preprocessing for GLM or connectivity analysis""" + from .fmri_postprocess2 import postprocess_subjects + postprocess_subjects( + subjects=subjects, config_file=config_file,fmriprep_dir=fmriprep_dir, + output_dir=output_dir, processing_stream=processing_stream, + batch=batch, submit=submit, log_dir=log_dir, pybids_db_path=index_dir, + refresh_index=refresh_index, debug=debug, cache=cache) + + +@click.command(GLM_SETUP_COMMAND_NAME) +@click.argument('subjects', nargs=-1, required=False, default=None) +@click.option('-config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, + help='Use a given configuration file.') +@click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, + help='Use a given GLM configuration file.') +@click.option('-drop_tps', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = False, + help='Drop timepoints csv sheet') +@click.option('-submit', is_flag=True, default=False, help='Flag to submit commands to the HPC.') +@click.option('-batch/-single', default=True, + help='Submit to batch, or run in current session. Mainly used internally.') +@click.option('-debug', is_flag=True, default=False, + help='Print detailed processing information and traceback for errors.') +def glm_setup_cli(subjects, config_file, glm_config_file, submit, batch, debug, + drop_tps): + """Additional preprocessing for GLM analysis""" + from .glm_setup import glm_setup + glm_setup( + subjects=subjects, config_file=config_file, + glm_config_file=glm_config_file, + submit=submit, batch=batch, debug=debug, drop_tps=drop_tps) + + +@click.command(L1_PREPARE_FSF_COMMAND_NAME) +@click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, + help='Use a given GLM configuration file.') +@click.option('-l1_name', default=None, required = True, + help='Name for a given L1 model') +@click.option('-debug', is_flag=True, help='Flag to enable detailed error messages and traceback') +def glm_l1_preparefsf_cli(glm_config_file, l1_name, debug): + """Propagate an .fsf file template for L1 GLM analysis""" + from .glm_l1 import glm_l1_preparefsf + glm_l1_preparefsf( + glm_config_file=glm_config_file, l1_name=l1_name, debug=debug) + + +@click.command(L2_PREPARE_FSF_COMMAND_NAME) +@click.option('-glm_config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, + required=True, help='Use a given GLM configuration file.') +@click.option('-l2_name', default=None, required=True, + help='Name for a given L2 model') +@click.option('-debug', is_flag=True, + help='Flag to enable detailed error messages and traceback') +def glm_l2_preparefsf_cli(glm_config_file, l2_name, debug): + """Propagate an .fsf file template for L2 GLM analysis""" + from .glm_l2 import glm_l2_preparefsf + glm_l2_preparefsf(glm_config_file=glm_config_file, l2_name=l2_name, + debug=debug) + + +@click.command(APPLY_MUMFORD_COMMAND_NAME) +@click.option('-glm_config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, + required=False, + help='Location of your GLM config file.') +@click.option('-l1_feat_folders_path', type=CLICK_DIR_TYPE_EXISTS, + default=None, required=False, + help='Location of your L1 FEAT folders.') +@click.option('-debug', is_flag=True, + help='Flag to enable detailed error messages and traceback') +def glm_apply_mumford_workaround_cli(glm_config_file, l1_feat_folders_path, + debug): + """ + Apply the Mumford registration workaround to L1 FEAT folders. + Applied by default in glm-l2-preparefsf. + """ + from .glm_l2 import glm_apply_mumford_workaround + if not (glm_config_file or l1_feat_folders_path): + click.echo(("Error: At least one of either option '-glm_config_file' " + "or '-l1_feat_folders_path' required.")) + glm_apply_mumford_workaround( + glm_config_file=glm_config_file, + l1_feat_folders_path=l1_feat_folders_path, debug=debug + ) + + +@click.command(GLM_LAUNCH_COMMAND_NAME) +@click.argument('level') +@click.argument('model') +@click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, + file_okay=True), default=None, required = True, + help=CONFIG_HELP) +@click.option('-test_one', is_flag=True, + help=TEST_ONE_HELP) +@click.option('-submit', is_flag=True, + help=SUBMIT_HELP) +@click.option('-debug', is_flag=True, + help=DEBUG_HELP) +def glm_launch_cli(level, model, glm_config_file, test_one, submit, debug): + """Launch all prepared .fsf files for L1 or L2 GLM analysis""" + from .glm_launch import glm_launch_controller + glm_launch_controller(glm_config_file=glm_config_file, level=level, + model=model, test_one=test_one, + submit=submit, debug=debug) + + +@click.command() +@click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, + file_okay=True), default=None, required = True, + help=CONFIG_HELP) +@click.option('-l1_name', default=None, required = True, + help=L1_MODEL_HELP) +@click.option('-test_one', is_flag=True, + help=TEST_ONE_HELP) +@click.option('-submit', is_flag=True, + help=SUBMIT_HELP) +@click.option('-debug', is_flag=True, + help=DEBUG_HELP) +def glm_l1_launch_cli(glm_config_file, l1_name, test_one, submit, debug): + """Launch all prepared .fsf files for L1 GLM analysis""" + from .glm_launch import glm_launch_controller + glm_launch_controller(glm_config_file=glm_config_file, model=l1_name, + test_one=test_one, submit=submit, debug=debug) + + +@click.command() +@click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, + file_okay=True), default=None, required = True, + help=CONFIG_HELP) +@click.option('-l2_name', default=None, required = True, + help=L2_MODEL_HELP) +@click.option('-test_one', is_flag=True, + help=TEST_ONE_HELP) +@click.option('-submit', is_flag=True, + help=SUBMIT_HELP) +@click.option('-debug', is_flag=True, + help=DEBUG_HELP) +def glm_l2_launch_cli(glm_config_file, l2_name, test_one, submit, debug): + """Launch all prepared .fsf files for L2 GLM analysis""" + from .glm_launch import glm_launch_controller + glm_launch_controller(glm_config_file=glm_config_file, level="L2", + model=l2_name, test_one=test_one, submit=submit, + debug=debug) + + +@click.command(ONSET_EXTRACT_COMMAND_NAME) +@click.option('-config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), + default=None, required = True, + help='Use a given configuration file.') +@click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), + default=None, required = True, + help='Use a given GLM configuration file.') +@click.option('-debug', is_flag=True, default=False, + help='Print detailed processing information and traceback for errors.') +def fsl_onset_extract_cli(config_file, glm_config_file, debug): + """Convert onset files to FSL's 3 column format""" + from .fsl_onset_extract import fsl_onset_extract + fsl_onset_extract( + config_file=config_file, glm_config_file=glm_config_file, debug=debug) + + +@click.command(OUTLIERS_COMMAND_NAME) +@click.option('--confounds_dir', type=click.Path(exists=True, dir_okay=True, file_okay=False), + help="Path to a directory containing subjects and confounds files.") +@click.option('--confounds_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), + help="Path to confounds file") +@click.option('--output_file', type=click.Path(dir_okay=False, file_okay=True), + help="Path to save outlier count results.") +@click.option('--confound_suffix', help="Confound file to search for, like 'confounds.tsv'", + default='confounds.tsv') +def report_outliers_cli(confounds_dir, confounds_file, output_file, + confound_suffix): + """Generate a confound outliers report.""" + from .outliers_report import get_study_outliers, get_image_confounds + if confounds_dir: + get_study_outliers(confounds_dir, output_file, confound_suffix) + else: + get_image_confounds(confounds_file) + + +@click.command(STATUS_COMMAND_NAME) +@click.option('-config_file', type=CLICK_FILE_TYPE_EXISTS, + help=CONFIG_HELP, required=False) +@click.option('-cache_file', type=CLICK_FILE_TYPE_EXISTS, + help=CACHE_FILE_HELP, required=False) +def status_cli(config_file, cache_file): + """Check the status of your project""" + from .status import show_latest_by_step + show_latest_by_step(config_file=config_file, cache_path=cache_file) + +_add_commands() diff --git a/clpipe/config.py b/clpipe/config.py index 56264e9e..b25c18d4 100644 --- a/clpipe/config.py +++ b/clpipe/config.py @@ -24,4 +24,110 @@ "compute session." ) VERSION_HELP = "Display clpipe's version." -BATCH_HELP = 'Flag to create batch jobs without prompt.' \ No newline at end of file +BATCH_HELP = 'Flag to create batch jobs without prompt.' +WORKING_DIR_HELP = ( + "Where to generate the working directory." +) +GLM_CONFIG_HELP = 'Use a given GLM configuration file.' + +SETUP_COMMAND_NAME = "setup" +PROJECT_DIR_HELP = "Where the project will be located." +SOURCE_DATA_HELP = \ + "Where the raw data (usually DICOMs) are located." +MOVE_SOURCE_DATA_HELP = \ + "Move source data into project/data_DICOMs folder. USE WITH CAUTION." +SYM_LINK_HELP = \ + "Symlink the source data into project/data_dicoms. Usually safe to do." + +CONVERSION_COMMAND_NAME = "convert" +CONVERSION_CONFIG_HELP = ( + "A dcm2bids conversion definition .json file." +) +HEURISTIC_FILE_HELP = ( + "A heudiconv heuristic file to use for the conversion." +) +DICOM_DIR_HELP = "The folder where subject dicoms are located." +DICOM_DIR_FORMAT_HELP = ( + "Format string for how subjects/sessions are organized within the " + "dicom_dir." +) +BIDS_DIR_HELP = "The dicom info output file name." +OVERWRITE_HELP = "Overwrite existing BIDS data?" +SUBJECT_HELP = ( + "DEPRECATED: specify one subject to process - can give an " + "arbitrary number of subjects as arguments now." +) +SESSION_HELP = ( + "A session to convert using the supplied configuration file. Use in " + "combination with -subject to convert single subject/sessions, " + "else leave empty" +) +LONGITUDINAL_HELP = ( + "Convert all subjects/sessions into individual pseudo-subjects. " + "Use if you do not want T1w averaged across sessions during FMRIprep" +) +MODE_HELP = ( + "Specify which converter to use." +) + +VALIDATOR_COMMAND_NAME = "validate" +VERBOSE_HELP = ( + "Creates verbose validator output. Use if you want to see ALL files " + "with errors/warnings." +) + +FMRIPREP_COMMAND_NAME = "preprocess" +FMRIPREP_CONFIG_HELP = ( + "Use a given configuration file. If left blank, uses the default config " + "file, requiring definition of BIDS, working and output directories." +) +FMRIPREP_BIDS_DIR_HELP = ( + "Which BIDS directory to process. If a configuration file is provided " + "with a BIDS directory, this argument is not necessary." +) +FMRIPREP_OUTPUT_DIR_HELP = ( + "Where to put the preprocessed data. If a configuration file is provided " + "with a output directory, this argument is not necessary." +) + +POSTPROCESS_COMMAND_NAME = "postprocess" + +POSTPROCESS2_COMMAND_NAME = "postprocess2" +FMRIPREP_DIR_HELP = ( + "Which fmriprep directory to process. " + "If a configuration file is provided with a BIDS directory, " + "this argument is not necessary. Note, must point to the ``fmriprep`` " + "directory, not its parent directory." +) +OUTPUT_DIR_HELP = ( + "Where to put the postprocessed data. If a configuration file is " + "provided with a output directory, this argument is not necessary." +) +PROCESSING_STREAM_HELP = \ + "Specify a processing stream to use defined in your configuration file." +INDEX_HELP = 'Give the path to an existing pybids index database.' +REFRESH_INDEX_HELP = \ + 'Refresh the pybids index database to reflect new fmriprep artifacts.' +DEFAULT_PROCESSING_STREAM_NAME = "smooth-filter-normalize" + +GLM_SETUP_COMMAND_NAME = "setup" + +L1_PREPARE_FSF_COMMAND_NAME = "l1_prepare_fsf" + +L2_PREPARE_FSF_COMMAND_NAME = "l2_prepare_fsf" + +APPLY_MUMFORD_COMMAND_NAME = "apply_mumford" + +GLM_LAUNCH_COMMAND_NAME = "launch" +L1_MODEL_HELP = 'Name of your L1 model' +L2_MODEL_HELP = 'Name of your L2 model' +LEVEL_HELP = "Level of your model, L1 or L2" +MODEL_HELP = 'Name of your model' +TEST_ONE_HELP = 'Only submit one job for testing purposes.' + +ONSET_EXTRACT_COMMAND_NAME = "fsl_onset_extract" + +OUTLIERS_COMMAND_NAME = "report_outliers" + +STATUS_COMMAND_NAME = "status" +CACHE_FILE_HELP = "Path to your status cache file." \ No newline at end of file diff --git a/clpipe/fmri_postprocess.py b/clpipe/fmri_postprocess.py index cc9db9c0..ac817687 100644 --- a/clpipe/fmri_postprocess.py +++ b/clpipe/fmri_postprocess.py @@ -18,39 +18,6 @@ from .config_json_parser import ClpipeConfigParser from .error_handler import exception_handler -COMMAND_NAME = "postprocess" - - -@click.command(COMMAND_NAME) -@click.argument('subjects', nargs=-1, required=False, default=None) -@click.option('-config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, help = 'Use a given configuration file. If left blank, uses the default config file, requiring definition of BIDS, working and output directories.') -@click.option('-target_dir', type=click.Path(exists=True, dir_okay=True, file_okay=False), help='Which fmriprep directory to process. If a configuration file is provided with a BIDS directory, this argument is not necessary. Note, must point to the ``fmriprep`` directory, not its parent directory.') -@click.option('-target_suffix', help= 'Which file suffix to use. If a configuration file is provided with a target suffix, this argument is not necessary. Defaults to "preproc_bold.nii.gz"') -@click.option('-output_dir', type=click.Path(dir_okay=True, file_okay=False), help = 'Where to put the postprocessed data. If a configuration file is provided with a output directory, this argument is not necessary.') -@click.option('-output_suffix', help = 'What suffix to append to the postprocessed files. If a configuration file is provided with a output suffix, this argument is not necessary.') -@click.option('-task', help = 'Which task to postprocess. If left blank, defaults to all tasks.') -@click.option('-TR', help = 'The TR of the scans. If a config file is not provided, this option is required. If a config file is provided, this information is found from the sidecar jsons.') -@click.option('-processing_stream', help = 'Optional processing stream selector.') -@click.option('-log_dir', type=click.Path(dir_okay=True, file_okay=False), help = 'Where to put HPC output files. If not specified, defaults to /batchOutput.') -@click.option('-beta_series', is_flag = True, default = False, help = "Flag to activate beta-series correlation correlation. ADVANCED METHOD, refer to the documentation.") -@click.option('-submit', is_flag = True, default=False, help = 'Flag to submit commands to the HPC.') -@click.option('-batch/-single', default=True, help = 'Submit to batch, or run in current session. Mainly used internally.') -@click.option('-debug', is_flag = True, default=False, help = 'Print detailed processing information and traceback for errors.') -def fmri_postprocess_cli(config_file=None, subjects=None, target_dir=None, - target_suffix=None, output_dir=None, - output_suffix=None, log_dir=None, - submit=False, batch=True, task=None, tr=None, - processing_stream = None, debug = False, - beta_series = False): - """Additional preprocessing for connectivity analysis""" - - fmri_postprocess( - config_file=config_file, subjects=subjects, target_dir=target_dir, - target_suffix=target_suffix, output_dir=output_dir, - output_suffix=output_suffix, log_dir=log_dir, submit=submit, - batch=batch, task=task, tr=tr, processing_stream=processing_stream, - debug=debug, beta_series=beta_series) - def fmri_postprocess(config_file=None, subjects=None, target_dir=None, target_suffix=None, output_dir=None, output_suffix=None, log_dir=None, diff --git a/clpipe/fmri_postprocess2.py b/clpipe/fmri_postprocess2.py index 0f032f8a..72a4f8f5 100644 --- a/clpipe/fmri_postprocess2.py +++ b/clpipe/fmri_postprocess2.py @@ -26,9 +26,7 @@ from bids.layout import BIDSFile from .config_json_parser import ClpipeConfigParser -from .config import BATCH_HELP, CLICK_FILE_TYPE_EXISTS, \ - CLICK_DIR_TYPE_EXISTS, CLICK_DIR_TYPE, CONFIG_HELP, LOG_DIR_HELP, \ - SUBMIT_HELP, DEBUG_HELP, BATCH_HELP +from .config import * from .batch_manager import BatchManager, Job import nipype.pipeline.engine as pe from .postprocutils.workflows import build_image_postprocessing_workflow, \ @@ -38,8 +36,6 @@ from .utils import add_file_handler, get_logger from .errors import * -COMMAND_NAME = "postprocess2" -DEFAULT_PROCESSING_STREAM_NAME = "smooth-filter-normalize" DEFAULT_GRAPH_STYLE = "colored" PROCESSING_DESCRIPTION_FILE_NAME = "processing_description.json" IMAGE_TIME_DIMENSION_INDEX = 3 @@ -55,55 +51,6 @@ "{debug}" ) -FMRIPREP_DIR_HELP = ( - "Which fmriprep directory to process. " - "If a configuration file is provided with a BIDS directory, " - "this argument is not necessary. Note, must point to the ``fmriprep`` " - "directory, not its parent directory." -) -OUTPUT_DIR_HELP = ( - "Where to put the postprocessed data. If a configuration file is " - "provided with a output directory, this argument is not necessary." -) -PROCESSING_STREAM_HELP = \ - "Specify a processing stream to use defined in your configuration file." -INDEX_HELP = 'Give the path to an existing pybids index database.' -REFRESH_INDEX_HELP = \ - 'Refresh the pybids index database to reflect new fmriprep artifacts.' - - -@click.command(COMMAND_NAME) -@click.argument('subjects', nargs=-1, required=False, default=None) -@click.option('-config_file', '-c', type=CLICK_FILE_TYPE_EXISTS, default=None, - required=True, help=CONFIG_HELP) -@click.option('-fmriprep_dir', '-i', type=CLICK_DIR_TYPE_EXISTS, - help=FMRIPREP_DIR_HELP) -@click.option('-output_dir', '-o', type=CLICK_DIR_TYPE, default=None, required=False, - help=OUTPUT_DIR_HELP) -@click.option('-processing_stream', '-p', default=DEFAULT_PROCESSING_STREAM_NAME, -required=False, help=PROCESSING_STREAM_HELP) -@click.option('-log_dir', '-l', type=CLICK_DIR_TYPE_EXISTS, default=None, - required=False, help=LOG_DIR_HELP) -@click.option('-index_dir', type=CLICK_DIR_TYPE, default=None, required=False, - help=INDEX_HELP) -@click.option('-refresh_index', '-r', is_flag=True, default=False, required=False, - help=REFRESH_INDEX_HELP) -@click.option('-batch/-no-batch', is_flag = True, default=True, - help=BATCH_HELP) -@click.option('-cache/-no-cache', is_flag=True, default=True) -@click.option('-submit', '-s', is_flag = True, default=False, help=SUBMIT_HELP) -@click.option('-debug', '-d', is_flag = True, default=False, help=DEBUG_HELP) -def fmri_postprocess2_cli(subjects, config_file, fmriprep_dir, output_dir, - processing_stream, batch, submit, log_dir, index_dir, - refresh_index, debug, cache): - """Additional preprocessing for GLM or connectivity analysis""" - - postprocess_subjects( - subjects=subjects, config_file=config_file,fmriprep_dir=fmriprep_dir, - output_dir=output_dir, processing_stream=processing_stream, - batch=batch, submit=submit, log_dir=log_dir, pybids_db_path=index_dir, - refresh_index=refresh_index, debug=debug, cache=cache) - def postprocess_subjects( subjects=None, config_file=None, bids_dir=None, fmriprep_dir=None, @@ -123,7 +70,7 @@ def postprocess_subjects( # Setup Logging clpipe_logs = project_dir / "logs" - logger = get_logger(COMMAND_NAME, debug=debug) + logger = get_logger(POSTPROCESS2_COMMAND_NAME, debug=debug) add_file_handler(clpipe_logs) if not fmriprep_dir: diff --git a/clpipe/fmri_preprocess.py b/clpipe/fmri_preprocess.py index 45ca5090..34b2b2bf 100644 --- a/clpipe/fmri_preprocess.py +++ b/clpipe/fmri_preprocess.py @@ -1,16 +1,11 @@ import os import sys -import click from .batch_manager import BatchManager, Job from .config_json_parser import ClpipeConfigParser from .utils import get_logger, add_file_handler from .status import needs_processing, write_record -from .config import LOG_DIR_HELP, SUBMIT_HELP, DEBUG_HELP, STATUS_CACHE_HELP, \ - CLICK_DIR_TYPE, CLICK_DIR_TYPE_EXISTS, CLICK_FILE_TYPE_EXISTS, \ - CLICK_FILE_TYPE -COMMAND_NAME = "preprocess" STEP_NAME = "fmriprep-process" BASE_SINGULARITY_CMD = ( "unset PYTHONPATH; {templateflow1} singularity run -B {templateflow2}" @@ -32,48 +27,6 @@ USE_AROMA_FLAG = "--use-aroma" N_THREADS_FLAG = "--nthreads" -CONFIG_HELP = ( - "Use a given configuration file. If left blank, uses the default config " - "file, requiring definition of BIDS, working and output directories." -) -BIDS_DIR_HELP = ( - "Which BIDS directory to process. If a configuration file is provided " - "with a BIDS directory, this argument is not necessary." -) -WORKING_DIR_HELP = ( - "Where to generate the working directory. If a configuration file is " - "provided with a working directory, this argument is not necessary." -) -OUTPUT_DIR_HELP = ( - "Where to put the preprocessed data. If a configuration file is provided " - "with a output directory, this argument is not necessary." -) - -@click.command(COMMAND_NAME) -@click.argument('subjects', nargs=-1, required=False, default=None) -@click.option('-config_file', default=None, type=CLICK_FILE_TYPE_EXISTS, - help=CONFIG_HELP) -@click.option('-bids_dir', type=CLICK_DIR_TYPE_EXISTS, - help=BIDS_DIR_HELP) -@click.option('-working_dir', type=CLICK_DIR_TYPE, - help=WORKING_DIR_HELP) -@click.option('-output_dir', type=CLICK_DIR_TYPE, - help=OUTPUT_DIR_HELP) -@click.option('-log_dir', type=CLICK_DIR_TYPE, help=LOG_DIR_HELP) -@click.option('-submit', is_flag=True, default=False, help=SUBMIT_HELP) -@click.option('-debug', is_flag=True, help=DEBUG_HELP) -@click.option('-status_cache', default=None, type=CLICK_FILE_TYPE, - help=STATUS_CACHE_HELP, hidden=True) -def fmriprep_process_cli(bids_dir, working_dir, output_dir, config_file, - subjects, log_dir, submit, debug, status_cache): - """Submit BIDS-formatted images to fMRIPrep""" - - fmriprep_process( - bids_dir=bids_dir, working_dir=working_dir, - output_dir=output_dir, config_file=config_file, - subjects=subjects, log_dir=log_dir, submit=submit, debug=debug, - status_cache=status_cache) - def fmriprep_process(bids_dir=None, working_dir=None, output_dir=None, config_file=None, subjects=None, log_dir=None, diff --git a/clpipe/fsl_onset_extract.py b/clpipe/fsl_onset_extract.py index 9028e297..263c86c0 100644 --- a/clpipe/fsl_onset_extract.py +++ b/clpipe/fsl_onset_extract.py @@ -4,27 +4,11 @@ import sys import numpy import pandas -import click import numpy as np from .error_handler import exception_handler from .config_json_parser import ClpipeConfigParser, GLMConfigParser -COMMAND_NAME = "fsl_onset_extract" - - -@click.command(COMMAND_NAME) -@click.option('-config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, - help='Use a given configuration file.') -@click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, - help='Use a given GLM configuration file.') -@click.option('-debug', is_flag=True, default=False, - help='Print detailed processing information and traceback for errors.') -def fsl_onset_extract_cli(config_file, glm_config_file, debug): - """Convert onset files to FSL's 3 column format""" - fsl_onset_extract( - config_file=config_file, glm_config_file=glm_config_file, debug=debug) - def fsl_onset_extract(config_file=None, glm_config_file = None, debug = None): if not debug: diff --git a/clpipe/glm_l1.py b/clpipe/glm_l1.py index c7f5605b..5e163411 100644 --- a/clpipe/glm_l1.py +++ b/clpipe/glm_l1.py @@ -12,27 +12,11 @@ import glob import logging import sys -import click import nibabel as nib from .config_json_parser import GLMConfigParser from .error_handler import exception_handler -PREPARE_FSF_COMMAND_NAME = "l1_prepare_fsf" - - -@click.command(PREPARE_FSF_COMMAND_NAME) -@click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, - help='Use a given GLM configuration file.') -@click.option('-l1_name', default=None, required = True, - help='Name for a given L1 model') -@click.option('-debug', is_flag=True, help='Flag to enable detailed error messages and traceback') -def glm_l1_preparefsf_cli(glm_config_file, l1_name, debug): - """Propagate an .fsf file template for L1 GLM analysis""" - - glm_l1_preparefsf( - glm_config_file=glm_config_file, l1_name=l1_name, debug=debug) - def glm_l1_preparefsf(glm_config_file=None, l1_name=None, debug=None): if not debug: diff --git a/clpipe/glm_l2.py b/clpipe/glm_l2.py index 20212d9f..a3105279 100644 --- a/clpipe/glm_l2.py +++ b/clpipe/glm_l2.py @@ -2,56 +2,15 @@ import logging import shutil from pathlib import Path -import click import pandas as pd +from .config import * from .config_json_parser import GLMConfigParser from .utils import get_logger -from .config import CLICK_FILE_TYPE_EXISTS, CLICK_DIR_TYPE_EXISTS - -PREPARE_FSF_COMMAND_NAME = "l2_prepare_fsf" -APPLY_MUMFORD_COMMAND_NAME = "apply_mumford" - - -@click.command(PREPARE_FSF_COMMAND_NAME) -@click.option('-glm_config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, - required=True, help='Use a given GLM configuration file.') -@click.option('-l2_name', default=None, required=True, - help='Name for a given L2 model') -@click.option('-debug', is_flag=True, - help='Flag to enable detailed error messages and traceback') -def glm_l2_preparefsf_cli(glm_config_file, l2_name, debug): - """Propagate an .fsf file template for L2 GLM analysis""" - glm_l2_preparefsf(glm_config_file=glm_config_file, l2_name=l2_name, - debug=debug) - - -@click.command(APPLY_MUMFORD_COMMAND_NAME) -@click.option('-glm_config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, - required=False, - help='Location of your GLM config file.') -@click.option('-l1_feat_folders_path', type=CLICK_DIR_TYPE_EXISTS, - default=None, required=False, - help='Location of your L1 FEAT folders.') -@click.option('-debug', is_flag=True, - help='Flag to enable detailed error messages and traceback') -def glm_apply_mumford_workaround_cli(glm_config_file, l1_feat_folders_path, - debug): - """ - Apply the Mumford registration workaround to L1 FEAT folders. - Applied by default in glm-l2-preparefsf. - """ - if not (glm_config_file or l1_feat_folders_path): - click.echo(("Error: At least one of either option '-glm_config_file' " - "or '-l1_feat_folders_path' required.")) - glm_apply_mumford_workaround( - glm_config_file=glm_config_file, - l1_feat_folders_path=l1_feat_folders_path, debug=debug - ) def glm_l2_preparefsf(glm_config_file=None, l2_name=None, debug=None): - logger = get_logger(APPLY_MUMFORD_COMMAND_NAME, debug=debug) + logger = get_logger(L1_PREPARE_FSF_COMMAND_NAME, debug=debug) glm_config = GLMConfigParser(glm_config_file) diff --git a/clpipe/glm_launch.py b/clpipe/glm_launch.py index b8b410d5..863d6d88 100644 --- a/clpipe/glm_launch.py +++ b/clpipe/glm_launch.py @@ -1,9 +1,8 @@ import os -import click import sys from pathlib import Path -from .config import DEFAULT_BATCH_CONFIG_PATH, SUBMIT_HELP, DEBUG_HELP +from .config import DEFAULT_BATCH_CONFIG_PATH from .config_json_parser import GLMConfigParser, ClpipeConfigParser from .batch_manager import BatchManager, Job from .utils import add_file_handler, get_logger @@ -16,7 +15,7 @@ DEFAULT_L2_TIME_USAGE = "5:00:00" DEFAULT_L2_N_THREADS = "4" -COMMAND_NAME = "launch" + STEP_NAME = "glm-launch" # Unset PYTHONPATH to ensure FSL uses its own internal python @@ -28,72 +27,6 @@ L1 = VALID_L1[1] L2 = VALID_L2[1] -CONFIG_HELP = 'Use a given GLM configuration file.' -L1_MODEL_HELP = 'Name of your L1 model' -L2_MODEL_HELP = 'Name of your L2 model' -LEVEL_HELP = "Level of your model, L1 or L2" -MODEL_HELP = 'Name of your model' -TEST_ONE_HELP = 'Only submit one job for testing purposes.' - - -@click.command(COMMAND_NAME) -@click.argument('level') -@click.argument('model') -@click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, - file_okay=True), default=None, required = True, - help=CONFIG_HELP) -@click.option('-test_one', is_flag=True, - help=TEST_ONE_HELP) -@click.option('-submit', is_flag=True, - help=SUBMIT_HELP) -@click.option('-debug', is_flag=True, - help=DEBUG_HELP) -def glm_launch_cli(level, model, glm_config_file, test_one, submit, debug): - """Launch all prepared .fsf files for L1 or L2 GLM analysis""" - - glm_launch_controller(glm_config_file=glm_config_file, level=level, - model=model, test_one=test_one, - submit=submit, debug=debug) - - -@click.command() -@click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, - file_okay=True), default=None, required = True, - help=CONFIG_HELP) -@click.option('-l1_name', default=None, required = True, - help=L1_MODEL_HELP) -@click.option('-test_one', is_flag=True, - help=TEST_ONE_HELP) -@click.option('-submit', is_flag=True, - help=SUBMIT_HELP) -@click.option('-debug', is_flag=True, - help=DEBUG_HELP) -def glm_l1_launch_cli(glm_config_file, l1_name, test_one, submit, debug): - """Launch all prepared .fsf files for L1 GLM analysis""" - - glm_launch_controller(glm_config_file=glm_config_file, model=l1_name, - test_one=test_one, submit=submit, debug=debug) - - -@click.command() -@click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, - file_okay=True), default=None, required = True, - help=CONFIG_HELP) -@click.option('-l2_name', default=None, required = True, - help=L2_MODEL_HELP) -@click.option('-test_one', is_flag=True, - help=TEST_ONE_HELP) -@click.option('-submit', is_flag=True, - help=SUBMIT_HELP) -@click.option('-debug', is_flag=True, - help=DEBUG_HELP) -def glm_l2_launch_cli(glm_config_file, l2_name, test_one, submit, debug): - """Launch all prepared .fsf files for L2 GLM analysis""" - - glm_launch_controller(glm_config_file=glm_config_file, level="L2", - model=l2_name, test_one=test_one, submit=submit, - debug=debug) - def glm_launch_controller(glm_config_file: str=None, level: int=L1, model: str=None, test_one: bool=False, diff --git a/clpipe/glm_setup.py b/clpipe/glm_setup.py index 5879ca7d..d8ca0fa1 100644 --- a/clpipe/glm_setup.py +++ b/clpipe/glm_setup.py @@ -20,31 +20,6 @@ import clpipe.postprocutils import numpy as np -COMMAND_NAME = "setup" - - -@click.command(COMMAND_NAME) -@click.argument('subjects', nargs=-1, required=False, default=None) -@click.option('-config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, - help='Use a given configuration file.') -@click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, - help='Use a given GLM configuration file.') -@click.option('-drop_tps', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = False, - help='Drop timepoints csv sheet') -@click.option('-submit', is_flag=True, default=False, help='Flag to submit commands to the HPC.') -@click.option('-batch/-single', default=True, - help='Submit to batch, or run in current session. Mainly used internally.') -@click.option('-debug', is_flag=True, default=False, - help='Print detailed processing information and traceback for errors.') -def glm_setup_cli(subjects, config_file, glm_config_file, submit, batch, debug, - drop_tps): - """Additional preprocessing for GLM analysis""" - - glm_setup( - subjects=subjects, config_file=config_file, - glm_config_file=glm_config_file, - submit=submit, batch=batch, debug=debug, drop_tps=drop_tps) - def glm_setup(subjects = None, config_file=None, glm_config_file = None, submit=False, batch=True, debug = None, drop_tps = None): diff --git a/clpipe/outliers_report.py b/clpipe/outliers_report.py index 4c362183..6f974c12 100644 --- a/clpipe/outliers_report.py +++ b/clpipe/outliers_report.py @@ -1,28 +1,10 @@ # Reports how many columns in a given confounds file are outliers. import logging -import click import pandas as pd from pathlib import Path from multiprocessing import Pool -COMMAND_NAME = "report_outliers" - - -@click.command(COMMAND_NAME) -@click.option('--confounds_dir', type=click.Path(exists=True, dir_okay=True, file_okay=False), help="Path to a directory containing subjects and confounds files.") -@click.option('--confounds_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), help="Path to confounds file") -@click.option('--output_file', type=click.Path(dir_okay=False, file_okay=True), help="Path to save outlier count results.") -@click.option('--confound_suffix', help="Confound file to search for, like 'confounds.tsv'", default='confounds.tsv') -def report_outliers_cli(confounds_dir, confounds_file, output_file, - confound_suffix): - """Generate a confound outliers report.""" - - if confounds_dir: - get_study_outliers(confounds_dir, output_file, confound_suffix) - else: - get_image_confounds(confounds_file) - def get_image_confounds(confounds_file): confound_file_stem = Path(confounds_file).stem diff --git a/clpipe/project_setup.py b/clpipe/project_setup.py index 4fedae3b..d0320424 100644 --- a/clpipe/project_setup.py +++ b/clpipe/project_setup.py @@ -4,47 +4,13 @@ from pkg_resources import resource_stream import json -from .config import DEFAULT_CONFIG_PATH, DEFAULT_CONFIG_FILE_NAME, \ - CLICK_DIR_TYPE_NOT_EXIST, CLICK_DIR_TYPE_EXISTS, LOG_DIR_HELP, DEBUG_HELP +from .config import DEFAULT_CONFIG_PATH, DEFAULT_CONFIG_FILE_NAME from .utils import get_logger, add_file_handler STEP_NAME = "project-setup" -COMMAND_NAME = "setup" DEFAULT_DICOM_DIR = 'data_DICOMs' DCM2BIDS_SCAFFOLD_TEMPLATE = 'dcm2bids_scaffold -o {}' -PROJECT_DIR_HELP = "Where the project will be located." -SOURCE_DATA_HELP = \ - "Where the raw data (usually DICOMs) are located." -MOVE_SOURCE_DATA_HELP = \ - "Move source data into project/data_DICOMs folder. USE WITH CAUTION." -SYM_LINK_HELP = \ - "Symlink the source data into project/data_dicoms. Usually safe to do." - - -@click.command(COMMAND_NAME) -@click.option('-project_title', required=True, default=None) -@click.option('-project_dir', required=True ,type=CLICK_DIR_TYPE_NOT_EXIST, - default=None, help=PROJECT_DIR_HELP) -@click.option('-source_data', type=CLICK_DIR_TYPE_EXISTS, - help=SOURCE_DATA_HELP) -@click.option('-move_source_data', is_flag=True, default=False, - help=MOVE_SOURCE_DATA_HELP) -@click.option('-symlink_source_data', is_flag=True, default=False, - help=SYM_LINK_HELP) -@click.option('-debug', is_flag=True, help=DEBUG_HELP) - -def project_setup_cli(project_title=None, project_dir=None, source_data=None, - move_source_data=None, symlink_source_data=None, log_dir=None, - debug=False): - """Set up a clpipe project""" - - project_setup( - project_title=project_title, - project_dir=project_dir, source_data=source_data, - move_source_data=move_source_data, - symlink_source_data=symlink_source_data,debug=debug) - def project_setup(project_title=None, project_dir=None, source_data=None, move_source_data=None, diff --git a/clpipe/status.py b/clpipe/status.py index 378375a9..ed52b61a 100644 --- a/clpipe/status.py +++ b/clpipe/status.py @@ -24,20 +24,6 @@ DEFAULT_STEP = STEPS[1] DEFAULT_EVENT = "submitted" DEFAULT_NOTE = "clpipe generated" -CACHE_FILE_HELP = "Path to your status cache file." - -COMMAND_NAME = "status" - - -#TODO: Move cli commands back to specific modules to avoid circular import here -@click.command(COMMAND_NAME) -@click.option('-config_file', type=CLICK_FILE_TYPE_EXISTS, - help=CONFIG_HELP, required=False) -@click.option('-cache_file', type=CLICK_FILE_TYPE_EXISTS, - help=CACHE_FILE_HELP, required=False) -def status_cli(config_file, cache_file): - """Check the status of your project""" - show_latest_by_step(config_file=config_file, cache_path=cache_file) def _load_records(cache_path: os.PathLike) -> pd.DataFrame: From 3f61d2b4e79e6edf56c67ba2ceef52fe20a73363 Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Thu, 17 Nov 2022 15:27:20 -0500 Subject: [PATCH 03/16] Remap old CLI commands to cli module. --- setup.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index f3f20a10..496fe04b 100644 --- a/setup.py +++ b/setup.py @@ -33,20 +33,20 @@ entry_points=''' [console_scripts] clpipe=clpipe.cli:cli - project_setup=clpipe.project_setup:project_setup_cli - convert2bids=clpipe.bids_conversion:convert2bids_cli - bids_validate=clpipe.bids_validator:bids_validate_cli - fmriprep_process=clpipe.fmri_preprocess:fmriprep_process_cli - fmri_postprocess=clpipe.fmri_postprocess:fmri_postprocess_cli - fmri_postprocess2=clpipe.fmri_postprocess2:fmri_postprocess2_cli + project_setup=clpipe.cli:project_setup_cli + convert2bids=clpipe.cli:convert2bids_cli + bids_validate=clpipe.cli:bids_validate_cli + fmriprep_process=clpipe.cli:fmriprep_process_cli + fmri_postprocess=clpipe.cli:fmri_postprocess_cli + fmri_postprocess2=clpipe.cli:fmri_postprocess2_cli postprocess_subject=clpipe.fmri_postprocess2:postprocess_subject_cli postprocess_image=clpipe.fmri_postprocess2:postprocess_image_cli - glm_setup=clpipe.glm_setup:glm_setup_cli - glm_l1_preparefsf=clpipe.glm_l1:glm_l1_preparefsf_cli - glm_l1_launch=clpipe.glm_launch:glm_l1_launch_cli - glm_l2_preparefsf=clpipe.glm_l2:glm_l2_preparefsf_cli - glm_l2_launch=clpipe.glm_launch:glm_l2_launch_cli - fsl_onset_extract=clpipe.fsl_onset_extract:fsl_onset_extract_cli + glm_setup=clpipe.cli:glm_setup_cli + glm_l1_preparefsf=clpipe.cli:glm_l1_preparefsf_cli + glm_l1_launch=clpipe.cli:glm_l1_launch_cli + glm_l2_preparefsf=clpipe.cli:glm_l2_preparefsf_cli + glm_l2_launch=clpipe.cli:glm_l2_launch_cli + fsl_onset_extract=clpipe.cli:fsl_onset_extract_cli fmri_process_check=clpipe.fmri_process_check:fmri_process_check get_reports=clpipe.get_reports:get_reports get_config_file=clpipe.grab_config_file:get_config_file From 857a99f549479841c55dae442c51387f9185b161 Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Thu, 17 Nov 2022 16:04:19 -0500 Subject: [PATCH 04/16] Add "no_args_is_help" flag to commands that have non-optional arguments (aside from -config_file). This will show a help message when no arguments are given. --- clpipe/cli.py | 20 ++++++++++---------- clpipe/glm_launch.py | 1 - 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/clpipe/cli.py b/clpipe/cli.py index 4b349a62..275b8465 100644 --- a/clpipe/cli.py +++ b/clpipe/cli.py @@ -93,7 +93,7 @@ def _add_commands(): cli.add_command(glm_cli) -@click.command(SETUP_COMMAND_NAME) +@click.command(SETUP_COMMAND_NAME, no_args_is_help=True) @click.option('-project_title', required=True, default=None) @click.option('-project_dir', required=True ,type=CLICK_DIR_TYPE_NOT_EXIST, default=None, help=PROJECT_DIR_HELP) @@ -269,7 +269,7 @@ def fmri_postprocess2_cli(subjects, config_file, fmriprep_dir, output_dir, refresh_index=refresh_index, debug=debug, cache=cache) -@click.command(GLM_SETUP_COMMAND_NAME) +@click.command(GLM_SETUP_COMMAND_NAME, no_args_is_help=True) @click.argument('subjects', nargs=-1, required=False, default=None) @click.option('-config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, help='Use a given configuration file.') @@ -292,7 +292,7 @@ def glm_setup_cli(subjects, config_file, glm_config_file, submit, batch, debug, submit=submit, batch=batch, debug=debug, drop_tps=drop_tps) -@click.command(L1_PREPARE_FSF_COMMAND_NAME) +@click.command(L1_PREPARE_FSF_COMMAND_NAME, no_args_is_help=True) @click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, help='Use a given GLM configuration file.') @click.option('-l1_name', default=None, required = True, @@ -305,7 +305,7 @@ def glm_l1_preparefsf_cli(glm_config_file, l1_name, debug): glm_config_file=glm_config_file, l1_name=l1_name, debug=debug) -@click.command(L2_PREPARE_FSF_COMMAND_NAME) +@click.command(L2_PREPARE_FSF_COMMAND_NAME, no_args_is_help=True) @click.option('-glm_config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, required=True, help='Use a given GLM configuration file.') @click.option('-l2_name', default=None, required=True, @@ -319,7 +319,7 @@ def glm_l2_preparefsf_cli(glm_config_file, l2_name, debug): debug=debug) -@click.command(APPLY_MUMFORD_COMMAND_NAME) +@click.command(APPLY_MUMFORD_COMMAND_NAME, no_args_is_help=True) @click.option('-glm_config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, required=False, help='Location of your GLM config file.') @@ -344,7 +344,7 @@ def glm_apply_mumford_workaround_cli(glm_config_file, l1_feat_folders_path, ) -@click.command(GLM_LAUNCH_COMMAND_NAME) +@click.command(GLM_LAUNCH_COMMAND_NAME, no_args_is_help=True) @click.argument('level') @click.argument('model') @click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, @@ -364,7 +364,7 @@ def glm_launch_cli(level, model, glm_config_file, test_one, submit, debug): submit=submit, debug=debug) -@click.command() +@click.command(no_args_is_help=True) @click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, help=CONFIG_HELP) @@ -383,7 +383,7 @@ def glm_l1_launch_cli(glm_config_file, l1_name, test_one, submit, debug): test_one=test_one, submit=submit, debug=debug) -@click.command() +@click.command(no_args_is_help=True) @click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, help=CONFIG_HELP) @@ -403,7 +403,7 @@ def glm_l2_launch_cli(glm_config_file, l2_name, test_one, submit, debug): debug=debug) -@click.command(ONSET_EXTRACT_COMMAND_NAME) +@click.command(ONSET_EXTRACT_COMMAND_NAME, no_args_is_help=True) @click.option('-config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, help='Use a given configuration file.') @@ -419,7 +419,7 @@ def fsl_onset_extract_cli(config_file, glm_config_file, debug): config_file=config_file, glm_config_file=glm_config_file, debug=debug) -@click.command(OUTLIERS_COMMAND_NAME) +@click.command(OUTLIERS_COMMAND_NAME, no_args_is_help=True) @click.option('--confounds_dir', type=click.Path(exists=True, dir_okay=True, file_okay=False), help="Path to a directory containing subjects and confounds files.") @click.option('--confounds_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), diff --git a/clpipe/glm_launch.py b/clpipe/glm_launch.py index 863d6d88..d5ff0070 100644 --- a/clpipe/glm_launch.py +++ b/clpipe/glm_launch.py @@ -15,7 +15,6 @@ DEFAULT_L2_TIME_USAGE = "5:00:00" DEFAULT_L2_N_THREADS = "4" - STEP_NAME = "glm-launch" # Unset PYTHONPATH to ensure FSL uses its own internal python From 549cbe933c37603f3517cde0be3624b06b4ede4b Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Thu, 17 Nov 2022 16:10:44 -0500 Subject: [PATCH 05/16] Add -h and -help as alternative help commands. -help puts the help command in line with all other options of clpipe. --- clpipe/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clpipe/cli.py b/clpipe/cli.py index 275b8465..3b75f132 100644 --- a/clpipe/cli.py +++ b/clpipe/cli.py @@ -6,6 +6,8 @@ DEFAULT_HELP_PRIORITY = 5 +CONTEXT_SETTINGS = dict(help_option_names=['-h', '-help', '--help']) + class OrderedHelpGroup(click.Group): """ @@ -44,7 +46,7 @@ def add_command(self, cmd: click.Command, name: str = None, return super().add_command(cmd, name) -@click.group(cls=OrderedHelpGroup, invoke_without_command=True) +@click.group(cls=OrderedHelpGroup, context_settings=CONTEXT_SETTINGS) @click.pass_context @click.option("-v", "--version", is_flag=True, default=False, help=VERSION_HELP) From 44498e660324f1a0325d8ba709a26d8cded254c8 Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Thu, 17 Nov 2022 16:14:06 -0500 Subject: [PATCH 06/16] Add config comments. --- clpipe/config.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/clpipe/config.py b/clpipe/config.py index b25c18d4..fbcaf57c 100644 --- a/clpipe/config.py +++ b/clpipe/config.py @@ -2,6 +2,7 @@ APPLICATION_NAME = "clpipe" +# Click path validation types CLICK_FILE_TYPE = click.Path(dir_okay=False, file_okay=True) CLICK_FILE_TYPE_EXISTS = click.Path( exists=True, dir_okay=False, file_okay=True) @@ -10,10 +11,12 @@ CLICK_DIR_TYPE_NOT_EXIST = click.Path( exists=False, dir_okay=True, file_okay=False) +# Default paths DEFAULT_BATCH_CONFIG_PATH = "slurmUNCConfig.json" DEFAULT_CONFIG_PATH = "data/defaultConvConfig.json" DEFAULT_CONFIG_FILE_NAME = 'clpipe_config.json' +# Global use help messages CONFIG_HELP = "Uses a given configuration file" LOG_DIR_HELP = "Where to put HPC output files (such as SLURM output files)" SUBMIT_HELP = "Flag to submit commands to the HPC" @@ -30,6 +33,7 @@ ) GLM_CONFIG_HELP = 'Use a given GLM configuration file.' +# Project setup help SETUP_COMMAND_NAME = "setup" PROJECT_DIR_HELP = "Where the project will be located." SOURCE_DATA_HELP = \ @@ -39,6 +43,7 @@ SYM_LINK_HELP = \ "Symlink the source data into project/data_dicoms. Usually safe to do." +# BIDS conversion help CONVERSION_COMMAND_NAME = "convert" CONVERSION_CONFIG_HELP = ( "A dcm2bids conversion definition .json file." @@ -70,12 +75,14 @@ "Specify which converter to use." ) +# BIDS Validation Help VALIDATOR_COMMAND_NAME = "validate" VERBOSE_HELP = ( "Creates verbose validator output. Use if you want to see ALL files " "with errors/warnings." ) +# FMRIPrep help FMRIPREP_COMMAND_NAME = "preprocess" FMRIPREP_CONFIG_HELP = ( "Use a given configuration file. If left blank, uses the default config " @@ -90,8 +97,10 @@ "with a output directory, this argument is not necessary." ) +# Postprocess help POSTPROCESS_COMMAND_NAME = "postprocess" +# Postprocess2 help POSTPROCESS2_COMMAND_NAME = "postprocess2" FMRIPREP_DIR_HELP = ( "Which fmriprep directory to process. " @@ -110,14 +119,15 @@ 'Refresh the pybids index database to reflect new fmriprep artifacts.' DEFAULT_PROCESSING_STREAM_NAME = "smooth-filter-normalize" +# GLM Help GLM_SETUP_COMMAND_NAME = "setup" - L1_PREPARE_FSF_COMMAND_NAME = "l1_prepare_fsf" - L2_PREPARE_FSF_COMMAND_NAME = "l2_prepare_fsf" - APPLY_MUMFORD_COMMAND_NAME = "apply_mumford" +ONSET_EXTRACT_COMMAND_NAME = "fsl_onset_extract" +OUTLIERS_COMMAND_NAME = "report_outliers" +# GLM Launch Help GLM_LAUNCH_COMMAND_NAME = "launch" L1_MODEL_HELP = 'Name of your L1 model' L2_MODEL_HELP = 'Name of your L2 model' @@ -125,9 +135,6 @@ MODEL_HELP = 'Name of your model' TEST_ONE_HELP = 'Only submit one job for testing purposes.' -ONSET_EXTRACT_COMMAND_NAME = "fsl_onset_extract" - -OUTLIERS_COMMAND_NAME = "report_outliers" - +# Other Help STATUS_COMMAND_NAME = "status" CACHE_FILE_HELP = "Path to your status cache file." \ No newline at end of file From d062917d41affdc1710cb688f4aa608eed2254f0 Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Thu, 17 Nov 2022 16:29:57 -0500 Subject: [PATCH 07/16] Improve help messages. --- clpipe/config.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/clpipe/config.py b/clpipe/config.py index fbcaf57c..680353ac 100644 --- a/clpipe/config.py +++ b/clpipe/config.py @@ -17,21 +17,21 @@ DEFAULT_CONFIG_FILE_NAME = 'clpipe_config.json' # Global use help messages -CONFIG_HELP = "Uses a given configuration file" -LOG_DIR_HELP = "Where to put HPC output files (such as SLURM output files)" -SUBMIT_HELP = "Flag to submit commands to the HPC" -DEBUG_HELP = "Flag to enable detailed error messages and traceback" +CONFIG_HELP = "The path to your clpipe configuration file." +LOG_DIR_HELP = "Where to put your HPC output files (such as SLURM output files)." +SUBMIT_HELP = "Flag to submit commands to the HPC." +DEBUG_HELP = "Flag to enable detailed error messages and traceback." STATUS_CACHE_HELP = "Path to a status cache file for pipeline automation." INTERACTIVE_HELP = ( - "Run in an interactive session. Only use in an interactive " + "Run in interactive mode. Only use in an interactive " "compute session." ) VERSION_HELP = "Display clpipe's version." -BATCH_HELP = 'Flag to create batch jobs without prompt.' +BATCH_HELP = 'Flag to create batch jobs without prompting.' WORKING_DIR_HELP = ( "Where to generate the working directory." ) -GLM_CONFIG_HELP = 'Use a given GLM configuration file.' +GLM_CONFIG_HELP = 'The path to your GLM configuration file.' # Project setup help SETUP_COMMAND_NAME = "setup" @@ -46,10 +46,8 @@ # BIDS conversion help CONVERSION_COMMAND_NAME = "convert" CONVERSION_CONFIG_HELP = ( - "A dcm2bids conversion definition .json file." -) -HEURISTIC_FILE_HELP = ( - "A heudiconv heuristic file to use for the conversion." + "A conversion definition file, either a dcm2bids conversion config .json " + "file or a heudiconv heuristic .py file." ) DICOM_DIR_HELP = "The folder where subject dicoms are located." DICOM_DIR_FORMAT_HELP = ( @@ -63,9 +61,7 @@ "arbitrary number of subjects as arguments now." ) SESSION_HELP = ( - "A session to convert using the supplied configuration file. Use in " - "combination with -subject to convert single subject/sessions, " - "else leave empty" + "Specify the session to convert." ) LONGITUDINAL_HELP = ( "Convert all subjects/sessions into individual pseudo-subjects. " @@ -84,10 +80,6 @@ # FMRIPrep help FMRIPREP_COMMAND_NAME = "preprocess" -FMRIPREP_CONFIG_HELP = ( - "Use a given configuration file. If left blank, uses the default config " - "file, requiring definition of BIDS, working and output directories." -) FMRIPREP_BIDS_DIR_HELP = ( "Which BIDS directory to process. If a configuration file is provided " "with a BIDS directory, this argument is not necessary." From 8f13d683987a937788842778031107ed2ab9b739 Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Thu, 17 Nov 2022 17:22:16 -0500 Subject: [PATCH 08/16] Expand the main CLI command's help information. --- clpipe/cli.py | 112 ++++++++++++++++++++++++++++++++++--------- clpipe/config.py | 8 ++-- clpipe/glm_launch.py | 4 +- 3 files changed, 96 insertions(+), 28 deletions(-) diff --git a/clpipe/cli.py b/clpipe/cli.py index 3b75f132..886505f6 100644 --- a/clpipe/cli.py +++ b/clpipe/cli.py @@ -51,7 +51,16 @@ def add_command(self, cmd: click.Command, name: str = None, @click.option("-v", "--version", is_flag=True, default=False, help=VERSION_HELP) def cli(ctx, version): - """Welcome to clpipe. Please choose a processing command.""" + """Welcome to clpipe. Please choose a processing command. + + You can get more help about about any of the below commands by running them + with the '-h' flag: + + > clpipe setup -h + + If you're not sure where to begin, please see the documentation at: + https://clpipe.readthedocs.io/en/latest/index.html + """ if ctx.invoked_subcommand is None: if version: @@ -107,9 +116,9 @@ def _add_commands(): help=SYM_LINK_HELP) @click.option('-debug', is_flag=True, help=DEBUG_HELP) def project_setup_cli(project_title=None, project_dir=None, source_data=None, - move_source_data=None, symlink_source_data=None, log_dir=None, + move_source_data=None, symlink_source_data=None, debug=False): - """Set up a clpipe project""" + """Set up a clpipe project.""" from .project_setup import project_setup project_setup( project_title=project_title, @@ -150,7 +159,15 @@ def convert2bids_cli(dicom_dir, dicom_dir_format, bids_dir, config_file, overwrite, clear_cache, clear_outputs, log_dir, subject, subjects, session, longitudinal, submit, batch, debug, status_cache): - """Convert DICOM files to BIDS format""" + """Convert DICOM files to BIDS format. + + Providing no SUBJECTS will default to all subjects. + List subject IDs in SUBJECTS to process specific subjects: + + > clpipe bids convert 123 124 125 ... + + Available subject IDs are determined by the dicom_dir_format string. + """ from .bids_conversion import convert2bids convert2bids( dicom_dir=dicom_dir, dicom_dir_format=dicom_dir_format, @@ -161,9 +178,9 @@ def convert2bids_cli(dicom_dir, dicom_dir_format, bids_dir, @click.command(VALIDATOR_COMMAND_NAME) +@click.argument('bids_dir', type=CLICK_DIR_TYPE_EXISTS, required=False) @click.option('-config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, help=CONFIG_HELP) -@click.argument('bids_dir', type=CLICK_DIR_TYPE_EXISTS, required=False) @click.option('-log_dir', type=CLICK_FILE_TYPE_EXISTS, default=None, help=LOG_DIR_HELP) @click.option('-verbose', is_flag=True, default=False, @@ -174,7 +191,13 @@ def convert2bids_cli(dicom_dir, dicom_dir_format, bids_dir, @click.option('-debug', is_flag=True, help=DEBUG_HELP) def bids_validate_cli(bids_dir, config_file, log_dir, interactive, submit, verbose, debug): - """Check that the given directory conforms to the BIDS standard""" + """Validate if a directory BIDS standard. + + Validates the directory at BIDS_DIR, or at the BIDS directory + in your config file's DICOMToBIDSOptions if -config_file is given. + + Results are viewable in logs/bids_validation_logs unless -interactive is used. + """ from .bids_validator import bids_validate bids_validate( bids_dir=bids_dir, config_file=config_file, log_dir=log_dir, @@ -198,7 +221,13 @@ def bids_validate_cli(bids_dir, config_file, log_dir, interactive, submit, help=STATUS_CACHE_HELP, hidden=True) def fmriprep_process_cli(bids_dir, working_dir, output_dir, config_file, subjects, log_dir, submit, debug, status_cache): - """Submit BIDS-formatted images to fMRIPrep""" + """Submit BIDS-formatted images to fMRIPrep. + + Providing no SUBJECTS will default to all subjects. + List subject IDs in SUBJECTS to process specific subjects: + + > clpipe preprocess 123 124 125 ... + """ from .fmri_preprocess import fmriprep_process fmriprep_process( bids_dir=bids_dir, working_dir=working_dir, @@ -228,7 +257,13 @@ def fmri_postprocess_cli(config_file=None, subjects=None, target_dir=None, submit=False, batch=True, task=None, tr=None, processing_stream = None, debug = False, beta_series = False): - """Additional preprocessing for connectivity analysis""" + """Additional processing for connectivity analysis. + + Providing no SUBJECTS will default to all subjects. + List subject IDs in SUBJECTS to process specific subjects: + + > clpipe postprocess 123 124 125 ... + """ from .fmri_postprocess import fmri_postprocess fmri_postprocess( config_file=config_file, subjects=subjects, target_dir=target_dir, @@ -262,7 +297,13 @@ def fmri_postprocess_cli(config_file=None, subjects=None, target_dir=None, def fmri_postprocess2_cli(subjects, config_file, fmriprep_dir, output_dir, processing_stream, batch, submit, log_dir, index_dir, refresh_index, debug, cache): - """Additional preprocessing for GLM or connectivity analysis""" + """Additional processing for GLM or connectivity analysis. + + Providing no SUBJECTS will default to all subjects. + List subject IDs in SUBJECTS to process specific subjects: + + > clpipe postprocess2 123 124 125 ... + """ from .fmri_postprocess2 import postprocess_subjects postprocess_subjects( subjects=subjects, config_file=config_file,fmriprep_dir=fmriprep_dir, @@ -286,7 +327,13 @@ def fmri_postprocess2_cli(subjects, config_file, fmriprep_dir, output_dir, help='Print detailed processing information and traceback for errors.') def glm_setup_cli(subjects, config_file, glm_config_file, submit, batch, debug, drop_tps): - """Additional preprocessing for GLM analysis""" + """Additional preprocessing for GLM analysis. + + Providing no SUBJECTS will default to all subjects. + List subject IDs in SUBJECTS to process specific subjects: + + > clpipe glm setup 123 124 125 ... + """ from .glm_setup import glm_setup glm_setup( subjects=subjects, config_file=config_file, @@ -296,12 +343,15 @@ def glm_setup_cli(subjects, config_file, glm_config_file, submit, batch, debug, @click.command(L1_PREPARE_FSF_COMMAND_NAME, no_args_is_help=True) @click.option('-glm_config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, required = True, - help='Use a given GLM configuration file.') + help='Your GLM configuration file.') @click.option('-l1_name', default=None, required = True, - help='Name for a given L1 model') + help='Name for a given L1 model as defined in your GLM configuration file.') @click.option('-debug', is_flag=True, help='Flag to enable detailed error messages and traceback') def glm_l1_preparefsf_cli(glm_config_file, l1_name, debug): - """Propagate an .fsf file template for L1 GLM analysis""" + """Propagate an .fsf file template for L1 GLM analysis. + + You must create a template .fsf file in FSL's FEAT GUI first. + """ from .glm_l1 import glm_l1_preparefsf glm_l1_preparefsf( glm_config_file=glm_config_file, l1_name=l1_name, debug=debug) @@ -309,13 +359,16 @@ def glm_l1_preparefsf_cli(glm_config_file, l1_name, debug): @click.command(L2_PREPARE_FSF_COMMAND_NAME, no_args_is_help=True) @click.option('-glm_config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, - required=True, help='Use a given GLM configuration file.') + required=True, help='Your GLM configuration file.') @click.option('-l2_name', default=None, required=True, help='Name for a given L2 model') @click.option('-debug', is_flag=True, help='Flag to enable detailed error messages and traceback') def glm_l2_preparefsf_cli(glm_config_file, l2_name, debug): - """Propagate an .fsf file template for L2 GLM analysis""" + """Propagate an .fsf file template for L2 GLM analysis. + + You must create a group-level template .fsf file in FSL's FEAT GUI first. + """ from .glm_l2 import glm_l2_preparefsf glm_l2_preparefsf(glm_config_file=glm_config_file, l2_name=l2_name, debug=debug) @@ -324,7 +377,7 @@ def glm_l2_preparefsf_cli(glm_config_file, l2_name, debug): @click.command(APPLY_MUMFORD_COMMAND_NAME, no_args_is_help=True) @click.option('-glm_config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, required=False, - help='Location of your GLM config file.') + help='Your GLM configuration file.') @click.option('-l1_feat_folders_path', type=CLICK_DIR_TYPE_EXISTS, default=None, required=False, help='Location of your L1 FEAT folders.') @@ -334,7 +387,10 @@ def glm_apply_mumford_workaround_cli(glm_config_file, l1_feat_folders_path, debug): """ Apply the Mumford registration workaround to L1 FEAT folders. - Applied by default in glm-l2-preparefsf. + + Applied by default in glm-l2-preparefsf. This command is useful for applying + the Mumford workaround to single-run subjects who skip L2, allowing you to still + combine them with multiple-run subjects at L3. """ from .glm_l2 import glm_apply_mumford_workaround if not (glm_config_file or l1_feat_folders_path): @@ -359,7 +415,14 @@ def glm_apply_mumford_workaround_cli(glm_config_file, l1_feat_folders_path, @click.option('-debug', is_flag=True, help=DEBUG_HELP) def glm_launch_cli(level, model, glm_config_file, test_one, submit, debug): - """Launch all prepared .fsf files for L1 or L2 GLM analysis""" + """Launch all prepared .fsf files for L1 or L2 GLM analysis. + + L1 can be any of: 1, L1, l1, or level1. + + L2 can be any of: 2, L2, l2, or level2. + + MODEL must be a a corresponding L1 or L2 model from your GLM configuration file. + """ from .glm_launch import glm_launch_controller glm_launch_controller(glm_config_file=glm_config_file, level=level, model=model, test_one=test_one, @@ -379,7 +442,7 @@ def glm_launch_cli(level, model, glm_config_file, test_one, submit, debug): @click.option('-debug', is_flag=True, help=DEBUG_HELP) def glm_l1_launch_cli(glm_config_file, l1_name, test_one, submit, debug): - """Launch all prepared .fsf files for L1 GLM analysis""" + """Launch all prepared .fsf files for L1 GLM analysis.""" from .glm_launch import glm_launch_controller glm_launch_controller(glm_config_file=glm_config_file, model=l1_name, test_one=test_one, submit=submit, debug=debug) @@ -398,7 +461,7 @@ def glm_l1_launch_cli(glm_config_file, l1_name, test_one, submit, debug): @click.option('-debug', is_flag=True, help=DEBUG_HELP) def glm_l2_launch_cli(glm_config_file, l2_name, test_one, submit, debug): - """Launch all prepared .fsf files for L2 GLM analysis""" + """Launch all prepared .fsf files for L2 GLM analysis.""" from .glm_launch import glm_launch_controller glm_launch_controller(glm_config_file=glm_config_file, level="L2", model=l2_name, test_one=test_one, submit=submit, @@ -415,7 +478,7 @@ def glm_l2_launch_cli(glm_config_file, l2_name, test_one, submit, debug): @click.option('-debug', is_flag=True, default=False, help='Print detailed processing information and traceback for errors.') def fsl_onset_extract_cli(config_file, glm_config_file, debug): - """Convert onset files to FSL's 3 column format""" + """Convert onset files to FSL's 3 column format.""" from .fsl_onset_extract import fsl_onset_extract fsl_onset_extract( config_file=config_file, glm_config_file=glm_config_file, debug=debug) @@ -432,7 +495,10 @@ def fsl_onset_extract_cli(config_file, glm_config_file, debug): default='confounds.tsv') def report_outliers_cli(confounds_dir, confounds_file, output_file, confound_suffix): - """Generate a confound outliers report.""" + """Generate a confound outliers report. + + Must provide one of either --confounds_dir or --confounds_file. + """ from .outliers_report import get_study_outliers, get_image_confounds if confounds_dir: get_study_outliers(confounds_dir, output_file, confound_suffix) @@ -446,7 +512,7 @@ def report_outliers_cli(confounds_dir, confounds_file, output_file, @click.option('-cache_file', type=CLICK_FILE_TYPE_EXISTS, help=CACHE_FILE_HELP, required=False) def status_cli(config_file, cache_file): - """Check the status of your project""" + """Check the status of your project.""" from .status import show_latest_by_step show_latest_by_step(config_file=config_file, cache_path=cache_file) diff --git a/clpipe/config.py b/clpipe/config.py index 680353ac..198b16f7 100644 --- a/clpipe/config.py +++ b/clpipe/config.py @@ -51,8 +51,9 @@ ) DICOM_DIR_HELP = "The folder where subject dicoms are located." DICOM_DIR_FORMAT_HELP = ( - "Format string for how subjects/sessions are organized within the " - "dicom_dir." + "Format string which specifies how subjects/sessions are organized within the " + "dicom_dir. Example: {subject}_{session}. See " + "https://clpipe.readthedocs.io/en/latest/bids_convert.html for more details." ) BIDS_DIR_HELP = "The dicom info output file name." OVERWRITE_HELP = "Overwrite existing BIDS data?" @@ -61,7 +62,8 @@ "arbitrary number of subjects as arguments now." ) SESSION_HELP = ( - "Specify the session to convert." + "Specify the session to convert. Available sessions determined by the {session} " + "placeholder given by dicom_dir_format." ) LONGITUDINAL_HELP = ( "Convert all subjects/sessions into individual pseudo-subjects. " diff --git a/clpipe/glm_launch.py b/clpipe/glm_launch.py index d5ff0070..4cace425 100644 --- a/clpipe/glm_launch.py +++ b/clpipe/glm_launch.py @@ -21,8 +21,8 @@ # libraries SUBMISSION_STRING_TEMPLATE = ("unset PYTHONPATH; feat {fsf_file}") -VALID_L1 = ["1", "L1", "l1"] -VALID_L2 = ["2", "L2", "l2"] +VALID_L1 = ["1", "L1", "l1", "level1"] +VALID_L2 = ["2", "L2", "l2", "level2"] L1 = VALID_L1[1] L2 = VALID_L2[1] From d060f8432ab4ac5117698e61c00fef0bd0d964e6 Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Thu, 17 Nov 2022 17:50:16 -0500 Subject: [PATCH 09/16] Add ROI extraction commands to the CLI. --- clpipe/cli.py | 57 +++++++++++++++++++++++++++++++++++++++-- clpipe/roi_extractor.py | 25 +----------------- 2 files changed, 56 insertions(+), 26 deletions(-) diff --git a/clpipe/cli.py b/clpipe/cli.py index 886505f6..ede49676 100644 --- a/clpipe/cli.py +++ b/clpipe/cli.py @@ -75,12 +75,16 @@ def cli(ctx, version): @click.group("glm", cls=OrderedHelpGroup) def glm_cli(): - """GLM Commands""" + """General Linear Model (GLM) Commands.""" @click.group("bids") def bids_cli(): - """BIDS Commands""" + """BIDS Commands.""" + +@click.group("roi", cls=OrderedHelpGroup) +def roi_cli(): + """Region of Interest (ROI) Commands.""" def _add_commands(): cli.add_command(project_setup_cli, help_priority=1) @@ -100,8 +104,12 @@ def _add_commands(): glm_cli.add_command(fsl_onset_extract_cli, help_priority=2) glm_cli.add_command(report_outliers_cli, help_priority=7) + roi_cli.add_command(get_availablea_atlases_cli, help_priority=1) + roi_cli.add_command(fmri_roi_extraction_cli, help_priority=2) + cli.add_command(bids_cli, help_priority=2) cli.add_command(glm_cli) + cli.add_command(roi_cli) @click.command(SETUP_COMMAND_NAME, no_args_is_help=True) @@ -484,6 +492,51 @@ def fsl_onset_extract_cli(config_file, glm_config_file, debug): config_file=config_file, glm_config_file=glm_config_file, debug=debug) +@click.command("extract") +@click.argument('subjects', nargs=-1, required=False, default=None) +@click.option('-config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, + help='Use a given configuration file. If left blank, uses the default config file, requiring definition of BIDS, working and output directories. This will extract all ROI sets specified in the configuration file.') +@click.option('-target_dir', type=click.Path(exists=True, dir_okay=True, file_okay=False), + help='Which postprocessed directory to process. If a configuration file is provided with a target directory, this argument is not necessary.') +@click.option('-target_suffix', + help='Which target suffix to process. If a configuration file is provided with a target suffix, this argument is not necessary.') +@click.option('-output_dir', type=click.Path(dir_okay=True, file_okay=False), + help='Where to put the ROI extracted data. If a configuration file is provided with a output directory, this argument is not necessary.') +@click.option('-task', help = 'Which task to process. If none, then all tasks are processed.') +@click.option('-atlas_name', help = "What atlas to use. Please refer to documentation, or use the command get_available_atlases to see which are available. When specified for a custom atlas, this is what the output files will be named.") +@click.option('-custom_atlas', help = 'A custom atlas image, in .nii or .nii.gz for label or maps, or a .txt tab delimited set of ROI coordinates if for a sphere atlas. Not needed if specified in config.') +@click.option('-custom_label', help = 'A custom atlas label file. Not needed if specified in config.') +@click.option('-custom_type', help = 'What type of atlas? (label, maps, or spheres). Not needed if specified in config.') +@click.option('-radius', help = "If a sphere atlas, what radius sphere, in mm. Not needed if specified in config.", default = '5') +@click.option('-overlap_ok', is_flag=True, default=False, help = "Are overlapping ROIs allowed?") +@click.option('-overwrite', is_flag=True, default=False, help = "Overwrite existing ROI timeseries?") +@click.option('-log_output_dir', type=click.Path(dir_okay=True, file_okay=False), + help='Where to put HPC output files (such as SLURM output files). If not specified, defaults to /batchOutput.') +@click.option('-submit', is_flag=True, default=False, help='Flag to submit commands to the HPC') +@click.option('-single', is_flag=True, default=False, help='Flag to directly run command. Used internally.') +@click.option('-debug', is_flag=True, help='Flag to enable detailed error messages and traceback') +def fmri_roi_extraction_cli(subjects, config_file, target_dir, target_suffix, + output_dir, task, log_output_dir, atlas_name, custom_atlas, + custom_label, custom_type, radius, submit, single, + overlap_ok, debug, overwrite): + """Extract ROIs with a given atlas.""" + from .roi_extractor import fmri_roi_extraction + fmri_roi_extraction( + subjects=subjects,config_file=config_file, target_dir=target_dir, + target_suffix=target_suffix, output_dir=output_dir, task=task, + log_output_dir=log_output_dir, atlas_name=atlas_name, custom_atlas=custom_atlas, + custom_label=custom_label, custom_type=custom_type, radius=radius, + submit=submit, single=single, overlap_ok=overlap_ok, debug=debug, + overwrite=overwrite) + + +@click.command("atlases") +def get_availablea_atlases_cli(): + """Display all available atlases.""" + from .roi_extractor import get_available_atlases + get_available_atlases() + + @click.command(OUTLIERS_COMMAND_NAME, no_args_is_help=True) @click.option('--confounds_dir', type=click.Path(exists=True, dir_okay=True, file_okay=False), help="Path to a directory containing subjects and confounds files.") diff --git a/clpipe/roi_extractor.py b/clpipe/roi_extractor.py index ea6611b4..13358045 100644 --- a/clpipe/roi_extractor.py +++ b/clpipe/roi_extractor.py @@ -18,29 +18,7 @@ import glob import shutil -@click.command() -@click.argument('subjects', nargs=-1, required=False, default=None) -@click.option('-config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, - help='Use a given configuration file. If left blank, uses the default config file, requiring definition of BIDS, working and output directories. This will extract all ROI sets specified in the configuration file.') -@click.option('-target_dir', type=click.Path(exists=True, dir_okay=True, file_okay=False), - help='Which postprocessed directory to process. If a configuration file is provided with a target directory, this argument is not necessary.') -@click.option('-target_suffix', - help='Which target suffix to process. If a configuration file is provided with a target suffix, this argument is not necessary.') -@click.option('-output_dir', type=click.Path(dir_okay=True, file_okay=False), - help='Where to put the ROI extracted data. If a configuration file is provided with a output directory, this argument is not necessary.') -@click.option('-task', help = 'Which task to process. If none, then all tasks are processed.') -@click.option('-atlas_name', help = "What atlas to use. Please refer to documentation, or use the command get_available_atlases to see which are available. When specified for a custom atlas, this is what the output files will be named.") -@click.option('-custom_atlas', help = 'A custom atlas image, in .nii or .nii.gz for label or maps, or a .txt tab delimited set of ROI coordinates if for a sphere atlas. Not needed if specified in config.') -@click.option('-custom_label', help = 'A custom atlas label file. Not needed if specified in config.') -@click.option('-custom_type', help = 'What type of atlas? (label, maps, or spheres). Not needed if specified in config.') -@click.option('-radius', help = "If a sphere atlas, what radius sphere, in mm. Not needed if specified in config.", default = '5') -@click.option('-overlap_ok', is_flag=True, default=False, help = "Are overlapping ROIs allowed?") -@click.option('-overwrite', is_flag=True, default=False, help = "Overwrite existing ROI timeseries?") -@click.option('-log_output_dir', type=click.Path(dir_okay=True, file_okay=False), - help='Where to put HPC output files (such as SLURM output files). If not specified, defaults to /batchOutput.') -@click.option('-submit', is_flag=True, default=False, help='Flag to submit commands to the HPC') -@click.option('-single', is_flag=True, default=False, help='Flag to directly run command. Used internally.') -@click.option('-debug', is_flag=True, help='Flag to enable detailed error messages and traceback') + def fmri_roi_extraction(subjects=None,config_file=None, target_dir=None, target_suffix = None, output_dir=None, task=None, log_output_dir=None, atlas_name = None, custom_atlas = None, @@ -273,7 +251,6 @@ def _fmri_roi_extract_image(data, atlas_path, atlas_type, radius, overlap_ok,ma return timeseries -@click.command() def get_available_atlases(): with resource_stream(__name__, 'data/atlasLibrary.json') as at_lib: From f388e44d13d61a74824e834951396e828e3cbc9b Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Thu, 17 Nov 2022 17:50:36 -0500 Subject: [PATCH 10/16] Adjust setup for new ROI command locations. --- clpipe/cli.py | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clpipe/cli.py b/clpipe/cli.py index ede49676..97d6072d 100644 --- a/clpipe/cli.py +++ b/clpipe/cli.py @@ -492,7 +492,7 @@ def fsl_onset_extract_cli(config_file, glm_config_file, debug): config_file=config_file, glm_config_file=glm_config_file, debug=debug) -@click.command("extract") +@click.command("extract", no_args_is_help=True) @click.argument('subjects', nargs=-1, required=False, default=None) @click.option('-config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, help='Use a given configuration file. If left blank, uses the default config file, requiring definition of BIDS, working and output directories. This will extract all ROI sets specified in the configuration file.') diff --git a/setup.py b/setup.py index 496fe04b..73d7154f 100644 --- a/setup.py +++ b/setup.py @@ -51,10 +51,10 @@ get_reports=clpipe.get_reports:get_reports get_config_file=clpipe.grab_config_file:get_config_file get_glm_config_file=clpipe.grab_config_file:get_glm_config_file - fmri_roi_extraction=clpipe.roi_extractor:fmri_roi_extraction + fmri_roi_extraction=clpipe.cli:fmri_roi_extraction_cli test_batch_setup=clpipe.test_batch_setup:test_batch_setup susan_smoothing = clpipe.susan_smoothing:susan_smoothing - get_available_atlases=clpipe.roi_extractor:get_available_atlases + get_available_atlases=clpipe.cli:get_available_atlases_cli update_config_file=clpipe.config_json_parser:update_config_file templateflow_setup=clpipe.template_flow:templateflow_setup test_func=clpipe.utils:test_func From 0ff6de44b27548d617e2165df81f34696988e563 Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Thu, 17 Nov 2022 17:55:20 -0500 Subject: [PATCH 11/16] Update CLI message. --- clpipe/cli.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/clpipe/cli.py b/clpipe/cli.py index 97d6072d..1b364ae0 100644 --- a/clpipe/cli.py +++ b/clpipe/cli.py @@ -54,9 +54,9 @@ def cli(ctx, version): """Welcome to clpipe. Please choose a processing command. You can get more help about about any of the below commands by running them - with the '-h' flag: + without arguments: - > clpipe setup -h + > clpipe setup If you're not sure where to begin, please see the documentation at: https://clpipe.readthedocs.io/en/latest/index.html @@ -135,7 +135,7 @@ def project_setup_cli(project_title=None, project_dir=None, source_data=None, symlink_source_data=symlink_source_data,debug=debug) -@click.command(CONVERSION_COMMAND_NAME) +@click.command(CONVERSION_COMMAND_NAME, no_args_is_help=True) @click.argument('subjects', nargs=-1, required=False, default=None) @click.option('-config_file', '-c', type=CLICK_FILE_TYPE_EXISTS, default=None, help=CONFIG_HELP) @@ -185,7 +185,7 @@ def convert2bids_cli(dicom_dir, dicom_dir_format, bids_dir, longitudinal=longitudinal, submit=submit, status_cache=status_cache, debug=debug, dcm2bids=dcm2bids) -@click.command(VALIDATOR_COMMAND_NAME) +@click.command(VALIDATOR_COMMAND_NAME, no_args_is_help=True) @click.argument('bids_dir', type=CLICK_DIR_TYPE_EXISTS, required=False) @click.option('-config_file', type=CLICK_FILE_TYPE_EXISTS, default=None, help=CONFIG_HELP) @@ -212,7 +212,7 @@ def bids_validate_cli(bids_dir, config_file, log_dir, interactive, submit, interactive=interactive, submit=submit, verbose=verbose, debug=debug) -@click.command(FMRIPREP_COMMAND_NAME) +@click.command(FMRIPREP_COMMAND_NAME, no_args_is_help=True) @click.argument('subjects', nargs=-1, required=False, default=None) @click.option('-config_file', default=None, type=CLICK_FILE_TYPE_EXISTS, help=CONFIG_HELP) @@ -244,7 +244,7 @@ def fmriprep_process_cli(bids_dir, working_dir, output_dir, config_file, status_cache=status_cache) -@click.command(POSTPROCESS_COMMAND_NAME) +@click.command(POSTPROCESS_COMMAND_NAME, no_args_is_help=True) @click.argument('subjects', nargs=-1, required=False, default=None) @click.option('-config_file', type=click.Path(exists=True, dir_okay=False, file_okay=True), default=None, help = 'Use a given configuration file. If left blank, uses the default config file, requiring definition of BIDS, working and output directories.') @click.option('-target_dir', type=click.Path(exists=True, dir_okay=True, file_okay=False), help='Which fmriprep directory to process. If a configuration file is provided with a BIDS directory, this argument is not necessary. Note, must point to the ``fmriprep`` directory, not its parent directory.') @@ -281,7 +281,7 @@ def fmri_postprocess_cli(config_file=None, subjects=None, target_dir=None, debug=debug, beta_series=beta_series) -@click.command(POSTPROCESS2_COMMAND_NAME) +@click.command(POSTPROCESS2_COMMAND_NAME, no_args_is_help=True) @click.argument('subjects', nargs=-1, required=False, default=None) @click.option('-config_file', '-c', type=CLICK_FILE_TYPE_EXISTS, default=None, required=True, help=CONFIG_HELP) From 6bcc66bebdb17fc8fad309fa33a4ac8249b985ef Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Thu, 17 Nov 2022 18:08:05 -0500 Subject: [PATCH 12/16] Add "more information" messages. --- clpipe/cli.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/clpipe/cli.py b/clpipe/cli.py index 1b364ae0..d60f71da 100644 --- a/clpipe/cli.py +++ b/clpipe/cli.py @@ -51,12 +51,9 @@ def add_command(self, cmd: click.Command, name: str = None, @click.option("-v", "--version", is_flag=True, default=False, help=VERSION_HELP) def cli(ctx, version): - """Welcome to clpipe. Please choose a processing command. + """Welcome to clpipe. - You can get more help about about any of the below commands by running them - without arguments: - - > clpipe setup + Please choose one of the commands below for more information. If you're not sure where to begin, please see the documentation at: https://clpipe.readthedocs.io/en/latest/index.html @@ -75,16 +72,26 @@ def cli(ctx, version): @click.group("glm", cls=OrderedHelpGroup) def glm_cli(): - """General Linear Model (GLM) Commands.""" + """General Linear Model (GLM) Commands. + + Please choose one of the commands below for more + information. + """ @click.group("bids") def bids_cli(): - """BIDS Commands.""" + """BIDS Commands. + + Please choose one of the commands below for more information. + """ @click.group("roi", cls=OrderedHelpGroup) def roi_cli(): - """Region of Interest (ROI) Commands.""" + """Region of Interest (ROI) Commands. + + Please choose one of the commands below for more information. + """ def _add_commands(): cli.add_command(project_setup_cli, help_priority=1) From 6262f7aab5838db34201f274a53581799ed2c304 Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Tue, 22 Nov 2022 16:28:04 -0500 Subject: [PATCH 13/16] Reference dict within GLMConfigParser. --- clpipe/glm_l2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clpipe/glm_l2.py b/clpipe/glm_l2.py index a3105279..8d7a0951 100644 --- a/clpipe/glm_l2.py +++ b/clpipe/glm_l2.py @@ -89,7 +89,7 @@ def glm_apply_mumford_workaround(glm_config_file=None, logger = get_logger(APPLY_MUMFORD_COMMAND_NAME, debug=debug) if glm_config_file: - glm_config = GLMConfigParser(glm_config_file) + glm_config = GLMConfigParser(glm_config_file).config l1_feat_folders_path = glm_config["Level1Setups"]["OutputDir"] logger.info(f"Applying Mumford workaround to: {l1_feat_folders_path}") From acf520a0efd391e3653e6916e0ea90751d64824a Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Tue, 22 Nov 2022 16:35:24 -0500 Subject: [PATCH 14/16] Add a flag for removing reg_standard folder. --- clpipe/glm_l2.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/clpipe/glm_l2.py b/clpipe/glm_l2.py index 8d7a0951..c293db40 100644 --- a/clpipe/glm_l2.py +++ b/clpipe/glm_l2.py @@ -63,7 +63,7 @@ def _glm_l2_propagate(l2_block, glm_setup_options, logger): if not os.path.exists(feat): raise FileNotFoundError("Cannot find "+ feat) else: - _apply_mumford_workaround(feat, logger) + _apply_mumford_workaround(feat, logger, remove_reg_standard=True) new_fsf[image_files_ind[counter - 1]] = "set feat_files(" + str(counter) + ") \"" + os.path.abspath( feat) + "\"\n" counter = counter + 1 @@ -85,6 +85,7 @@ def _glm_l2_propagate(l2_block, glm_setup_options, logger): def glm_apply_mumford_workaround(glm_config_file=None, l1_feat_folders_path=None, + remove_reg_standard=False, debug=False): logger = get_logger(APPLY_MUMFORD_COMMAND_NAME, debug=debug) @@ -97,12 +98,13 @@ def glm_apply_mumford_workaround(glm_config_file=None, for l1_feat_folder in os.scandir(l1_feat_folders_path): if os.path.isdir(l1_feat_folder): logger.info(f"Processing L1 FEAT folder: {l1_feat_folder.path}") - _apply_mumford_workaround(l1_feat_folder, logger) + _apply_mumford_workaround(l1_feat_folder, logger, + remove_reg_standard=remove_reg_standard) logger.info(f"Finished applying Mumford workaround.") -def _apply_mumford_workaround(l1_feat_folder, logger): +def _apply_mumford_workaround(l1_feat_folder, logger, remove_reg_standard=False): """ When using an image registration other than FSL's, such as fMRIPrep's, this work-around is necessary to run FEAT L2 analysis in FSL. @@ -123,11 +125,12 @@ def _apply_mumford_workaround(l1_feat_folder, logger): logger.debug(f"Removing: {mat}") os.remove(mat) - # Delete the reg_standard folder if it exists - reg_standard_path = l1_feat_folder / "reg_standard" - if reg_standard_path.exists(): - logger.debug(f"Removing: {reg_standard_path}") - shutil.rmtree(reg_standard_path) + if remove_reg_standard: + # Delete the reg_standard folder if it exists + reg_standard_path = l1_feat_folder / "reg_standard" + if reg_standard_path.exists(): + logger.debug(f"Removing: {reg_standard_path}") + shutil.rmtree(reg_standard_path) try: # Grab the FSLDIR environment var to get path to standard matrices From c7c662a875a8a0ad0d8e7c68b058970fb239502d Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Tue, 22 Nov 2022 16:35:52 -0500 Subject: [PATCH 15/16] Remove double printed log message. --- clpipe/glm_l2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clpipe/glm_l2.py b/clpipe/glm_l2.py index c293db40..10e606f1 100644 --- a/clpipe/glm_l2.py +++ b/clpipe/glm_l2.py @@ -92,7 +92,6 @@ def glm_apply_mumford_workaround(glm_config_file=None, if glm_config_file: glm_config = GLMConfigParser(glm_config_file).config l1_feat_folders_path = glm_config["Level1Setups"]["OutputDir"] - logger.info(f"Applying Mumford workaround to: {l1_feat_folders_path}") logger.info(f"Applying Mumford workaround to: {l1_feat_folders_path}") for l1_feat_folder in os.scandir(l1_feat_folders_path): From f37bc10ed12e5772be9c9af30c0116060985a1cc Mon Sep 17 00:00:00 2001 From: Will Asciutto Date: Tue, 22 Nov 2022 16:44:36 -0500 Subject: [PATCH 16/16] Update apply_mumford option with remove_reg_standard flag. --- clpipe/cli.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/clpipe/cli.py b/clpipe/cli.py index d60f71da..b4e0f5dc 100644 --- a/clpipe/cli.py +++ b/clpipe/cli.py @@ -395,17 +395,21 @@ def glm_l2_preparefsf_cli(glm_config_file, l2_name, debug): help='Your GLM configuration file.') @click.option('-l1_feat_folders_path', type=CLICK_DIR_TYPE_EXISTS, default=None, required=False, - help='Location of your L1 FEAT folders.') -@click.option('-debug', is_flag=True, + help='Directory containing your L1 FEAT folders.') +@click.option('-remove_reg_standard', is_flag=True, default=False, + help='Remove reg_standard folders (generated by L2) in addition to reg.') +@click.option('-debug', is_flag=True, default=False, help='Flag to enable detailed error messages and traceback') def glm_apply_mumford_workaround_cli(glm_config_file, l1_feat_folders_path, - debug): + remove_reg_standard, debug): """ Apply the Mumford registration workaround to L1 FEAT folders. Applied by default in glm-l2-preparefsf. This command is useful for applying the Mumford workaround to single-run subjects who skip L2, allowing you to still combine them with multiple-run subjects at L3. + + Must provide GLM config file OR a path to your L1 FEAT folders. """ from .glm_l2 import glm_apply_mumford_workaround if not (glm_config_file or l1_feat_folders_path): @@ -413,7 +417,8 @@ def glm_apply_mumford_workaround_cli(glm_config_file, l1_feat_folders_path, "or '-l1_feat_folders_path' required.")) glm_apply_mumford_workaround( glm_config_file=glm_config_file, - l1_feat_folders_path=l1_feat_folders_path, debug=debug + l1_feat_folders_path=l1_feat_folders_path, debug=debug, + remove_reg_standard=remove_reg_standard )