From 0e07ef1b9a3fea9b28321ae36294fae7e6483428 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Wed, 20 Nov 2024 16:04:39 -0500 Subject: [PATCH] Write out correlations when `--create-matrices` doesn't include "all" (#1326) --- xcp_d/cli/parser.py | 37 ++------ xcp_d/cli/parser_utils.py | 2 +- xcp_d/config.py | 8 +- xcp_d/data/tests/config.toml | 2 +- .../data/test_ds001419_cifti_outputs.txt | 40 --------- xcp_d/tests/test_cli.py | 1 - xcp_d/tests/test_cli_run.py | 25 +++--- xcp_d/tests/test_cli_utils.py | 4 +- xcp_d/tests/test_workflows_connectivity.py | 4 +- xcp_d/utils/doc.py | 4 +- xcp_d/utils/modified_data.py | 1 + xcp_d/utils/utils.py | 9 ++ xcp_d/workflows/base.py | 6 +- xcp_d/workflows/bold/concatenation.py | 87 ++++++++++--------- xcp_d/workflows/bold/connectivity.py | 8 +- xcp_d/workflows/bold/outputs.py | 4 +- xcp_d/workflows/bold/postprocessing.py | 4 +- 17 files changed, 97 insertions(+), 149 deletions(-) diff --git a/xcp_d/cli/parser.py b/xcp_d/cli/parser.py index d23d80740..9b52b8208 100644 --- a/xcp_d/cli/parser.py +++ b/xcp_d/cli/parser.py @@ -546,7 +546,7 @@ def _build_parser(): g_dcan.add_argument( '--create-matrices', '--create_matrices', - dest='dcan_correlation_lengths', + dest='correlation_lengths', required=False, default=None, nargs='+', @@ -967,9 +967,7 @@ def _validate_parameters(opts, build_log, parser): opts.confounds_config = ( '36P' if (opts.confounds_config == 'auto') else opts.confounds_config ) - opts.dcan_correlation_lengths = ( - [] if opts.dcan_correlation_lengths is None else opts.dcan_correlation_lengths - ) + opts.correlation_lengths = opts.correlation_lengths if opts.correlation_lengths else [] opts.despike = True if (opts.despike == 'auto') else opts.despike opts.fd_thresh = 0.3 if (opts.fd_thresh == 'auto') else opts.fd_thresh opts.file_format = 'cifti' if (opts.file_format == 'auto') else opts.file_format @@ -978,23 +976,18 @@ def _validate_parameters(opts, build_log, parser): opts.min_coverage = 0.5 if opts.min_coverage == 'auto' else opts.min_coverage if opts.motion_filter_type is None: error_messages.append(f"'--motion-filter-type' is required for '{opts.mode}' mode.") - opts.output_correlations = True if 'all' in opts.dcan_correlation_lengths else False if opts.output_type == 'censored': error_messages.append(f"'--output-type' cannot be 'censored' for '{opts.mode}' mode.") opts.output_type = 'interpolated' opts.process_surfaces = True if opts.process_surfaces == 'auto' else opts.process_surfaces opts.smoothing = 6 if opts.smoothing == 'auto' else opts.smoothing - # Remove "all" from the list of correlation lengths - opts.dcan_correlation_lengths = [c for c in opts.dcan_correlation_lengths if c != 'all'] elif opts.mode == 'hbcd': opts.abcc_qc = True if (opts.abcc_qc == 'auto') else opts.abcc_qc opts.combine_runs = True if (opts.combine_runs == 'auto') else opts.combine_runs opts.confounds_config = ( '36P' if (opts.confounds_config == 'auto') else opts.confounds_config ) - opts.dcan_correlation_lengths = ( - [] if opts.dcan_correlation_lengths is None else opts.dcan_correlation_lengths - ) + opts.correlation_lengths = opts.correlation_lengths if opts.correlation_lengths else [] opts.despike = True if (opts.despike == 'auto') else opts.despike opts.fd_thresh = 0.3 if (opts.fd_thresh == 'auto') else opts.fd_thresh opts.file_format = 'cifti' if (opts.file_format == 'auto') else opts.file_format @@ -1003,14 +996,11 @@ def _validate_parameters(opts, build_log, parser): opts.min_coverage = 0.5 if opts.min_coverage == 'auto' else opts.min_coverage if opts.motion_filter_type is None: error_messages.append(f"'--motion-filter-type' is required for '{opts.mode}' mode.") - opts.output_correlations = True if 'all' in opts.dcan_correlation_lengths else False if opts.output_type == 'censored': error_messages.append(f"'--output-type' cannot be 'censored' for '{opts.mode}' mode.") opts.output_type = 'interpolated' opts.process_surfaces = True if opts.process_surfaces == 'auto' else opts.process_surfaces opts.smoothing = 6 if opts.smoothing == 'auto' else opts.smoothing - # Remove "all" from the list of correlation lengths - opts.dcan_correlation_lengths = [c for c in opts.dcan_correlation_lengths if c != 'all'] elif opts.mode == 'linc': opts.abcc_qc = False if (opts.abcc_qc == 'auto') else opts.abcc_qc opts.combine_runs = False if opts.combine_runs == 'auto' else opts.combine_runs @@ -1023,7 +1013,6 @@ def _validate_parameters(opts, build_log, parser): opts.input_type = 'fmriprep' if opts.input_type == 'auto' else opts.input_type opts.linc_qc = True if (opts.linc_qc == 'auto') else opts.linc_qc opts.min_coverage = 0.5 if opts.min_coverage == 'auto' else opts.min_coverage - opts.output_correlations = True if opts.output_type == 'interpolated': error_messages.append( f"'--output-type' cannot be 'interpolated' for '{opts.mode}' mode." @@ -1031,29 +1020,26 @@ def _validate_parameters(opts, build_log, parser): opts.output_type = 'censored' opts.process_surfaces = False if opts.process_surfaces == 'auto' else opts.process_surfaces opts.smoothing = 6 if opts.smoothing == 'auto' else opts.smoothing - if opts.dcan_correlation_lengths is not None: + if opts.correlation_lengths is not None: error_messages.append(f"'--create-matrices' is not supported for '{opts.mode}' mode.") + # Patch 'all' into the list of correlation lengths + opts.correlation_lengths = ['all'] elif opts.mode == 'nichart': opts.abcc_qc = False if (opts.abcc_qc == 'auto') else opts.abcc_qc opts.combine_runs = False if opts.combine_runs == 'auto' else opts.combine_runs opts.confounds_config = ( '36P' if (opts.confounds_config == 'auto') else opts.confounds_config ) - opts.dcan_correlation_lengths = ( - 'all' if opts.dcan_correlation_lengths is None else opts.dcan_correlation_lengths - ) + opts.correlation_lengths = opts.correlation_lengths if opts.correlation_lengths else 'all' opts.despike = True if (opts.despike == 'auto') else opts.despike opts.fd_thresh = 0 if (opts.fd_thresh == 'auto') else opts.fd_thresh opts.file_format = 'nifti' if (opts.file_format == 'auto') else opts.file_format opts.input_type = 'fmriprep' if opts.input_type == 'auto' else opts.input_type opts.linc_qc = True if (opts.linc_qc == 'auto') else opts.linc_qc opts.min_coverage = 0.4 if opts.min_coverage == 'auto' else opts.min_coverage - opts.output_correlations = True if 'all' in opts.dcan_correlation_lengths else False opts.output_type = 'censored' if opts.output_type == 'auto' else opts.output_type opts.process_surfaces = False if opts.process_surfaces == 'auto' else opts.process_surfaces opts.smoothing = 0 if opts.smoothing == 'auto' else opts.smoothing - # Remove "all" from the list of correlation lengths - opts.dcan_correlation_lengths = [c for c in opts.dcan_correlation_lengths if c != 'all'] elif opts.mode == 'none': if opts.abcc_qc == 'auto': error_messages.append("'--abcc-qc' (y or n) is required for 'none' mode.") @@ -1064,9 +1050,7 @@ def _validate_parameters(opts, build_log, parser): if opts.confounds_config == 'auto': error_messages.append("'--nuisance-regressors' is required for 'none' mode.") - opts.dcan_correlation_lengths = ( - [] if opts.dcan_correlation_lengths is None else opts.dcan_correlation_lengths - ) + opts.correlation_lengths = opts.correlation_lengths if opts.correlation_lengths else [] if opts.despike == 'auto': error_messages.append("'--despike' (y or n) is required for 'none' mode.") @@ -1090,8 +1074,6 @@ def _validate_parameters(opts, build_log, parser): if opts.motion_filter_type is None: error_messages.append("'--motion-filter-type' is required for 'none' mode.") - opts.output_correlations = True if 'all' in opts.dcan_correlation_lengths else False - if opts.output_type == 'auto': error_messages.append("'--output-type' is required for 'none' mode.") @@ -1103,9 +1085,6 @@ def _validate_parameters(opts, build_log, parser): if opts.smoothing == 'auto': error_messages.append("'--smoothing' is required for 'none' mode.") - # Remove "all" from the list of correlation lengths - opts.dcan_correlation_lengths = [c for c in opts.dcan_correlation_lengths if c != 'all'] - # Load the confound configuration file if opts.confounds_config == 'none': opts.confounds_config = None diff --git a/xcp_d/cli/parser_utils.py b/xcp_d/cli/parser_utils.py index 84ed32201..67a708490 100644 --- a/xcp_d/cli/parser_utils.py +++ b/xcp_d/cli/parser_utils.py @@ -61,7 +61,7 @@ def _float_or_auto_or_none(string, is_parser=True): if floatarg < 0: raise error('Float argument must be nonnegative.') - return floatarg + return str(floatarg) def _restricted_float(x): diff --git a/xcp_d/config.py b/xcp_d/config.py index 4ddaf2f7f..7ac5e7d6a 100644 --- a/xcp_d/config.py +++ b/xcp_d/config.py @@ -562,8 +562,6 @@ class workflow(_Config): """Full-width at half-maximum (FWHM) of the smoothing kernel.""" output_interpolated = None """Output interpolated data, not censored data.""" - output_correlations = None - """Output correlations from censored data. This doesn't affect exact_scans.""" combine_runs = None """Combine runs of the same task.""" motion_filter_type = None @@ -590,8 +588,10 @@ class workflow(_Config): """Order of the band-pass filter.""" min_coverage = None """Coverage threshold to apply to parcels in each atlas.""" - dcan_correlation_lengths = None - """Produce correlation matrices limited to each requested amount of time.""" + correlation_lengths = None + """Produce correlation matrices limited to each requested amount of time. + If this list includes 'all' then correlations from the full (censored) time series will + be produced.""" process_surfaces = None """Warp fsnative-space surfaces to the MNI space.""" abcc_qc = None diff --git a/xcp_d/data/tests/config.toml b/xcp_d/data/tests/config.toml index b0e01f04e..9a6ee2fdc 100644 --- a/xcp_d/data/tests/config.toml +++ b/xcp_d/data/tests/config.toml @@ -47,7 +47,7 @@ low_pass = 0.1 bpf_order = 2 atlases = [] min_coverage = 0.5 -dcan_correlation_lengths = [] +correlation_lengths = [] process_surfaces = false abcc_qc = false diff --git a/xcp_d/tests/data/test_ds001419_cifti_outputs.txt b/xcp_d/tests/data/test_ds001419_cifti_outputs.txt index 16c3e9c6f..975e831e4 100644 --- a/xcp_d/tests/data/test_ds001419_cifti_outputs.txt +++ b/xcp_d/tests/data/test_ds001419_cifti_outputs.txt @@ -67,8 +67,6 @@ sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_den-91k_stat- sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_den-91k_stat-coverage_boldmap.pscalar.nii sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_den-91k_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_den-91k_stat-mean_timeseries.ptseries.nii -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_den-91k_stat-pearsoncorrelation_boldmap.json -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_den-91k_stat-pearsoncorrelation_boldmap.pconn.nii sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_stat-alff_bold.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_stat-alff_bold.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_stat-coverage_bold.json @@ -76,16 +74,12 @@ sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_stat-coverage sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_stat-mean_timeseries.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_stat-pearsoncorrelation_desc-26volumes_relmat.tsv -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_stat-pearsoncorrelation_relmat.json -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_stat-pearsoncorrelation_relmat.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_stat-reho_bold.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S156Parcels_stat-reho_bold.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_den-91k_stat-coverage_boldmap.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_den-91k_stat-coverage_boldmap.pscalar.nii sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_den-91k_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_den-91k_stat-mean_timeseries.ptseries.nii -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_den-91k_stat-pearsoncorrelation_boldmap.json -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_den-91k_stat-pearsoncorrelation_boldmap.pconn.nii sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_stat-alff_bold.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_stat-alff_bold.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_stat-coverage_bold.json @@ -93,16 +87,12 @@ sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_stat-coverage sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_stat-mean_timeseries.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_stat-pearsoncorrelation_desc-26volumes_relmat.tsv -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_stat-pearsoncorrelation_relmat.json -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_stat-pearsoncorrelation_relmat.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_stat-reho_bold.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S256Parcels_stat-reho_bold.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_den-91k_stat-coverage_boldmap.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_den-91k_stat-coverage_boldmap.pscalar.nii sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_den-91k_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_den-91k_stat-mean_timeseries.ptseries.nii -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_den-91k_stat-pearsoncorrelation_boldmap.json -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_den-91k_stat-pearsoncorrelation_boldmap.pconn.nii sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_stat-alff_bold.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_stat-alff_bold.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_stat-coverage_bold.json @@ -110,16 +100,12 @@ sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_stat-coverage sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_stat-mean_timeseries.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_stat-pearsoncorrelation_desc-26volumes_relmat.tsv -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_stat-pearsoncorrelation_relmat.json -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_stat-pearsoncorrelation_relmat.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_stat-reho_bold.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S356Parcels_stat-reho_bold.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_den-91k_stat-coverage_boldmap.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_den-91k_stat-coverage_boldmap.pscalar.nii sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_den-91k_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_den-91k_stat-mean_timeseries.ptseries.nii -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_den-91k_stat-pearsoncorrelation_boldmap.json -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_den-91k_stat-pearsoncorrelation_boldmap.pconn.nii sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_stat-alff_bold.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_stat-alff_bold.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_stat-coverage_bold.json @@ -127,8 +113,6 @@ sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_stat-coverage sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_stat-mean_timeseries.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_stat-pearsoncorrelation_desc-26volumes_relmat.tsv -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_stat-pearsoncorrelation_relmat.json -sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_stat-pearsoncorrelation_relmat.tsv sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_stat-reho_bold.json sub-01/func/sub-01_task-imagery_run-01_space-fsLR_seg-4S456Parcels_stat-reho_bold.tsv sub-01/func/sub-01_task-imagery_run-02_desc-abcc_qc.hdf5 @@ -153,8 +137,6 @@ sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_den-91k_stat- sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_den-91k_stat-coverage_boldmap.pscalar.nii sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_den-91k_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_den-91k_stat-mean_timeseries.ptseries.nii -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_den-91k_stat-pearsoncorrelation_boldmap.json -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_den-91k_stat-pearsoncorrelation_boldmap.pconn.nii sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_stat-alff_bold.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_stat-alff_bold.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_stat-coverage_bold.json @@ -162,16 +144,12 @@ sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_stat-coverage sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_stat-mean_timeseries.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_stat-pearsoncorrelation_desc-26volumes_relmat.tsv -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_stat-pearsoncorrelation_relmat.json -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_stat-pearsoncorrelation_relmat.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_stat-reho_bold.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S156Parcels_stat-reho_bold.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_den-91k_stat-coverage_boldmap.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_den-91k_stat-coverage_boldmap.pscalar.nii sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_den-91k_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_den-91k_stat-mean_timeseries.ptseries.nii -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_den-91k_stat-pearsoncorrelation_boldmap.json -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_den-91k_stat-pearsoncorrelation_boldmap.pconn.nii sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_stat-alff_bold.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_stat-alff_bold.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_stat-coverage_bold.json @@ -179,16 +157,12 @@ sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_stat-coverage sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_stat-mean_timeseries.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_stat-pearsoncorrelation_desc-26volumes_relmat.tsv -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_stat-pearsoncorrelation_relmat.json -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_stat-pearsoncorrelation_relmat.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_stat-reho_bold.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S256Parcels_stat-reho_bold.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_den-91k_stat-coverage_boldmap.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_den-91k_stat-coverage_boldmap.pscalar.nii sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_den-91k_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_den-91k_stat-mean_timeseries.ptseries.nii -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_den-91k_stat-pearsoncorrelation_boldmap.json -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_den-91k_stat-pearsoncorrelation_boldmap.pconn.nii sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_stat-alff_bold.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_stat-alff_bold.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_stat-coverage_bold.json @@ -196,16 +170,12 @@ sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_stat-coverage sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_stat-mean_timeseries.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_stat-pearsoncorrelation_desc-26volumes_relmat.tsv -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_stat-pearsoncorrelation_relmat.json -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_stat-pearsoncorrelation_relmat.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_stat-reho_bold.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S356Parcels_stat-reho_bold.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_den-91k_stat-coverage_boldmap.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_den-91k_stat-coverage_boldmap.pscalar.nii sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_den-91k_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_den-91k_stat-mean_timeseries.ptseries.nii -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_den-91k_stat-pearsoncorrelation_boldmap.json -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_den-91k_stat-pearsoncorrelation_boldmap.pconn.nii sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_stat-alff_bold.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_stat-alff_bold.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_stat-coverage_bold.json @@ -213,8 +183,6 @@ sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_stat-coverage sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_stat-mean_timeseries.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_stat-pearsoncorrelation_desc-26volumes_relmat.tsv -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_stat-pearsoncorrelation_relmat.json -sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_stat-pearsoncorrelation_relmat.tsv sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_stat-reho_bold.json sub-01/func/sub-01_task-imagery_run-02_space-fsLR_seg-4S456Parcels_stat-reho_bold.tsv sub-01/func/sub-01_task-imagery_space-fsLR_den-91k_desc-denoisedSmoothed_bold.dtseries.nii @@ -225,24 +193,16 @@ sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S156Parcels_den-91k_stat-mean_ti sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S156Parcels_den-91k_stat-mean_timeseries.ptseries.nii sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S156Parcels_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S156Parcels_stat-mean_timeseries.tsv -sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S156Parcels_stat-pearsoncorrelation_relmat.json -sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S156Parcels_stat-pearsoncorrelation_relmat.tsv sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S256Parcels_den-91k_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S256Parcels_den-91k_stat-mean_timeseries.ptseries.nii sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S256Parcels_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S256Parcels_stat-mean_timeseries.tsv -sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S256Parcels_stat-pearsoncorrelation_relmat.json -sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S256Parcels_stat-pearsoncorrelation_relmat.tsv sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S356Parcels_den-91k_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S356Parcels_den-91k_stat-mean_timeseries.ptseries.nii sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S356Parcels_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S356Parcels_stat-mean_timeseries.tsv -sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S356Parcels_stat-pearsoncorrelation_relmat.json -sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S356Parcels_stat-pearsoncorrelation_relmat.tsv sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S456Parcels_den-91k_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S456Parcels_den-91k_stat-mean_timeseries.ptseries.nii sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S456Parcels_stat-mean_timeseries.json sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S456Parcels_stat-mean_timeseries.tsv -sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S456Parcels_stat-pearsoncorrelation_relmat.json -sub-01/func/sub-01_task-imagery_space-fsLR_seg-4S456Parcels_stat-pearsoncorrelation_relmat.tsv sub-01_executive_summary.html diff --git a/xcp_d/tests/test_cli.py b/xcp_d/tests/test_cli.py index 090038c0c..320733009 100644 --- a/xcp_d/tests/test_cli.py +++ b/xcp_d/tests/test_cli.py @@ -112,7 +112,6 @@ def test_ds001419_cifti(data_dir, output_dir, working_dir): '--upper-bpf=0.0', '--min-time=100', '--create-matrices', - 'all', '80', '200', '--atlases', diff --git a/xcp_d/tests/test_cli_run.py b/xcp_d/tests/test_cli_run.py index 3faeed277..97c9c3736 100644 --- a/xcp_d/tests/test_cli_run.py +++ b/xcp_d/tests/test_cli_run.py @@ -46,7 +46,7 @@ def base_opts(): 'process_surfaces': 'auto', 'atlases': ['Glasser'], 'min_coverage': 'auto', - 'dcan_correlation_lengths': None, + 'correlation_lengths': None, 'despike': 'auto', 'abcc_qc': 'auto', 'linc_qc': 'auto', @@ -280,9 +280,10 @@ def test_validate_parameters_linc_mode(base_opts, base_parser, capsys): assert opts.file_format == 'cifti' assert opts.min_coverage == 0.5 assert opts.smoothing == 6.0 + assert opts.correlation_lengths == ['all'] # --create-matrices is not supported - opts.dcan_correlation_lengths = [300] + opts.correlation_lengths = ['300'] with pytest.raises(SystemExit, match='2'): parser._validate_parameters(deepcopy(opts), build_log, parser=base_parser) @@ -302,21 +303,19 @@ def test_validate_parameters_abcd_mode(base_opts, base_parser, capsys): assert opts.abcc_qc is True assert opts.combine_runs is True - assert opts.dcan_correlation_lengths == [] + assert opts.correlation_lengths == [] assert opts.despike is True assert opts.fd_thresh == 0.3 assert opts.file_format == 'cifti' assert opts.input_type == 'fmriprep' assert opts.linc_qc is True assert opts.min_coverage == 0.5 - assert opts.output_correlations is False assert opts.process_surfaces is True assert opts.smoothing == 6.0 - opts.dcan_correlation_lengths = ['300', 'all'] + opts.correlation_lengths = ['300', 'all'] opts = parser._validate_parameters(deepcopy(opts), build_log, parser=base_parser) - assert opts.dcan_correlation_lengths == ['300'] - assert opts.output_correlations is True + assert opts.correlation_lengths == ['300', 'all'] # --motion-filter-type is required opts.motion_filter_type = None @@ -339,21 +338,19 @@ def test_validate_parameters_hbcd_mode(base_opts, base_parser, capsys): assert opts.abcc_qc is True assert opts.combine_runs is True - assert opts.dcan_correlation_lengths == [] + assert opts.correlation_lengths == [] assert opts.despike is True assert opts.fd_thresh == 0.3 assert opts.file_format == 'cifti' assert opts.input_type == 'nibabies' assert opts.linc_qc is True assert opts.min_coverage == 0.5 - assert opts.output_correlations is False assert opts.process_surfaces is True assert opts.smoothing == 6.0 - opts.dcan_correlation_lengths = ['300', 'all'] + opts.correlation_lengths = ['300', 'all'] opts = parser._validate_parameters(deepcopy(opts), build_log, parser=base_parser) - assert opts.dcan_correlation_lengths == ['300'] - assert opts.output_correlations is True + assert opts.correlation_lengths == ['300', 'all'] # --motion-filter-type is required opts.motion_filter_type = None @@ -462,7 +459,7 @@ def test_build_parser_01(tmp_path_factory): opts = parser_obj.parse_args(args=test_args, namespace=None) assert opts.fmri_dir == data_path assert opts.output_dir == out_path - assert opts.dcan_correlation_lengths == ['all', 300, 480] + assert opts.correlation_lengths == ['all', '300.0', '480.0'] def test_build_parser_02(tmp_path_factory): @@ -499,7 +496,7 @@ def test_build_parser_02(tmp_path_factory): opts = parser_obj.parse_args(args=test_args, namespace=None) assert opts.fmri_dir == data_path assert opts.output_dir == out_path - assert opts.dcan_correlation_lengths == ['all', 300, 480] + assert opts.correlation_lengths == ['all', '300.0', '480.0'] @pytest.mark.parametrize( diff --git a/xcp_d/tests/test_cli_utils.py b/xcp_d/tests/test_cli_utils.py index 4ec6259de..63014119f 100644 --- a/xcp_d/tests/test_cli_utils.py +++ b/xcp_d/tests/test_cli_utils.py @@ -83,10 +83,10 @@ def test_float_or_auto_or_none(): assert out == 'none' out = parser_utils._float_or_auto_or_none('3') - assert out == 3.0 + assert out == '3.0' out = parser_utils._float_or_auto_or_none(3) - assert out == 3.0 + assert out == '3.0' def test_is_file(tmp_path_factory): diff --git a/xcp_d/tests/test_workflows_connectivity.py b/xcp_d/tests/test_workflows_connectivity.py index bd20d55e6..9190e16ca 100644 --- a/xcp_d/tests/test_workflows_connectivity.py +++ b/xcp_d/tests/test_workflows_connectivity.py @@ -167,7 +167,7 @@ def test_init_functional_connectivity_nifti_wf(ds001419_data, tmp_path_factory): config.workflow.min_coverage = 0.5 config.nipype.omp_nthreads = 2 config.execution.atlases = atlas_names - config.workflow.output_correlations = True + config.workflow.correlation_lengths = ['all'] connectivity_wf = init_functional_connectivity_nifti_wf( mem_gb=mem_gbx, @@ -308,7 +308,7 @@ def test_init_functional_connectivity_cifti_wf(ds001419_data, tmp_path_factory): config.workflow.min_coverage = 0.5 config.nipype.omp_nthreads = 2 config.execution.atlases = atlas_names - config.workflow.output_correlations = True + config.workflow.correlation_lengths = ['all'] connectivity_wf = init_functional_connectivity_cifti_wf( mem_gb=mem_gbx, diff --git a/xcp_d/utils/doc.py b/xcp_d/utils/doc.py index c7aca4554..779e4a06d 100644 --- a/xcp_d/utils/doc.py +++ b/xcp_d/utils/doc.py @@ -252,8 +252,8 @@ ================= ================= """ -docdict['dcan_correlation_lengths'] = """ -dcan_correlation_lengths : :obj:`list` of :obj:`float`, optional +docdict['correlation_lengths'] = """ +correlation_lengths : :obj:`list` of :obj:`str`, optional If used, this parameter will produce correlation matrices limited to each requested amount of time. If there is more than the required amount of low-motion data, diff --git a/xcp_d/utils/modified_data.py b/xcp_d/utils/modified_data.py index 91d08e490..99db7331f 100644 --- a/xcp_d/utils/modified_data.py +++ b/xcp_d/utils/modified_data.py @@ -233,6 +233,7 @@ def calculate_exact_scans(exact_times, scan_length, t_r, bold_file): float_times = [] non_float_times = [] + exact_times = [time for time in exact_times if time != 'all'] for time in exact_times: try: float_times.append(float(time)) diff --git a/xcp_d/utils/utils.py b/xcp_d/utils/utils.py index e82744141..887be0097 100644 --- a/xcp_d/utils/utils.py +++ b/xcp_d/utils/utils.py @@ -602,3 +602,12 @@ def _create_mem_gb(bold_fname): mem_gbz['resampled'] = 3 return mem_gbz + + +def is_number(s): + """Check if a string is a number.""" + try: + float(s) + return True + except ValueError: + return False diff --git a/xcp_d/workflows/base.py b/xcp_d/workflows/base.py index bd62dbfae..e97617ce7 100644 --- a/xcp_d/workflows/base.py +++ b/xcp_d/workflows/base.py @@ -39,7 +39,7 @@ ) from xcp_d.utils.doc import fill_doc from xcp_d.utils.modified_data import calculate_exact_scans, flag_bad_run -from xcp_d.utils.utils import estimate_brain_radius +from xcp_d.utils.utils import estimate_brain_radius, is_number from xcp_d.workflows.anatomical.parcellation import init_parcellate_surfaces_wf from xcp_d.workflows.anatomical.surface import init_postprocess_surfaces_wf from xcp_d.workflows.anatomical.volume import init_postprocess_anat_wf @@ -458,9 +458,9 @@ def init_single_subject_wf(subject_id: str): # Reduce exact_times to only include values greater than the post-scrubbing duration. exact_scans = [] - if config.workflow.dcan_correlation_lengths: + if any(is_number(length) for length in config.workflow.correlation_lengths): exact_scans = calculate_exact_scans( - exact_times=config.workflow.dcan_correlation_lengths, + exact_times=config.workflow.correlation_lengths, scan_length=post_scrubbing_duration, t_r=run_data['bold_metadata']['RepetitionTime'], bold_file=bold_file, diff --git a/xcp_d/workflows/bold/concatenation.py b/xcp_d/workflows/bold/concatenation.py index e4742bcbe..f36a8ac49 100644 --- a/xcp_d/workflows/bold/concatenation.py +++ b/xcp_d/workflows/bold/concatenation.py @@ -362,52 +362,53 @@ def init_concatenate_data_wf(TR, head_radius, name='concatenate_data_wf'): (make_timeseries_dict, ds_timeseries, [('metadata', 'meta_dict')]), ]) # fmt:skip - correlate_timeseries = pe.MapNode( - TSVConnect(), - run_without_submitting=True, - mem_gb=1, - name='correlate_timeseries', - iterfield=['timeseries'], - ) - workflow.connect([ - (concatenate_inputs, correlate_timeseries, [ - ('timeseries', 'timeseries'), - ('temporal_mask', 'temporal_mask'), - ]), - ]) # fmt:skip + if 'all' in config.workflow.correlation_lengths: + correlate_timeseries = pe.MapNode( + TSVConnect(), + run_without_submitting=True, + mem_gb=1, + name='correlate_timeseries', + iterfield=['timeseries'], + ) + workflow.connect([ + (concatenate_inputs, correlate_timeseries, [ + ('timeseries', 'timeseries'), + ('temporal_mask', 'temporal_mask'), + ]), + ]) # fmt:skip - make_correlations_dict = pe.MapNode( - BIDSURI( - numinputs=1, - dataset_links=config.execution.dataset_links, - out_dir=str(output_dir), - ), - run_without_submitting=True, - mem_gb=1, - name='make_correlations_dict', - iterfield=['in1'], - ) - workflow.connect([(ds_timeseries, make_correlations_dict, [('out_file', 'in1')])]) + make_correlations_dict = pe.MapNode( + BIDSURI( + numinputs=1, + dataset_links=config.execution.dataset_links, + out_dir=str(output_dir), + ), + run_without_submitting=True, + mem_gb=1, + name='make_correlations_dict', + iterfield=['in1'], + ) + workflow.connect([(ds_timeseries, make_correlations_dict, [('out_file', 'in1')])]) - ds_correlations = pe.MapNode( - DerivativesDataSink( - dismiss_entities=['desc'], - statistic='pearsoncorrelation', - suffix='relmat', - extension='.tsv', - ), - name='ds_correlations', - run_without_submitting=True, - mem_gb=1, - iterfield=['segmentation', 'in_file', 'meta_dict'], - ) - ds_correlations.inputs.segmentation = atlases + ds_correlations = pe.MapNode( + DerivativesDataSink( + dismiss_entities=['desc'], + statistic='pearsoncorrelation', + suffix='relmat', + extension='.tsv', + ), + name='ds_correlations', + run_without_submitting=True, + mem_gb=1, + iterfield=['segmentation', 'in_file', 'meta_dict'], + ) + ds_correlations.inputs.segmentation = atlases - workflow.connect([ - (clean_name_source, ds_correlations, [('name_source', 'source_file')]), - (correlate_timeseries, ds_correlations, [('correlations', 'in_file')]), - (make_correlations_dict, ds_correlations, [('metadata', 'meta_dict')]), - ]) # fmt:skip + workflow.connect([ + (clean_name_source, ds_correlations, [('name_source', 'source_file')]), + (correlate_timeseries, ds_correlations, [('correlations', 'in_file')]), + (make_correlations_dict, ds_correlations, [('metadata', 'meta_dict')]), + ]) # fmt:skip if file_format == 'cifti': cifti_ts_src = pe.MapNode( diff --git a/xcp_d/workflows/bold/connectivity.py b/xcp_d/workflows/bold/connectivity.py index 49da4c53f..b34a63986 100644 --- a/xcp_d/workflows/bold/connectivity.py +++ b/xcp_d/workflows/bold/connectivity.py @@ -130,7 +130,7 @@ def init_functional_connectivity_nifti_wf(mem_gb, name='connectivity_wf'): ]), ]) # fmt:skip - if config.workflow.output_correlations: + if 'all' in config.workflow.correlation_lengths: functional_connectivity = pe.MapNode( TSVConnect(), name='functional_connectivity', @@ -405,10 +405,12 @@ def init_functional_connectivity_cifti_wf(mem_gb, exact_scans, name='connectivit ]), ]) # fmt:skip - if config.workflow.output_correlations: + if 'all' in config.workflow.correlation_lengths: # Correlate the parcellated data correlate_bold = pe.MapNode( - CiftiCorrelation(num_threads=config.nipype.omp_nthreads), + CiftiCorrelation( + num_threads=config.nipype.omp_nthreads, + ), name='correlate_bold', iterfield=['in_file'], n_procs=config.nipype.omp_nthreads, diff --git a/xcp_d/workflows/bold/outputs.py b/xcp_d/workflows/bold/outputs.py index b82ba0acf..bb7d709f3 100644 --- a/xcp_d/workflows/bold/outputs.py +++ b/xcp_d/workflows/bold/outputs.py @@ -465,7 +465,7 @@ def init_postproc_derivatives_wf( (ds_timeseries, outputnode, [('out_file', 'timeseries')]), ]) # fmt:skip - if config.workflow.output_correlations: + if 'all' in config.workflow.correlation_lengths: make_corrs_meta_dict1 = pe.MapNode( BIDSURI( numinputs=1, @@ -589,7 +589,7 @@ def init_postproc_derivatives_wf( (ds_timeseries_ciftis, outputnode, [('out_file', 'timeseries_ciftis')]), ]) # fmt:skip - if config.workflow.output_correlations: + if 'all' in config.workflow.correlation_lengths: make_ccorrs_meta_dict1 = pe.MapNode( BIDSURI( numinputs=1, diff --git a/xcp_d/workflows/bold/postprocessing.py b/xcp_d/workflows/bold/postprocessing.py index 874baabdc..24e044473 100644 --- a/xcp_d/workflows/bold/postprocessing.py +++ b/xcp_d/workflows/bold/postprocessing.py @@ -29,7 +29,7 @@ ) from xcp_d.utils.doc import fill_doc from xcp_d.utils.plotting import plot_design_matrix as _plot_design_matrix -from xcp_d.utils.utils import fwhm2sigma +from xcp_d.utils.utils import fwhm2sigma, is_number @fill_doc @@ -327,7 +327,7 @@ def init_prepare_confounds_wf( ]), ]) # fmt:skip - if config.workflow.dcan_correlation_lengths: + if any(is_number(length) for length in config.workflow.correlation_lengths): random_censor = pe.Node( RandomCensor(exact_scans=exact_scans, random_seed=config.seeds.master), name='random_censor',