diff --git a/.circleci/config.yml b/.circleci/config.yml index d6af8986..dec58598 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -702,6 +702,8 @@ workflows: - Recon_PYAFQExternalTrk - Recon_ScalarMap - Recon_multises_pre1_IO_test + - Recon_multises_post1_IO_test + - Recon_msmt_Multishell_HSVS filters: branches: ignore: diff --git a/qsirecon/config.py b/qsirecon/config.py index d9facf2f..0b337e34 100644 --- a/qsirecon/config.py +++ b/qsirecon/config.py @@ -537,7 +537,9 @@ def _process_value(value): cls._processing_list = [] for dwi_relpath, anat_relpath in cls.processing_list: - cls._processing_list.append((cls.layout.get_file(dwi_relpath), cls.layout.get_file(anat_relpath))) + cls._processing_list.append( + (cls.layout.get_file(dwi_relpath), cls.layout.get_file(anat_relpath)) + ) cls.processing_list = cls._processing_list dataset_links = { diff --git a/qsirecon/tests/test_cli.py b/qsirecon/tests/test_cli.py index 156b2e42..0733259c 100644 --- a/qsirecon/tests/test_cli.py +++ b/qsirecon/tests/test_cli.py @@ -9,14 +9,19 @@ from qsirecon.cli import run from qsirecon.cli.parser import parse_args -from qsirecon.cli.workflow import build_boilerplate, build_workflow +from qsirecon.cli.workflow import build_boilerplate, build_workflow, copy_boilerplate from qsirecon.reports.core import generate_reports from qsirecon.tests.utils import ( check_generated_files, download_test_data, + freesurfer_license, get_test_data_path, ) -from qsirecon.utils.bids import write_derivative_description +from qsirecon.utils.bids import ( + write_atlas_dataset_description, + write_bidsignore, + write_derivative_description, +) nipype_config.enable_debug_mode() @@ -58,7 +63,7 @@ def test_mrtrix_singleshell_ss3t_act(data_dir, output_dir, working_dir): "Gordon333Ext", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -95,7 +100,7 @@ def test_mrtrix_multishell_msmt_hsvs(data_dir, output_dir, working_dir): "--report-output-level=root", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -136,7 +141,7 @@ def test_mrtrix_singleshell_ss3t_noact(data_dir, output_dir, working_dir): "--report-output-level=subject", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -160,7 +165,7 @@ def test_multises_post1_qsiprep_reportroot(data_dir, output_dir, working_dir): "--recon-spec=test_workflow", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -185,7 +190,7 @@ def test_multises_post1_qsiprep_reportsubject(data_dir, output_dir, working_dir) "--recon-spec=test_workflow", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -210,7 +215,7 @@ def test_multises_post1_qsiprep_reportsession(data_dir, output_dir, working_dir) "--recon-spec=test_workflow", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -234,7 +239,7 @@ def test_multises_pre1_qsiprep_reportroot(data_dir, output_dir, working_dir): "--recon-spec=test_workflow", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -259,7 +264,7 @@ def test_multises_pre1_qsiprep_reportsubject(data_dir, output_dir, working_dir): "--recon-spec=test_workflow", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -284,7 +289,7 @@ def test_multises_pre1_qsiprep_reportsession(data_dir, output_dir, working_dir): "--recon-spec=test_workflow", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -323,7 +328,7 @@ def test_amico_noddi(data_dir, output_dir, working_dir): "--report-output-level=session", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -362,7 +367,7 @@ def test_autotrack(data_dir, output_dir, working_dir): "--recon-spec=dsi_studio_autotrack", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -404,7 +409,7 @@ def test_dipy_mapmri(data_dir, output_dir, working_dir): "--output-resolution=5", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -445,7 +450,7 @@ def test_dipy_dki(data_dir, output_dir, working_dir): "--recon-spec=dipy_dki", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -478,7 +483,7 @@ def test_scalar_mapper(data_dir, output_dir, working_dir): "--nthreads=1", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -512,7 +517,7 @@ def test_pyafq_recon_external_trk(data_dir, output_dir, working_dir): "--recon-spec=mrtrix_multishell_msmt_pyafq_tractometry", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -546,7 +551,7 @@ def test_pyafq_recon_full(data_dir, output_dir, working_dir): "--recon-spec=pyafq_tractometry", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -586,7 +591,7 @@ def test_mrtrix3_recon(data_dir, output_dir, working_dir): "4S156Parcels", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) @pytest.mark.integration @@ -617,10 +622,10 @@ def test_tortoise_recon(data_dir, output_dir, working_dir): "--recon-spec=TORTOISE", ] - _run_and_generate(TEST_NAME, parameters, test_main=True) + _run_and_generate(TEST_NAME, parameters, test_main=False) -def _run_and_generate(test_name, parameters, test_main=True): +def _run_and_generate(test_name, parameters, test_main=False): from qsirecon import config # TODO: Add --clean-workdir param to CLI @@ -637,25 +642,75 @@ def _run_and_generate(test_name, parameters, test_main=True): assert e.value.code == 0 else: - # XXX: This isn't working because config.execution.fs_license_file is None. parse_args(parameters) config_file = config.execution.work_dir / f"config-{config.execution.run_uuid}.toml" config.loggers.cli.warning(f"Saving config file to {config_file}") + config.execution.fs_license_file = freesurfer_license(config.execution.work_dir) config.to_filename(config_file) retval = build_workflow(config_file, retval={}) qsirecon_wf = retval["workflow"] - qsirecon_wf.run() - write_derivative_description(config.execution.fmri_dir, config.execution.output_dir) - build_boilerplate(str(config_file), qsirecon_wf) - generate_reports( - output_level=config.execution.report_output_level, - output_dir=config.execution.output_dir, - run_uuid=config.execution.run_uuid, - qsirecon_suffix="", + config.loggers.workflow.log( + 15, + "\n".join(["config:"] + ["\t\t%s" % s for s in config.dumps().splitlines()]), + ) + + qsirecon_wf.run(**config.nipype.get_plugin()) + + write_derivative_description( + config.execution.bids_dir, + config.execution.output_dir, + atlases=config.execution.atlases, + dataset_links=config.execution.dataset_links, ) + if config.execution.atlases: + write_atlas_dataset_description(config.execution.output_dir / "atlases") + + # Compile list of output folders + qsirecon_suffixes = config.workflow.qsirecon_suffixes + config.loggers.cli.info(f"QSIRecon pipeline suffixes: {qsirecon_suffixes}") + failed_reports = [] + for qsirecon_suffix in qsirecon_suffixes: + suffix_dir = str( + config.execution.output_dir / "derivatives" / f"qsirecon-{qsirecon_suffix}" + ) + + # Add other pipeline-specific suffixes to the dataset links + other_suffixes = [s for s in qsirecon_suffixes if s != qsirecon_suffix] + dataset_links = config.execution.dataset_links.copy() + dataset_links["qsirecon"] = str(config.execution.output_dir) + dataset_links.update( + { + f"qsirecon-{s}": str( + config.execution.output_dir / "derivatives" / f"qsirecon-{s}" + ) + for s in other_suffixes + } + ) + + # Copy the boilerplate files + copy_boilerplate(config.execution.output_dir, suffix_dir) + + suffix_failed_reports = generate_reports( + output_level=config.execution.report_output_level, + output_dir=suffix_dir, + run_uuid=config.execution.run_uuid, + qsirecon_suffix=qsirecon_suffix, + ) + failed_reports += suffix_failed_reports + write_derivative_description( + config.execution.bids_dir, + suffix_dir, + atlases=config.execution.atlases, + dataset_links=dataset_links, + ) + write_bidsignore(suffix_dir) + + if failed_reports: + print(failed_reports) + output_list_file = os.path.join(get_test_data_path(), f"{test_name}_outputs.txt") optional_outputs_list = os.path.join(get_test_data_path(), f"{test_name}_optional_outputs.txt") if not os.path.isfile(optional_outputs_list): diff --git a/qsirecon/tests/utils.py b/qsirecon/tests/utils.py index e2ab8e28..c8a8efe0 100644 --- a/qsirecon/tests/utils.py +++ b/qsirecon/tests/utils.py @@ -1,5 +1,6 @@ """Utility functions for tests.""" +import base64 import lzma import os import tarfile @@ -139,3 +140,12 @@ def reorder_expected_outputs(): with open(expected_output_file, "w") as fo: fo.writelines(file_contents) + + +def freesurfer_license(base_dir): + """Create a freesurfer license in base_dir.""" + _lic = b"bWFyay5iZXJnbWFuQHVwaHMudXBlbm4uZWR1CjIwMjU5CiAqQ3J2L0QwTXpxcVE2CiBGU3pTdGZYUmlGN2RN" + license_file = base_dir / "license.txt" + with license_file.open("w") as licensef: + licensef.write(base64.b64decode(_lic).decode("utf-8")) + return license_file