From 6b48d5113c218c31f1b745b6452132c05b9d9ac7 Mon Sep 17 00:00:00 2001 From: Nicolas Traut Date: Tue, 14 Mar 2017 13:02:00 +0100 Subject: [PATCH 01/13] Allowing to execute longitudinal steps separately --- run.py | 170 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 90 insertions(+), 80 deletions(-) diff --git a/run.py b/run.py index 151389f..37f2fe8 100755 --- a/run.py +++ b/run.py @@ -24,7 +24,8 @@ def run(command, env={}, ignore_errors=False): if process.returncode != 0 and not ignore_errors: raise Exception("Non zero return code: %d"%process.returncode) -__version__ = open('/version').read() +__version__ = open(os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'version')).read() parser = argparse.ArgumentParser(description='FreeSurfer recon-all + custom template generation.') parser.add_argument('bids_dir', help='The directory with the input dataset ' @@ -51,6 +52,11 @@ def run(command, env={}, ignore_errors=False): choices=["autorecon1", "autorecon2", "autorecon2-cp", "autorecon2-wm", "autorecon-pial", "autorecon3", "autorecon-all", "all"], default=["autorecon-all"], nargs="+") +parser.add_argument('--steps', help='Longitudinal steps to run.', + choices=[1, 2, 3], + default=[1, 2, 3], + type=int, + nargs="+") parser.add_argument('--template_name', help='Name for the custom group level template generated for this dataset', default="average") parser.add_argument('--license_key', help='FreeSurfer license key - letters and numbers after "*" in the email you received after registration. To register (for free) visit https://surfer.nmr.mgh.harvard.edu/registration.html', @@ -154,45 +160,75 @@ def run(command, env={}, ignore_errors=False): timepoints = [] if len(sessions) > 0 and longitudinal_study == True: - # Running each session separately, prior to doing longitudinal pipeline - for session_label in sessions: - T1s = glob(os.path.join(args.bids_dir, - "sub-%s"%subject_label, - "ses-%s"%session_label, - "anat", - "%s_T1w.nii*"%acq_tpl)) - input_args = "" - for T1 in T1s: - if (round(max(nibabel.load(T1).header.get_zooms()),1) < 1.0 and args.hires_mode == "auto") or args.hires_mode == "enable": - input_args += " -hires" - input_args += " -i %s"%T1 - - T2s = glob(os.path.join(args.bids_dir, "sub-%s"%subject_label, - "ses-%s"%session_label, "anat", - "*%s_T2w.nii*"%acq_t2)) - FLAIRs = glob(os.path.join(args.bids_dir, "sub-%s"%subject_label, - "ses-%s"%session_label, "anat", - "*%s_FLAIR.nii*"%acq_t2)) - if args.refine_pial == "T2": - for T2 in T2s: - if max(nibabel.load(T2).header.get_zooms()) < 1.2: - input_args += " " + " ".join(["-T2 %s"%T2]) - input_args += " -T2pial" - elif args.refine_pial == "FLAIR": - for FLAIR in FLAIRs: - if max(nibabel.load(FLAIR).header.get_zooms()) < 1.2: - input_args += " " + " ".join(["-FLAIR %s"%FLAIR]) - input_args += " -FLAIRpial" - - fsid = "sub-%s_ses-%s"%(subject_label, session_label) - timepoints.append(fsid) - cmd = "recon-all -subjid %s -sd %s %s -all -parallel -openmp %d"%(fsid, - output_dir, - input_args, - args.n_cpus) - resume_cmd = "recon-all -subjid %s -sd %s -all -parallel -openmp %d"%(fsid, + timepoints = ["sub-%s_ses-%s"%(subject_label, session_label) for session_label in sessions] + if (1 in args.steps): + # Running each session separately, prior to doing longitudinal pipeline + for session_label in sessions: + T1s = glob(os.path.join(args.bids_dir, + "sub-%s"%subject_label, + "ses-%s"%session_label, + "anat", + "%s_T1w.nii*"%acq_tpl)) + input_args = "" + for T1 in T1s: + if (round(max(nibabel.load(T1).header.get_zooms()),1) < 1.0 and args.hires_mode == "auto") or args.hires_mode == "enable": + input_args += " -hires" + input_args += " -i %s"%T1 + + T2s = glob(os.path.join(args.bids_dir, "sub-%s"%subject_label, + "ses-%s"%session_label, "anat", + "*%s_T2w.nii*"%acq_t2)) + FLAIRs = glob(os.path.join(args.bids_dir, "sub-%s"%subject_label, + "ses-%s"%session_label, "anat", + "*%s_FLAIR.nii*"%acq_t2)) + if args.refine_pial == "T2": + for T2 in T2s: + if max(nibabel.load(T2).header.get_zooms()) < 1.2: + input_args += " " + " ".join(["-T2 %s"%T2]) + input_args += " -T2pial" + elif args.refine_pial == "FLAIR": + for FLAIR in FLAIRs: + if max(nibabel.load(FLAIR).header.get_zooms()) < 1.2: + input_args += " " + " ".join(["-FLAIR %s"%FLAIR]) + input_args += " -FLAIRpial" + + fsid = "sub-%s_ses-%s"%(subject_label, session_label) + timepoints.append(fsid) + cmd = "recon-all -subjid %s -sd %s %s -all -parallel -openmp %d"%(fsid, output_dir, + input_args, args.n_cpus) + resume_cmd = "recon-all -subjid %s -sd %s -all -parallel -openmp %d"%(fsid, + output_dir, + args.n_cpus) + + if os.path.isfile(os.path.join(output_dir, fsid,"scripts/IsRunning.lh+rh")): + rmtree(os.path.join(output_dir, fsid)) + print("DELETING OUTPUT SUBJECT DIR AND RE-RUNNING COMMAND:") + print(cmd) + run(cmd) + elif os.path.exists(os.path.join(output_dir, fsid)): + print("SUBJECT DIR ALREADY EXISTS (without IsRunning.lh+rh), RUNNING COMMAND:") + print(resume_cmd) + run(resume_cmd) + else: + print(cmd) + run(cmd) + + if (2 in args.steps): + # creating a subject specific template + input_args = " ".join(["-tp %s"%tp for tp in timepoints]) + fsid = "sub-%s"%subject_label + stages = " ".join(["-" + stage for stage in args.stages]) + cmd = "recon-all -base %s -sd %s %s %s -parallel -openmp %d"%(fsid, + output_dir, + input_args, + stages, + args.n_cpus) + resume_cmd = "recon-all -base %s -sd %s %s -parallel -openmp %d"%(fsid, + output_dir, + stages, + args.n_cpus) if os.path.isfile(os.path.join(output_dir, fsid,"scripts/IsRunning.lh+rh")): rmtree(os.path.join(output_dir, fsid)) @@ -206,49 +242,23 @@ def run(command, env={}, ignore_errors=False): else: print(cmd) run(cmd) + + if (3 in args.steps): + for tp in timepoints: + # longitudinally process all timepoints + fsid = "sub-%s"%subject_label + stages = " ".join(["-" + stage for stage in args.stages]) + cmd = "recon-all -long %s %s -sd %s %s -parallel -openmp %d"%(tp, + fsid, + output_dir, + stages, + args.n_cpus) - # creating a subject specific template - input_args = " ".join(["-tp %s"%tp for tp in timepoints]) - fsid = "sub-%s"%subject_label - stages = " ".join(["-" + stage for stage in args.stages]) - cmd = "recon-all -base %s -sd %s %s %s -parallel -openmp %d"%(fsid, - output_dir, - input_args, - stages, - args.n_cpus) - resume_cmd = "recon-all -base %s -sd %s %s -parallel -openmp %d"%(fsid, - output_dir, - stages, - args.n_cpus) - - if os.path.isfile(os.path.join(output_dir, fsid,"scripts/IsRunning.lh+rh")): - rmtree(os.path.join(output_dir, fsid)) - print("DELETING OUTPUT SUBJECT DIR AND RE-RUNNING COMMAND:") - print(cmd) - run(cmd) - elif os.path.exists(os.path.join(output_dir, fsid)): - print("SUBJECT DIR ALREADY EXISTS (without IsRunning.lh+rh), RUNNING COMMAND:") - print(resume_cmd) - run(resume_cmd) - else: - print(cmd) - run(cmd) - - for tp in timepoints: - # longitudinally process all timepoints - fsid = "sub-%s"%subject_label - stages = " ".join(["-" + stage for stage in args.stages]) - cmd = "recon-all -long %s %s -sd %s %s -parallel -openmp %d"%(tp, - fsid, - output_dir, - stages, - args.n_cpus) - - if os.path.isfile(os.path.join(output_dir, tp + ".long." + fsid,"scripts/IsRunning.lh+rh")): - rmtree(os.path.join(output_dir, tp + ".long." + fsid)) - print("DELETING OUTPUT SUBJECT DIR AND RE-RUNNING COMMAND:") - print(cmd) - run(cmd) + if os.path.isfile(os.path.join(output_dir, tp + ".long." + fsid,"scripts/IsRunning.lh+rh")): + rmtree(os.path.join(output_dir, tp + ".long." + fsid)) + print("DELETING OUTPUT SUBJECT DIR AND RE-RUNNING COMMAND:") + print(cmd) + run(cmd) elif len(sessions) > 0 and longitudinal_study == False: # grab all T1s/T2s from multiple sessions and combine From 61ddc624b672469d5d344c201c5cc9a1977e4b21 Mon Sep 17 00:00:00 2001 From: Nicolas Traut Date: Tue, 14 Mar 2017 13:05:23 +0100 Subject: [PATCH 02/13] Allowing to execute longitudinal steps separately --- run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/run.py b/run.py index 37f2fe8..39bdae8 100755 --- a/run.py +++ b/run.py @@ -193,7 +193,6 @@ def run(command, env={}, ignore_errors=False): input_args += " -FLAIRpial" fsid = "sub-%s_ses-%s"%(subject_label, session_label) - timepoints.append(fsid) cmd = "recon-all -subjid %s -sd %s %s -all -parallel -openmp %d"%(fsid, output_dir, input_args, From 15bb3405cee41c8c38df91f0efb7b89f1942c379 Mon Sep 17 00:00:00 2001 From: Nicolas Traut Date: Tue, 14 Mar 2017 15:19:36 +0100 Subject: [PATCH 03/13] skipping already segmented subjects --- run.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/run.py b/run.py index 39bdae8..9212624 100755 --- a/run.py +++ b/run.py @@ -201,11 +201,13 @@ def run(command, env={}, ignore_errors=False): output_dir, args.n_cpus) - if os.path.isfile(os.path.join(output_dir, fsid,"scripts/IsRunning.lh+rh")): + if os.path.isfile(os.path.join(output_dir, fsid, "scripts/IsRunning.lh+rh")): rmtree(os.path.join(output_dir, fsid)) print("DELETING OUTPUT SUBJECT DIR AND RE-RUNNING COMMAND:") print(cmd) run(cmd) + elif os.path.isfile(os.path.join(output_dir, fsid, "mri/aseg.mgz")): + print("SUBJECT ALREADY SEGMENTED, SKIPPING") elif os.path.exists(os.path.join(output_dir, fsid)): print("SUBJECT DIR ALREADY EXISTS (without IsRunning.lh+rh), RUNNING COMMAND:") print(resume_cmd) @@ -234,6 +236,8 @@ def run(command, env={}, ignore_errors=False): print("DELETING OUTPUT SUBJECT DIR AND RE-RUNNING COMMAND:") print(cmd) run(cmd) + elif os.path.isfile(os.path.join(output_dir, fsid, "mri/aseg.mgz")): + print("SUBJECT ALREADY SEGMENTED, SKIPPING") elif os.path.exists(os.path.join(output_dir, fsid)): print("SUBJECT DIR ALREADY EXISTS (without IsRunning.lh+rh), RUNNING COMMAND:") print(resume_cmd) @@ -256,8 +260,13 @@ def run(command, env={}, ignore_errors=False): if os.path.isfile(os.path.join(output_dir, tp + ".long." + fsid,"scripts/IsRunning.lh+rh")): rmtree(os.path.join(output_dir, tp + ".long." + fsid)) print("DELETING OUTPUT SUBJECT DIR AND RE-RUNNING COMMAND:") - print(cmd) - run(cmd) + print(cmd) + run(cmd) + elif os.path.isfile(os.path.join(output_dir, fsid, "mri/aseg.mgz")): + print("SUBJECT ALREADY SEGMENTED, SKIPPING") + else: + print(cmd) + run(cmd) elif len(sessions) > 0 and longitudinal_study == False: # grab all T1s/T2s from multiple sessions and combine From 6f2460d29a12d1a9579146414c65c16f62ad0677 Mon Sep 17 00:00:00 2001 From: Nicolas Traut Date: Tue, 14 Mar 2017 16:39:16 +0100 Subject: [PATCH 04/13] skip sessions without T1w images --- run.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/run.py b/run.py index 9212624..4890af2 100755 --- a/run.py +++ b/run.py @@ -155,10 +155,13 @@ def run(command, env={}, ignore_errors=False): ignore_errors=True) for subject_label in subjects_to_analyze: - session_dirs = glob(os.path.join(args.bids_dir,"sub-%s"%subject_label,"ses-*")) - sessions = [os.path.split(dr)[-1].split("-")[-1] for dr in session_dirs] + T1s = glob(os.path.join(args.bids_dir, + "sub-%s"%subject_label, + "ses-*", + "anat", + "%s_T1w.nii*"%acq_tpl)) + sessions = set([os.path.normpath(t1).split(os.sep)[-3].split("-")[-1] for t1 in T1s]) - timepoints = [] if len(sessions) > 0 and longitudinal_study == True: timepoints = ["sub-%s_ses-%s"%(subject_label, session_label) for session_label in sessions] if (1 in args.steps): From 655fa8e8c95f8fcfaeec63dbef93d579f04bd587 Mon Sep 17 00:00:00 2001 From: Nicolas Traut Date: Thu, 16 Mar 2017 14:59:11 +0100 Subject: [PATCH 05/13] allowing to run sessions separately --- run.py | 296 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 155 insertions(+), 141 deletions(-) diff --git a/run.py b/run.py index 4890af2..b5fbd1b 100755 --- a/run.py +++ b/run.py @@ -46,6 +46,12 @@ def run(command, env={}, ignore_errors=False): 'provided all subjects should be analyzed. Multiple ' 'participants can be specified with a space separated list.', nargs="+") +parser.add_argument('--session_label', help='The label of the session that should be analyzed. The label ' + 'corresponds to ses- from the BIDS spec ' + '(so it does not include "ses-"). If this parameter is not ' + 'provided all sessions should be analyzed. Multiple ' + 'sessions can be specified with a space separated list.', + nargs="+") parser.add_argument('--n_cpus', help='Number of CPUs/cores available to use.', default=1, type=int) parser.add_argument('--stages', help='Autorecon stages to run.', @@ -155,56 +161,92 @@ def run(command, env={}, ignore_errors=False): ignore_errors=True) for subject_label in subjects_to_analyze: - T1s = glob(os.path.join(args.bids_dir, - "sub-%s"%subject_label, - "ses-*", - "anat", - "%s_T1w.nii*"%acq_tpl)) - sessions = set([os.path.normpath(t1).split(os.sep)[-3].split("-")[-1] for t1 in T1s]) - - if len(sessions) > 0 and longitudinal_study == True: - timepoints = ["sub-%s_ses-%s"%(subject_label, session_label) for session_label in sessions] - if (1 in args.steps): - # Running each session separately, prior to doing longitudinal pipeline - for session_label in sessions: - T1s = glob(os.path.join(args.bids_dir, - "sub-%s"%subject_label, - "ses-%s"%session_label, - "anat", - "%s_T1w.nii*"%acq_tpl)) - input_args = "" - for T1 in T1s: - if (round(max(nibabel.load(T1).header.get_zooms()),1) < 1.0 and args.hires_mode == "auto") or args.hires_mode == "enable": - input_args += " -hires" - input_args += " -i %s"%T1 - - T2s = glob(os.path.join(args.bids_dir, "sub-%s"%subject_label, - "ses-%s"%session_label, "anat", - "*%s_T2w.nii*"%acq_t2)) - FLAIRs = glob(os.path.join(args.bids_dir, "sub-%s"%subject_label, - "ses-%s"%session_label, "anat", - "*%s_FLAIR.nii*"%acq_t2)) - if args.refine_pial == "T2": - for T2 in T2s: - if max(nibabel.load(T2).header.get_zooms()) < 1.2: - input_args += " " + " ".join(["-T2 %s"%T2]) - input_args += " -T2pial" - elif args.refine_pial == "FLAIR": - for FLAIR in FLAIRs: - if max(nibabel.load(FLAIR).header.get_zooms()) < 1.2: - input_args += " " + " ".join(["-FLAIR %s"%FLAIR]) - input_args += " -FLAIRpial" - - fsid = "sub-%s_ses-%s"%(subject_label, session_label) - cmd = "recon-all -subjid %s -sd %s %s -all -parallel -openmp %d"%(fsid, - output_dir, - input_args, - args.n_cpus) - resume_cmd = "recon-all -subjid %s -sd %s -all -parallel -openmp %d"%(fsid, + if glob(os.path.join(args.bids_dir, "sub-%s"%subject_label, "ses-*")): + T1s = glob(os.path.join(args.bids_dir, + "sub-%s"%subject_label, + "ses-*", + "anat", + "%s_T1w.nii*"%acq_tpl)) + sessions = set([os.path.normpath(t1).split(os.sep)[-3].split("-")[-1] for t1 in T1s]) + if args.session_label: + sessions = sessions.intersect(args.session_label) + + if len(sessions) > 0 and longitudinal_study == True: + timepoints = ["sub-%s_ses-%s"%(subject_label, session_label) for session_label in sessions] + if (1 in args.steps): + # Running each session separately, prior to doing longitudinal pipeline + for session_label in sessions: + T1s = glob(os.path.join(args.bids_dir, + "sub-%s"%subject_label, + "ses-%s"%session_label, + "anat", + "%s_T1w.nii*"%acq_tpl)) + input_args = "" + for T1 in T1s: + if (round(max(nibabel.load(T1).header.get_zooms()),1) < 1.0 and args.hires_mode == "auto") or args.hires_mode == "enable": + input_args += " -hires" + input_args += " -i %s"%T1 + + T2s = glob(os.path.join(args.bids_dir, "sub-%s"%subject_label, + "ses-%s"%session_label, "anat", + "*%s_T2w.nii*"%acq_t2)) + FLAIRs = glob(os.path.join(args.bids_dir, "sub-%s"%subject_label, + "ses-%s"%session_label, "anat", + "*%s_FLAIR.nii*"%acq_t2)) + if args.refine_pial == "T2": + for T2 in T2s: + if max(nibabel.load(T2).header.get_zooms()) < 1.2: + input_args += " " + " ".join(["-T2 %s"%T2]) + input_args += " -T2pial" + elif args.refine_pial == "FLAIR": + for FLAIR in FLAIRs: + if max(nibabel.load(FLAIR).header.get_zooms()) < 1.2: + input_args += " " + " ".join(["-FLAIR %s"%FLAIR]) + input_args += " -FLAIRpial" + + fsid = "sub-%s_ses-%s"%(subject_label, session_label) + stages = " ".join(["-" + stage for stage in args.stages]) + cmd = "recon-all -subjid %s -sd %s %s %s -parallel -openmp %d"%(fsid, output_dir, + input_args, + stages, args.n_cpus) + resume_cmd = "recon-all -subjid %s -sd %s %s -parallel -openmp %d"%(fsid, + output_dir, + stages, + args.n_cpus) + + if os.path.isfile(os.path.join(output_dir, fsid, "scripts/IsRunning.lh+rh")): + rmtree(os.path.join(output_dir, fsid)) + print("DELETING OUTPUT SUBJECT DIR AND RE-RUNNING COMMAND:") + print(cmd) + run(cmd) + elif os.path.isfile(os.path.join(output_dir, fsid, "mri/aseg.mgz")): + print("SUBJECT ALREADY SEGMENTED, SKIPPING") + elif os.path.exists(os.path.join(output_dir, fsid)): + print("SUBJECT DIR ALREADY EXISTS (without IsRunning.lh+rh), RUNNING COMMAND:") + print(resume_cmd) + run(resume_cmd) + else: + print(cmd) + run(cmd) + + if (2 in args.steps): + # creating a subject specific template + input_args = " ".join(["-tp %s"%tp for tp in timepoints]) + fsid = "sub-%s"%subject_label + stages = " ".join(["-" + stage for stage in args.stages]) + cmd = "recon-all -base %s -sd %s %s %s -parallel -openmp %d"%(fsid, + output_dir, + input_args, + stages, + args.n_cpus) + resume_cmd = "recon-all -base %s -sd %s %s -parallel -openmp %d"%(fsid, + output_dir, + stages, + args.n_cpus) - if os.path.isfile(os.path.join(output_dir, fsid, "scripts/IsRunning.lh+rh")): + if os.path.isfile(os.path.join(output_dir, fsid,"scripts/IsRunning.lh+rh")): rmtree(os.path.join(output_dir, fsid)) print("DELETING OUTPUT SUBJECT DIR AND RE-RUNNING COMMAND:") print(cmd) @@ -219,28 +261,79 @@ def run(command, env={}, ignore_errors=False): print(cmd) run(cmd) - if (2 in args.steps): - # creating a subject specific template - input_args = " ".join(["-tp %s"%tp for tp in timepoints]) + if (3 in args.steps): + for tp in timepoints: + # longitudinally process all timepoints + fsid = "sub-%s"%subject_label + stages = " ".join(["-" + stage for stage in args.stages]) + cmd = "recon-all -long %s %s -sd %s %s -parallel -openmp %d"%(tp, + fsid, + output_dir, + stages, + args.n_cpus) + + if os.path.isfile(os.path.join(output_dir, tp + ".long." + fsid,"scripts/IsRunning.lh+rh")): + rmtree(os.path.join(output_dir, tp + ".long." + fsid)) + print("DELETING OUTPUT SUBJECT DIR AND RE-RUNNING COMMAND:") + print(cmd) + run(cmd) + elif os.path.isfile(os.path.join(output_dir, tp + ".long." + fsid, "mri/aseg.mgz")): + print("SUBJECT ALREADY SEGMENTED, SKIPPING") + else: + print(cmd) + run(cmd) + + elif len(sessions) > 0 and longitudinal_study == False: + # grab all T1s/T2s from multiple sessions and combine + T1s = glob(os.path.join(args.bids_dir, + "sub-%s"%subject_label, + "ses-*", + "anat", + "%s_T1w.nii*"%acq_tpl)) + input_args = "" + for T1 in T1s: + if (round(max(nibabel.load(T1).header.get_zooms()),1) < 1.0 and args.hires_mode == "auto") or args.hires_mode == "enable": + input_args += " -hires" + input_args += " -i %s"%T1 + + T2s = glob(os.path.join(args.bids_dir, + "sub-%s"%subject_label, + "ses-*", + "anat", + "*%s_T2w.nii*"%acq_t2)) + FLAIRs = glob(os.path.join(args.bids_dir, + "sub-%s"%subject_label, + "ses-*", + "anat", + "*%s_FLAIR.nii*"%acq_t2)) + if args.refine_pial == "T2": + for T2 in T2s: + if max(nibabel.load(T2).header.get_zooms()) < 1.2: + input_args += " " + " ".join(["-T2 %s"%T2]) + input_args += " -T2pial" + elif args.refine_pial == "FLAIR": + for FLAIR in FLAIRs: + if max(nibabel.load(FLAIR).header.get_zooms()) < 1.2: + input_args += " " + " ".join(["-FLAIR %s"%FLAIR]) + input_args += " -FLAIRpial" + fsid = "sub-%s"%subject_label stages = " ".join(["-" + stage for stage in args.stages]) - cmd = "recon-all -base %s -sd %s %s %s -parallel -openmp %d"%(fsid, - output_dir, - input_args, - stages, - args.n_cpus) - resume_cmd = "recon-all -base %s -sd %s %s -parallel -openmp %d"%(fsid, - output_dir, - stages, - args.n_cpus) + cmd = "recon-all -subjid %s -sd %s %s %s -parallel -openmp %d"%(fsid, + output_dir, + input_args, + stages, + args.n_cpus) + resume_cmd = "recon-all -subjid %s -sd %s %s -parallel -openmp %d"%(fsid, + output_dir, + stages, + args.n_cpus) if os.path.isfile(os.path.join(output_dir, fsid,"scripts/IsRunning.lh+rh")): rmtree(os.path.join(output_dir, fsid)) print("DELETING OUTPUT SUBJECT DIR AND RE-RUNNING COMMAND:") print(cmd) run(cmd) - elif os.path.isfile(os.path.join(output_dir, fsid, "mri/aseg.mgz")): - print("SUBJECT ALREADY SEGMENTED, SKIPPING") elif os.path.exists(os.path.join(output_dir, fsid)): print("SUBJECT DIR ALREADY EXISTS (without IsRunning.lh+rh), RUNNING COMMAND:") print(resume_cmd) @@ -248,87 +341,8 @@ def run(command, env={}, ignore_errors=False): else: print(cmd) run(cmd) - - if (3 in args.steps): - for tp in timepoints: - # longitudinally process all timepoints - fsid = "sub-%s"%subject_label - stages = " ".join(["-" + stage for stage in args.stages]) - cmd = "recon-all -long %s %s -sd %s %s -parallel -openmp %d"%(tp, - fsid, - output_dir, - stages, - args.n_cpus) - - if os.path.isfile(os.path.join(output_dir, tp + ".long." + fsid,"scripts/IsRunning.lh+rh")): - rmtree(os.path.join(output_dir, tp + ".long." + fsid)) - print("DELETING OUTPUT SUBJECT DIR AND RE-RUNNING COMMAND:") - print(cmd) - run(cmd) - elif os.path.isfile(os.path.join(output_dir, fsid, "mri/aseg.mgz")): - print("SUBJECT ALREADY SEGMENTED, SKIPPING") - else: - print(cmd) - run(cmd) - - elif len(sessions) > 0 and longitudinal_study == False: - # grab all T1s/T2s from multiple sessions and combine - T1s = glob(os.path.join(args.bids_dir, - "sub-%s"%subject_label, - "ses-*", - "anat", - "%s_T1w.nii*"%acq_tpl)) - input_args = "" - for T1 in T1s: - if (round(max(nibabel.load(T1).header.get_zooms()),1) < 1.0 and args.hires_mode == "auto") or args.hires_mode == "enable": - input_args += " -hires" - input_args += " -i %s"%T1 - - T2s = glob(os.path.join(args.bids_dir, - "sub-%s"%subject_label, - "ses-*", - "anat", - "*%s_T2w.nii*"%acq_t2)) - FLAIRs = glob(os.path.join(args.bids_dir, - "sub-%s"%subject_label, - "ses-*", - "anat", - "*%s_FLAIR.nii*"%acq_t2)) - if args.refine_pial == "T2": - for T2 in T2s: - if max(nibabel.load(T2).header.get_zooms()) < 1.2: - input_args += " " + " ".join(["-T2 %s"%T2]) - input_args += " -T2pial" - elif args.refine_pial == "FLAIR": - for FLAIR in FLAIRs: - if max(nibabel.load(FLAIR).header.get_zooms()) < 1.2: - input_args += " " + " ".join(["-FLAIR %s"%FLAIR]) - input_args += " -FLAIRpial" - - fsid = "sub-%s"%subject_label - stages = " ".join(["-" + stage for stage in args.stages]) - cmd = "recon-all -subjid %s -sd %s %s %s -parallel -openmp %d"%(fsid, - output_dir, - input_args, - stages, - args.n_cpus) - resume_cmd = "recon-all -subjid %s -sd %s %s -parallel -openmp %d"%(fsid, - output_dir, - stages, - args.n_cpus) - - if os.path.isfile(os.path.join(output_dir, fsid,"scripts/IsRunning.lh+rh")): - rmtree(os.path.join(output_dir, fsid)) - print("DELETING OUTPUT SUBJECT DIR AND RE-RUNNING COMMAND:") - print(cmd) - run(cmd) - elif os.path.exists(os.path.join(output_dir, fsid)): - print("SUBJECT DIR ALREADY EXISTS (without IsRunning.lh+rh), RUNNING COMMAND:") - print(resume_cmd) - run(resume_cmd) else: - print(cmd) - run(cmd) + print("SKIPPING SUBJECT %s (no valid session)." % subject_label) else: # grab all T1s/T2s from single session (no ses-* directories) From 325d0353e3027a75fad80aee653e778560f138dc Mon Sep 17 00:00:00 2001 From: Nicolas Traut Date: Mon, 20 Mar 2017 11:24:05 +0100 Subject: [PATCH 06/13] fix of resume command for template creation in longitudinal pipeline --- run.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/run.py b/run.py index b5fbd1b..a8d5aff 100755 --- a/run.py +++ b/run.py @@ -241,10 +241,10 @@ def run(command, env={}, ignore_errors=False): input_args, stages, args.n_cpus) - resume_cmd = "recon-all -base %s -sd %s %s -parallel -openmp %d"%(fsid, - output_dir, - stages, - args.n_cpus) +# resume_cmd = "recon-all -base %s -sd %s %s -parallel -openmp %d"%(fsid, +# output_dir, +# stages, +# args.n_cpus) if os.path.isfile(os.path.join(output_dir, fsid,"scripts/IsRunning.lh+rh")): rmtree(os.path.join(output_dir, fsid)) @@ -252,11 +252,11 @@ def run(command, env={}, ignore_errors=False): print(cmd) run(cmd) elif os.path.isfile(os.path.join(output_dir, fsid, "mri/aseg.mgz")): - print("SUBJECT ALREADY SEGMENTED, SKIPPING") + print("TEMPLATE ALREADY CREATED, SKIPPING") elif os.path.exists(os.path.join(output_dir, fsid)): print("SUBJECT DIR ALREADY EXISTS (without IsRunning.lh+rh), RUNNING COMMAND:") - print(resume_cmd) - run(resume_cmd) + print(cmd) + run(cmd) else: print(cmd) run(cmd) From 0a1b4e4f31ce2dd487293d5c4ab2058bd25dacd3 Mon Sep 17 00:00:00 2001 From: Nicolas Traut Date: Tue, 21 Mar 2017 16:44:50 +0100 Subject: [PATCH 07/13] bug fix with session_label option --- run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.py b/run.py index a8d5aff..b6ca25f 100755 --- a/run.py +++ b/run.py @@ -169,7 +169,7 @@ def run(command, env={}, ignore_errors=False): "%s_T1w.nii*"%acq_tpl)) sessions = set([os.path.normpath(t1).split(os.sep)[-3].split("-")[-1] for t1 in T1s]) if args.session_label: - sessions = sessions.intersect(args.session_label) + sessions = sessions.intersection(args.session_label) if len(sessions) > 0 and longitudinal_study == True: timepoints = ["sub-%s_ses-%s"%(subject_label, session_label) for session_label in sessions] From 1b223029634b4d9dcbe16fa0e23ce00fdba25b47 Mon Sep 17 00:00:00 2001 From: Nicolas Traut Date: Tue, 16 May 2017 14:27:14 +0200 Subject: [PATCH 08/13] use python2 for freesurfer scripts --- Dockerfile | 6 +----- run.py | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6d7ce1c..1a2bd9d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,7 @@ RUN wget -qO- https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/frees RUN apt-get install -y python3 RUN apt-get install -y python3-pip RUN pip3 install nibabel +RUN apt-get install -y python2 RUN apt-get install -y tcsh RUN apt-get install -y bc @@ -60,11 +61,6 @@ ENV PATH /opt/freesurfer/bin:/opt/freesurfer/fsfast/bin:/opt/freesurfer/tktools: ENV PYTHONPATH="" RUN echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh -# make freesurfer python scripts python3 ready -RUN 2to3-3.4 -w $FREESURFER_HOME/bin/aparcstats2table -RUN 2to3-3.4 -w $FREESURFER_HOME/bin/asegstats2table -RUN 2to3-3.4 -w $FREESURFER_HOME/bin/*.py - RUN mkdir /scratch RUN mkdir /local-scratch diff --git a/run.py b/run.py index b6ca25f..0c0d8fa 100755 --- a/run.py +++ b/run.py @@ -451,7 +451,7 @@ def run(command, env={}, ignore_errors=False): if os.path.isfile(table_file): warn("Replace old file %s" % table_file) os.remove(table_file) - cmd = "python3 `which aparcstats2table` --hemi {h} --subjects {subjects} --parc {p} --meas {m} " \ + cmd = "python2 `which aparcstats2table` --hemi {h} --subjects {subjects} --parc {p} --meas {m} " \ "--tablefile {table_file}".format(h=h, subjects=subjects_str, p=p, m=m, table_file=table_file) print("Creating cortical stats table for {h} {p} {m}".format(h=h, p=p, m=m)) @@ -462,7 +462,7 @@ def run(command, env={}, ignore_errors=False): if os.path.isfile(table_file): warn("Replace old file %s" % table_file) os.remove(table_file) - cmd = "python3 `which asegstats2table` --subjects {subjects} --meas volume --tablefile {" \ + cmd = "python2 `which asegstats2table` --subjects {subjects} --meas volume --tablefile {" \ "table_file}".format(subjects=subjects_str, table_file=table_file) print("Creating subcortical stats table.") run(cmd, env={"SUBJECTS_DIR": output_dir}) From 7d14165f9388b923c978edeae1f94112e04bff04 Mon Sep 17 00:00:00 2001 From: Nicolas Traut Date: Tue, 16 May 2017 15:09:07 +0200 Subject: [PATCH 09/13] skip subject if he does not have a T1 image in single session studies --- run.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run.py b/run.py index 0c0d8fa..4cfac7a 100755 --- a/run.py +++ b/run.py @@ -350,6 +350,9 @@ def run(command, env={}, ignore_errors=False): "sub-%s"%subject_label, "anat", "%s_T1w.nii*"%acq_tpl)) + if not T1s: + print("No T1w nii files found for subject %s. Skipping subject." % subject_label) + continue input_args = "" for T1 in T1s: if (round(max(nibabel.load(T1).header.get_zooms()),1) < 1.0 and args.hires_mode == "auto") or args.hires_mode == "enable": From 8393a9d7380a0e0b26f429a99530cae5ee40dd37 Mon Sep 17 00:00:00 2001 From: Nicolas Traut Date: Tue, 16 May 2017 21:12:04 +0200 Subject: [PATCH 10/13] fixing python 2 ubuntu package name --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1a2bd9d..e2f34e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ RUN wget -qO- https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.0/frees RUN apt-get install -y python3 RUN apt-get install -y python3-pip RUN pip3 install nibabel -RUN apt-get install -y python2 +RUN apt-get install -y python2.7 RUN apt-get install -y tcsh RUN apt-get install -y bc From 432c72612da7e2a8186b6aad3bc11fdd4ae8980a Mon Sep 17 00:00:00 2001 From: Nicolas Traut Date: Thu, 1 Jun 2017 11:46:47 +0200 Subject: [PATCH 11/13] renaming longitudinal pipeline steps names --- run.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/run.py b/run.py index 492d571..a8cbfc3 100755 --- a/run.py +++ b/run.py @@ -57,10 +57,9 @@ def run(command, env={}, ignore_errors=False): choices=["autorecon1", "autorecon2", "autorecon2-cp", "autorecon2-wm", "autorecon-pial", "autorecon3", "autorecon-all", "all"], default=["autorecon-all"], nargs="+") -parser.add_argument('--steps', help='Longitudinal steps to run.', - choices=[1, 2, 3], - default=[1, 2, 3], - type=int, +parser.add_argument('--steps', help='Longitudinal pipeline steps to run.', + choices=['cross-sectional', 'template', 'longitudinal'], + default=['cross-sectional', 'template', 'longitudinal'], nargs="+") parser.add_argument('--template_name', help='Name for the custom group level template generated for this dataset', default="average") @@ -172,7 +171,7 @@ def run(command, env={}, ignore_errors=False): if len(sessions) > 0 and longitudinal_study == True: timepoints = ["sub-%s_ses-%s"%(subject_label, session_label) for session_label in sessions] - if (1 in args.steps): + if ('cross-sectional' in args.steps): # Running each session separately, prior to doing longitudinal pipeline for session_label in sessions: T1s = glob(os.path.join(args.bids_dir, @@ -230,7 +229,7 @@ def run(command, env={}, ignore_errors=False): print(cmd) run(cmd) - if (2 in args.steps): + if ('template' in args.steps): # creating a subject specific template input_args = " ".join(["-tp %s"%tp for tp in timepoints]) fsid = "sub-%s"%subject_label @@ -240,10 +239,6 @@ def run(command, env={}, ignore_errors=False): input_args, stages, args.n_cpus) -# resume_cmd = "recon-all -base %s -sd %s %s -parallel -openmp %d"%(fsid, -# output_dir, -# stages, -# args.n_cpus) if os.path.isfile(os.path.join(output_dir, fsid,"scripts/IsRunning.lh+rh")): rmtree(os.path.join(output_dir, fsid)) @@ -260,7 +255,7 @@ def run(command, env={}, ignore_errors=False): print(cmd) run(cmd) - if (3 in args.steps): + if ('longitudinal' in args.steps): for tp in timepoints: # longitudinally process all timepoints fsid = "sub-%s"%subject_label From 88f103882b785bd3a44e6963f424c0bc1e6fdd27 Mon Sep 17 00:00:00 2001 From: Nicolas Traut Date: Thu, 1 Jun 2017 14:23:18 +0200 Subject: [PATCH 12/13] adding a test for longitudinal pipeline decomposition --- circle.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/circle.yml b/circle.yml index 28068db..cced8d9 100644 --- a/circle.yml +++ b/circle.yml @@ -29,6 +29,8 @@ test: - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test1:/bids_dataset bids/${CIRCLE_PROJECT_REPONAME,,} --version - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test1:/bids_dataset -v ${HOME}/outputs1:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 01 --license_key="~/test.key" --stages autorecon1 && cat ${HOME}/outputs1/sub-01/scripts/recon-all.done : timeout: 21600 + - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test2:/bids_dataset -v ${HOME}/outputs2:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 01 --steps cross-sectional --session_label test --license_key="~/test.key" --stages autorecon1 && cat ${HOME}/outputs2/sub-01/scripts/recon-all.done : + timeout: 21600 # group2 tests - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test1:/bids_dataset -v ${HOME}/data/ds114_test1_freesurfer_precomp_v6.0.0:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs group2 --license_key="~/test.key" && mkdir -p ${HOME}/outputs1/ && sudo mv ${HOME}/data/ds114_test1_freesurfer_precomp_v6.0.0/00_group* ${HOME}/outputs1/ && cat ${HOME}/outputs1/00_group2_stats_tables/lh.aparc.thickness.tsv : timeout: 21600 From bf10e11a316f166387c10cee9dfd328af8ac8abd Mon Sep 17 00:00:00 2001 From: Nicolas Traut Date: Thu, 1 Jun 2017 16:09:42 +0200 Subject: [PATCH 13/13] adding a test for longitudinal pipeline decomposition --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index cced8d9..c25993c 100644 --- a/circle.yml +++ b/circle.yml @@ -29,7 +29,7 @@ test: - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test1:/bids_dataset bids/${CIRCLE_PROJECT_REPONAME,,} --version - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test1:/bids_dataset -v ${HOME}/outputs1:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 01 --license_key="~/test.key" --stages autorecon1 && cat ${HOME}/outputs1/sub-01/scripts/recon-all.done : timeout: 21600 - - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test2:/bids_dataset -v ${HOME}/outputs2:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 01 --steps cross-sectional --session_label test --license_key="~/test.key" --stages autorecon1 && cat ${HOME}/outputs2/sub-01/scripts/recon-all.done : + - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test2:/bids_dataset -v ${HOME}/outputs2:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs participant --participant_label 01 --steps cross-sectional --session_label test --license_key="~/test.key" --stages autorecon1 && cat ${HOME}/outputs2/sub-01_ses-test/scripts/recon-all.done : timeout: 21600 # group2 tests - docker run -ti --rm --read-only -v /tmp:/tmp -v /var/tmp:/var/tmp -v ${HOME}/data/ds114_test1:/bids_dataset -v ${HOME}/data/ds114_test1_freesurfer_precomp_v6.0.0:/outputs bids/${CIRCLE_PROJECT_REPONAME,,} /bids_dataset /outputs group2 --license_key="~/test.key" && mkdir -p ${HOME}/outputs1/ && sudo mv ${HOME}/data/ds114_test1_freesurfer_precomp_v6.0.0/00_group* ${HOME}/outputs1/ && cat ${HOME}/outputs1/00_group2_stats_tables/lh.aparc.thickness.tsv :