From f1458cdd4e1cc4588285a2ba62e319cba03b5ba7 Mon Sep 17 00:00:00 2001 From: Michael Riedel Date: Tue, 7 May 2019 11:59:45 -0400 Subject: [PATCH 1/6] Added cis.py --- cis.py | 192 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 cis.py diff --git a/cis.py b/cis.py new file mode 100644 index 0000000..65bb3f4 --- /dev/null +++ b/cis.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +""" +Based on +https://github.com/BIDS-Apps/example/blob/aa0d4808974d79c9fbe54d56d3b47bb2cf4e0a0d/run.py +""" +import os +import os.path as op +import json +import shutil +import tarfile +import subprocess +from glob import glob +import datetime +import getpass + +import argparse +import pandas as pd + + +def run(command, env={}): + merged_env = os.environ + merged_env.update(env) + process = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, shell=True, + env=merged_env) + while True: + line = process.stdout.readline() + #line = str(line).encode('utf-8')[:-1] + line=str(line, 'utf-8')[:-1] + print(line) + if line == '' and process.poll() is not None: + break + + if process.returncode != 0: + raise Exception("Non zero return code: {0}\n" + "{1}\n\n{2}".format(process.returncode, command, + process.stdout.read())) + + +def get_parser(): + parser = argparse.ArgumentParser(description='Initiate XNAT download and CIS proc.') + parser.add_argument('-b', '--bidsdir', required=True, dest='bids_dir', + help=('Output directory for BIDS dataset and ' + 'derivatives.')) + parser.add_argument('-w', '--workdir', required=False, dest='work_dir', + default=None, + help='Path to a working directory. Defaults to work ' + 'subfolder in dset_dir.') + parser.add_argument('--config', required=True, dest='config', + help='Path to the config json file.') + parser.add_argument('--protocol_check', required=False, action='store_true', + help='Will perform a protocol check to determine if the correct number of scans and TRs are present.') + parser.add_argument('--autocheck', required=False, action='store_true', + help='Will automatically download all scans from XNAT that are not currently in the project folder.') + parser.add_argument('--xnat_experiment', required=False, dest='xnatexp', + default=None, + help='XNAT Experiment ID (i.e., XNAT_E*) for single session download.') + parser.add_argument('--n_procs', required=False, dest='n_procs', + help='Number of processes with which to run MRIQC.', + default=1, type=int) + return parser + + +def main(argv=None): + args = get_parser().parse_args(argv) + + CIS_DIR = '/scratch/cis_dataqc/' + + # Check inputs + if args.work_dir is None: + args.work_dir = CIS_DIR + + proj_dir = os.path.dirname(args.bids_dir) + if not op.isdir(proj_dir): + raise ValueError('Project directory must be an existing directory!') + + if not op.isfile(args.config): + raise ValueError('Argument "config" must be an existing file.') + + if args.n_procs < 1: + raise ValueError('Argument "n_procs" must be positive integer greater ' + 'than zero.') + else: + n_procs = int(args.n_procs) + + with open(args.config, 'r') as fo: + config_options = json.load(fo) + + if 'project' not in config_options.keys(): + raise Exception('Config File must be updated with project field' + 'See Sample Config File for More information') + + proj_work_dir = op.join(args.work_dir, + '{0}'.format(config_options['project'])) + if not proj_work_dir.startswith('/scratch'): + raise ValueError('Working directory must be in scratch.') + + xnatdownload_file = op.join('/home/data/cis/singularity-images/', + config_options['xnatdownload']) + + # Additional checks and copying for XNAT Download file + if not op.isfile(xnatdownload_file): + raise ValueError('XNAT Download image specified in config files must be ' + 'an existing file.') + + + # Make folders/files + if not op.isdir(op.join(proj_dir, 'code/err')): + os.makedirs(op.join(proj_dir, 'code/err')) + + if not op.isdir(op.join(proj_dir, 'code/out')): + os.makedirs(op.join(proj_dir, 'code/out')) + + if not op.isdir(proj_work_dir): + os.makedirs(proj_work_dir) + + raw_dir = op.join(proj_dir, 'raw') + if not op.isdir(raw_dir): + os.makedirs(raw_dir) + + tar_list = os.listdir(raw_dir) + with open(op.join(proj_work_dir, '{0}-processed.txt'.format(config_options['project'])), 'a') as fo: + for tmp_tar_list in tar_list: + fo.write(tmp_tar_list + "\n") + + # Copy singularity images to scratch + scratch_xnatdownload = op.join(args.work_dir, op.basename(xnatdownload_file)) + if not op.isfile(scratch_xnatdownload): + shutil.copyfile(xnatdownload_file, scratch_xnatdownload) + os.chmod(scratch_xnatdownload, 0o775) + + # Run XNAT Download + if args.autocheck: + cmd = ('{sing} -w {work} --project {proj} --autocheck --processed {tar_list}'.format(sing=scratch_xnatdownload, work=proj_work_dir, proj=config_options['project'], tar_list=op.join(proj_work_dir, '{0}-processed.txt'.format(config_options['project'])))) + run(cmd) + elif args.xnatexp is not None: + cmd = ('{sing} -w {work} --project {proj} --session {xnat_exp} --processed {tar_list}'.format(sing=scratch_xnatdownload, work=proj_work_dir, proj=config_options['project'], xnat_exp=args.xnatexp, tar_list=op.join(proj_work_dir, '{0}-processed.txt'.format(config_options['project'])))) + run(cmd) + else: + raise Exception('A valid XNAT Experiment session was not entered for the project or you are not running autocheck.') + + os.remove(op.join(proj_work_dir, '{0}-processed.txt'.format(config_options['project']))) + os.remove(scratch_xnatdownload) + # Temporary raw directory in work_dir + raw_work_dir = op.join(proj_work_dir, 'raw') + + if op.isdir(raw_work_dir): + # Check if anything was downloaded + data_download = False + if os.listdir(raw_work_dir): + data_download = True + + if data_download: + + sub_list = os.listdir(raw_work_dir) + for tmp_sub in sub_list: + ses_list = os.listdir(op.join(raw_work_dir, '{0}'.format(tmp_sub))) + + for tmp_ses in ses_list: + #run the protocol check if requested + if args.protocol_check: + cmd = ('python /home/data/cis/cis-processing/protocol_check.py -w {work} --bids_dir {bids_dir} --sub {sub} --ses {ses}'.format(work=raw_work_dir, bids_dir = args.bids_dir, sub=tmp_sub, ses=tmp_ses)) + run(cmd) + + #tar the subject and session directory and copy to raw dir + with tarfile.open(op.join(raw_dir, '{sub}-{ses}.tar'.format(sub=tmp_sub, ses=tmp_ses)), "w") as tar: + tar.add(op.join(raw_work_dir, '{sub}'.format(sub=tmp_sub)), arcname=os.path.basename(op.join(raw_work_dir, '{sub}'.format(sub=tmp_sub)))) + shutil.rmtree(op.join(raw_work_dir, '{sub}'.format(sub=tmp_sub))) + + # run cis_proc.py + #cmd = ('python cis_proc.py -t {tarfile} -b {bidsdir} -w {work} --config {config} --sub {sub} --ses {ses} --n_procs {nprocs}'. format(tarfile=op.join(raw_dir, '{sub}-{ses}.tar'.format(sub=tmp_sub, ses=tmp_ses)), bidsdir=args.bids_dir, work=proj_work_dir, config=args.config, sub=tmp_sub.strip('sub-'), ses=tmp_ses.strip('ses-'), nprocs=args.n_procs)) + #cmd = ('srun -J cis_proc-{proj}-{sub}-{ses} -e {err_file_loc} -o {out_file_loc} -c {nprocs} -q {hpc_queue} -p investor python /home/data/cis/cis-processing/cis_proc.py -t {tarfile} -b {bidsdir} -w {work} --config {config} --sub {sub} --ses {ses} --n_procs {nprocs}'. format(hpc_queue=config_options['hpc_queue'], proj=config_options['project'], err_file_loc = op.join('/home/data/cis/cis-processing/err', config_options['project'], 'cis_proc-{sub}-{ses}'.format(sub=tmp_sub, ses=tmp_ses)), out_file_loc= op.join('/home/data/cis/cis-processing/out', config_options['project'], 'cis_proc-{sub}-{ses}'.format(sub=tmp_sub, ses=tmp_ses)), tarfile=op.join(raw_dir, '{sub}-{ses}.tar'.format(sub=tmp_sub, ses=tmp_ses)), bidsdir=args.bids_dir, work=proj_work_dir, config=args.config, sub=tmp_sub.strip('sub-'), ses=tmp_ses.strip('ses-'), nprocs=args.n_procs)) + cmd = ('bsub -J cis_proc-{proj}-{sub}-{ses} -eo {err_file_loc} -oo {out_file_loc} -n {nprocs} -q {hpc_queue} python /home/data/cis/cis-processing/cis_proc_mcr.py -t {tarfile} -b {bidsdir} -w {work} --config {config} --sub {sub} --ses {ses} --n_procs {nprocs}'. format(hpc_queue=config_options['hpc_queue'], proj=config_options['project'], err_file_loc = op.join(proj_dir, 'code/err', 'cis_proc-{sub}-{ses}'.format(sub=tmp_sub, ses=tmp_ses)), out_file_loc= op.join(proj_dir, 'code/out', 'cis_proc-{sub}-{ses}'.format(sub=tmp_sub, ses=tmp_ses)), tarfile=op.join(raw_dir, '{sub}-{ses}.tar'.format(sub=tmp_sub, ses=tmp_ses)), bidsdir=args.bids_dir, work=proj_work_dir, config=args.config, sub=tmp_sub.strip('sub-'), ses=tmp_ses.strip('ses-'), nprocs=args.n_procs)) + run(cmd) + + # get date and time + now = datetime.datetime.now() + date_time=now.strftime("%Y-%m-%d %H:%M") + # append the email message + + with open(op.join(proj_work_dir, '{0}-processed-message.txt'.format(config_options['project'])), 'a') as fo: + fo.write('Data transferred from XNAT to FIU-HPC for Project: {proj} Subject: {sub} Session: {ses} on {datetime}\n'.format(proj=config_options['project'], sub=tmp_sub, ses=tmp_ses, datetime=date_time)) + + + shutil.rmtree(op.join(raw_work_dir)) + + cmd = ("mail -s 'FIU XNAT-HPC Data Transfer Update Project {proj}' {email_list} < {message}".format(proj=config_options['project'], email_list=config_options['email'], message=op.join(proj_work_dir, '{0}-processed-message.txt'.format(config_options['project'])))) + run(cmd) + os.remove(op.join(proj_work_dir, '{0}-processed-message.txt'.format(config_options['project']))) + +if __name__ == '__main__': + main() From 441ccc82341b7894a09372cdb6532c76f29255b3 Mon Sep 17 00:00:00 2001 From: Michael Riedel Date: Tue, 7 May 2019 12:00:41 -0400 Subject: [PATCH 2/6] Update cis_proc.py --- cis_proc.py | 86 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/cis_proc.py b/cis_proc.py index c51b016..d9f0849 100755 --- a/cis_proc.py +++ b/cis_proc.py @@ -23,7 +23,8 @@ def run(command, env={}): env=merged_env) while True: line = process.stdout.readline() - line = str(line).encode('utf-8')[:-1] + #line = str(line).encode('utf-8')[:-1] + line=str(line, 'utf-8')[:-1] print(line) if line == '' and process.poll() is not None: break @@ -109,8 +110,7 @@ def main(argv=None): # Additional checks and copying for heuristics file heuristics_file = config_options['heuristics'] if not heuristics_file.startswith('/'): - heuristics_file = op.join('/home/data/cis/cis-processing', - heuristics_file) + heuristics_file = op.join(os.path.dirname(args.bids_dir), heuristics_file) if not op.isfile(heuristics_file): raise ValueError('Heuristics file specified in config files must be ' @@ -210,11 +210,31 @@ def main(argv=None): out_ses_dir = op.join(out_sub_dir, 'ses-{0}'.format(args.ses)) if not op.isdir(out_ses_dir): shutil.copytree(scratch_ses_dir, out_ses_dir) + else: print('Warning: Subject/session directory already exists in ' 'dataset.') else: print('Warning: Subject directory already exists in dataset.') + + if args.ses is not None: + tmp_df = pd.read_csv(op.join(out_sub_dir, 'ses-{ses}'.format(ses=args.ses), 'sub-{sub}_ses-{ses}_scans.tsv'.format(sub=args.sub, ses=args.ses)), sep='\t') + else: + tmp_df = pd.read_csv(op.join(out_sub_dir, 'sub-{sub}_scans.tsv'.format(sub=args.sub)), sep='\t') + + #append scans.tsv file with remove and annot fields + tmp_df['remove'] = 0 + tmp_df['annotation'] = '' + + #import master scans file + if op.isfile(op.join(os.path.dirname(args.bids_dir), 'code/{proj}_scans.tsv'.format(proj=config_options['project']))): + master_df = pd.read_csv(op.join(os.path.dirname(args.bids_dir), 'code/{proj}_scans.tsv'.format(proj=config_options['project'])), sep='\t') + master_df_headers = list(master_df) + master_df = master_df.append(tmp_df) + master_df.to_csv(op.join(os.path.dirname(args.bids_dir), 'code/{proj}_scans.tsv'.format(proj=config_options['project'])), sep='\t', index=False, columns=master_df_headers) + else: + tmp_df_headers = list(tmp_df) + tmp_df.to_csv(op.join(os.path.dirname(args.bids_dir), 'code/{proj}_scans.tsv'.format(proj=config_options['project'])), sep='\t', index=False, columns=tmp_df_headers) # Run MRIQC if not op.isdir(out_deriv_dir): @@ -229,20 +249,54 @@ def main(argv=None): if not op.isdir(op.join(out_deriv_dir, 'reports')): os.makedirs(op.join(out_deriv_dir, 'reports')) - kwargs = '' - for field in config_options['mriqc_settings'].keys(): - if isinstance(config_options['mriqc_settings'][field], list): - val = ' '.join(config_options['mriqc_settings'][field]) + # Run MRIQC anat + for tmp_mod in config_options['mriqc_options']['anat']['mod'].keys(): + kwargs = '' + for field in config_options['mriqc_options']['anat']['mod'][tmp_mod]['mriqc_settings'].keys(): + if isinstance(config_options['mriqc_options']['anat']['mod'][tmp_mod]['mriqc_settings'][field], list): + val = ' '.join(config_options['mriqc_options']['anat']['mod'][tmp_mod]['mriqc_settings'][field]) + else: + val = config_options['mriqc_options']['anat']['mod'][tmp_mod]['mriqc_settings'][field] + kwargs += '--{0} {1} '.format(field, val) + kwargs = kwargs.rstrip() + cmd = ('{sing} {bids} {out} participant --no-sub --verbose-reports ' + '-m {mod} ' + '-w {work} --n_procs {n_procs} ' + '{kwargs} '.format(sing=scratch_mriqc, bids=scratch_bids_dir, + out=scratch_deriv_dir, mod=tmp_mod, + work=mriqc_work_dir, n_procs=n_procs, kwargs=kwargs)) + run(cmd) + + # Run MRIQC func + for tmp_task in config_options['mriqc_options']['func']['task'].keys(): + print(op.join(scratch_bids_dir, 'sub-{sub}'.format(sub=args.sub), 'ses-{ses}'.format(ses=args.ses), 'func/sub-{sub}_ses-{ses}_task-{task}_run-01_bold.json'.format(sub=args.sub, ses=args.ses, task=tmp_task))) + if args.ses is not None: + if op.isfile(op.join(scratch_bids_dir, 'sub-{sub}'.format(sub=args.sub), 'ses-{ses}'.format(ses=args.ses), 'func/sub-{sub}_ses-{ses}_task-{task}_run-01_bold.json'.format(sub=args.sub, ses=args.ses, task=tmp_task))): + run_mriqc=True + else: + run_mriqc=False else: - val = config_options['mriqc_settings'][field] - kwargs += '--{0} {1} '.format(field, val) - kwargs = kwargs.rstrip() - cmd = ('{sing} {bids} {out} participant --no-sub --verbose-reports ' - '--ica --correct-slice-timing -w {work} --n_procs {n_procs} ' - '{kwargs} '.format(sing=scratch_mriqc, bids=scratch_bids_dir, - out=scratch_deriv_dir, work=mriqc_work_dir, - n_procs=n_procs, kwargs=kwargs)) - run(cmd) + if op.isfile(op.join(scratch_bids_dir, 'sub-{sub}'.format(sub=args.sub), 'func/sub-{sub}_task-{task}_run-01_bold.json'.format(sub=args.sub, task=tmp_task))): + run_mriqc=True + else: + run_mriqc=False + + if run_mriqc: + kwargs = '' + for field in config_options['mriqc_options']['func']['task'][tmp_task]['mriqc_settings'].keys(): + if isinstance(config_options['mriqc_options']['func']['task'][tmp_task]['mriqc_settings'][field], list): + val = ' '.join(config_options['mriqc_options']['func']['task'][tmp_task]['mriqc_settings'][field]) + else: + val = config_options['mriqc_options']['func']['task'][tmp_task]['mriqc_settings'][field] + kwargs += '--{0} {1} '.format(field, val) + kwargs = kwargs.rstrip() + cmd = ('{sing} {bids} {out} participant --no-sub --verbose-reports ' + '--task-id {task} -m bold ' + '-w {work} --n_procs {n_procs} --correct-slice-timing ' + '{kwargs} '.format(sing=scratch_mriqc, bids=scratch_bids_dir, + out=scratch_deriv_dir, task=tmp_task, + work=mriqc_work_dir, n_procs=n_procs, kwargs=kwargs)) + run(cmd) # Merge MRIQC results into final derivatives folder reports = glob(op.join(scratch_deriv_dir, 'reports/*.html')) From ef68abd50ec2116f321e8481c1dc7a7afb4fd28c Mon Sep 17 00:00:00 2001 From: Michael Riedel Date: Tue, 7 May 2019 12:01:59 -0400 Subject: [PATCH 3/6] Added protocol_check and mriqc_group --- mriqc_group.py | 151 ++++++++++++++++++++++++++++++++++++++++++++++ protocol_check.py | 84 ++++++++++++++++++++++++++ 2 files changed, 235 insertions(+) create mode 100644 mriqc_group.py create mode 100644 protocol_check.py diff --git a/mriqc_group.py b/mriqc_group.py new file mode 100644 index 0000000..aa89744 --- /dev/null +++ b/mriqc_group.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +""" +Based on +https://github.com/BIDS-Apps/example/blob/aa0d4808974d79c9fbe54d56d3b47bb2cf4e0a0d/run.py +""" +import os +import os.path as op +import json +import shutil +import tarfile +import subprocess +from glob import glob +import datetime + +import argparse +import pandas as pd + + +def run(command, env={}): + merged_env = os.environ + merged_env.update(env) + process = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, shell=True, + env=merged_env) + while True: + line = process.stdout.readline() + #line = str(line).encode('utf-8')[:-1] + line=str(line, 'utf-8')[:-1] + print(line) + if line == '' and process.poll() is not None: + break + + if process.returncode != 0: + raise Exception("Non zero return code: {0}\n" + "{1}\n\n{2}".format(process.returncode, command, + process.stdout.read())) + + +def get_parser(): + parser = argparse.ArgumentParser(description='Run MRIQC on BIDS dataset.') + parser.add_argument('-b', '--bidsdir', required=True, dest='bids_dir', + help=('Output directory for BIDS dataset and ' + 'derivatives.')) + parser.add_argument('-w', '--workdir', required=False, dest='work_dir', + default=None, + help='Path to a working directory. Defaults to work ' + 'subfolder in dset_dir.') + parser.add_argument('--config', required=True, dest='config', + help='Path to the config json file.') + parser.add_argument('--sub', required=False, dest='sub', + help='The label of the subject to analyze.') + parser.add_argument('--ses', required=False, dest='ses', + help='Session number', default=None) + parser.add_argument('--participant', required=False, action='store_true', + help='Run participant level MRIQC') + parser.add_argument('--group', required=False, action='store_true', + help='Run group level MRIQC') + parser.add_argument('--n_procs', required=False, dest='n_procs', + help='Number of processes with which to run MRIQC.', + default=1, type=int) + return parser + + +def main(argv=None): + args = get_parser().parse_args(argv) + + CIS_DIR = '/scratch/cis_dataqc/' + + # Check inputs + if args.work_dir is None: + args.work_dir = CIS_DIR + + if not op.isfile(args.config): + raise ValueError('Argument "config" must be an existing file.') + + if args.n_procs < 1: + raise ValueError('Argument "n_procs" must be positive integer greater ' + 'than zero.') + else: + n_procs = int(args.n_procs) + + with open(args.config, 'r') as fo: + config_options = json.load(fo) + + if 'project' not in config_options.keys(): + raise Exception('Config File must be updated with project field' + 'See Sample Config File for More information') + + if not args.work_dir.startswith('/scratch'): + raise ValueError('Working directory must be in scratch.') + + + mriqc_file = op.join('/home/data/cis/singularity-images/', + config_options['mriqc']) + mriqc_version = op.basename(mriqc_file).split('-')[0].split('_')[-1] + + out_deriv_dir = op.join(args.bids_dir, + 'derivatives/mriqc-{0}'.format(mriqc_version)) + + scratch_deriv_dir = op.join(args.work_dir, 'mriqc') + scratch_mriqc_work_dir = op.join(args.work_dir, 'mriqc-wkdir') + + if not op.isfile(mriqc_file): + raise ValueError('MRIQC image specified in config files must be ' + 'an existing file.') + + # Copy singularity images to scratch + scratch_mriqc = op.join(CIS_DIR, op.basename(mriqc_file)) + + if not op.isfile(scratch_mriqc): + shutil.copyfile(mriqc_file, scratch_mriqc) + os.chmod(scratch_mriqc, 0o775) + + if args.group: + shutil.copytree(out_deriv_dir, scratch_deriv_dir) + cmd = ('{sing} {bids} {out} group --no-sub --verbose-reports ' + '-w {work} --n_procs {n_procs} '.format(sing=scratch_mriqc, bids=args.bids_dir, + out=scratch_deriv_dir, + work=scratch_mriqc_work_dir, n_procs=n_procs)) + run(cmd) + + if op.isfile(op.join(scratch_deriv_dir, 'bold.csv')): + shutil.copy(op.join(scratch_deriv_dir, 'bold.csv'), out_deriv_dir) + shutil.copy(op.join(scratch_deriv_dir, 'reports/bold_group.html'), op.join(out_deriv_dir, 'reports')) + + if op.isfile(op.join(scratch_deriv_dir, 'T1w.csv')): + shutil.copy(op.join(scratch_deriv_dir, 'T1w.csv'), out_deriv_dir) + shutil.copy(op.join(scratch_deriv_dir, 'reports/T1w_group.html'), op.join(out_deriv_dir, 'reports')) + + if op.isfile(op.join(scratch_deriv_dir, 'T2w.csv')): + shutil.copy(op.join(scratch_deriv_dir, 'T2w.csv'), out_deriv_dir) + shutil.copy(op.join(scratch_deriv_dir, 'reports/T2w_group.html'), op.join(out_deriv_dir, 'reports')) + + # get date and time + now = datetime.datetime.now() + date_time=now.strftime("%Y-%m-%d %H:%M") + # append the email message + + with open(op.join(args.work_dir, '{0}-mriqc-message.txt'.format(config_options['project'])), 'a') as fo: + fo.write('Group quality control report for {proj} prepared on {datetime}\n'.format(proj=config_options['project'], datetime=date_time)) + + cmd=("mail -s '{proj} MRIQC Group Report' -a {mriqc_dir}/reports/bold_group.html -a {mriqc_dir}/reports/T1w_group.html {emails} < {message}".format(proj=config_options['project'], mriqc_dir=out_deriv_dir, emails=config_options['email'], message=op.join(args.work_dir, '{0}-mriqc-message.txt'.format(config_options['project'])))) + run(cmd) + + shutil.rmtree(scratch_deriv_dir) + os.remove(op.join(args.work_dir, '{0}-mriqc-message.txt'.format(config_options['project']))) + + +if __name__ == '__main__': + main() + diff --git a/protocol_check.py b/protocol_check.py new file mode 100644 index 0000000..7a58c4a --- /dev/null +++ b/protocol_check.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +""" +Based on +https://github.com/BIDS-Apps/example/blob/aa0d4808974d79c9fbe54d56d3b47bb2cf4e0a0d/run.py +""" +import os +import os.path as op +import json +import shutil +from glob import glob + +import argparse +import pandas as pd + + +def get_parser(): + parser = argparse.ArgumentParser(description='Check scans for project protocol compliance.') + parser.add_argument('-w', '--workdir', required=True, dest='work_dir', + help='Path to a working directory.') + parser.add_argument('--bids_dir', required=True, dest='bids_dir', + help='Full path to the BIDS directory.') + parser.add_argument('--sub', required=True, dest='sub', + help='The label of the subject to analyze.') + parser.add_argument('--ses', required=True, dest='ses', + help='Session number', default=None) + return parser + + +def main(argv=None): + args = get_parser().parse_args(argv) + + + # Check inputs + if not os.path.isdir(args.work_dir): + raise ValueError('Argument "workdir" must be an existing directory.') + + if not os.path.isdir(os.path.dirname(args.bids_dir)): + raise ValueError('Argument "bids_dir" must be an existing directory.') + + with open(op.join(os.path.dirname(args.bids_dir), 'code/config.json'), 'r') as fo: + config_options = json.load(fo) + + if not op.isfile(op.join(os.path.dirname(args.bids_dir), config_options['protocol'])): + raise ValueError('Argument "protocol" must exist.') + + + # Additional checks + if not os.path.isdir(op.join(args.work_dir, args.sub)): + raise ValueError('Subject directory does not exist in working directory.') + + if not os.path.isdir(op.join(args.work_dir, args.sub, args.ses)): + raise ValueError('Session directory does not exist in subjects working directory.') + + with open(op.join(os.path.dirname(args.bids_dir), config_options['protocol']), 'r') as fo: + protocol_options = json.load(fo) + + warning=False + scans = protocol_options.keys() + scan_list = os.listdir(op.join(args.work_dir, args.sub, args.ses)) + for tmp_scan in scans: + if (tmp_scan != 'email') and (tmp_scan != 'project'): + num = protocol_options[tmp_scan]['num'] + dicoms = protocol_options[tmp_scan]['dicoms'] + + tmp_scan_list = [tmp for tmp in scan_list if ((tmp_scan in tmp) and (("PMU" not in tmp) and ("setter" not in tmp)))] + + if len(tmp_scan_list) != num: + warning = True + with open(op.join(args.work_dir, '{sub}-{ses}-protocol_error.txt'.format(sub=args.sub, ses=args.ses)), 'a') as fo: + fo.write('There are {0} scans for {1}, but should be {2} \n'.format(len(tmp_scan_list), tmp_scan, num)) + + for t in tmp_scan_list: + if len(os.listdir(op.join(args.work_dir, args.sub, args.ses, t, 'resources/DICOM/files'))) != dicoms: + warning = True + with open(op.join(args.work_dir, '{sub}-{ses}-protocol_error.txt'.format(sub=args.sub, ses=args.ses)), 'a') as fo: + fo.write('There are {0} DICOMs for {1}, but should be {2} \n'.format(len(os.listdir(op.join(args.work_dir, args.sub, args.ses, t, 'resources/DICOM/files'))), t, dicoms)) + + if warning: + cmd = ("mail -s '{proj} Protocol Check Warning {sub} {ses}' {email_list} < {message}".format(proj=protocol_options['project'], sub=args.sub, ses=args.ses, email_list=protocol_options['email'], message=op.join(args.work_dir, '{sub}-{ses}-protocol_error.txt'.format(sub=args.sub, ses=args.ses)))) + os.system(cmd) + os.remove(op.join(args.work_dir, '{sub}-{ses}-protocol_error.txt'.format(sub=args.sub, ses=args.ses))) + +if __name__ == '__main__': + main() From ed5ee82a04faa182f0950012cc677c4cca946ecf Mon Sep 17 00:00:00 2001 From: Michael Riedel Date: Tue, 7 May 2019 12:03:08 -0400 Subject: [PATCH 4/6] Add files via upload --- config/Sutherland_ACE.json | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 config/Sutherland_ACE.json diff --git a/config/Sutherland_ACE.json b/config/Sutherland_ACE.json new file mode 100644 index 0000000..9ee207d --- /dev/null +++ b/config/Sutherland_ACE.json @@ -0,0 +1,40 @@ +{ + "project": "Sutherland_ACE", + "hpc_queue": "PQ_nbc", + "xnatdownload": "xnat_download_v0.0.3-2019-04-11-7651e03ddbc7.img", + "protocol": "code/protocol.json", + "bidsifier": "cis_bidsify_v0.0.1-2018-08-24-04f91fa82d17.img", + "heuristics": "code/heuristics.py", + "mriqc": "poldracklab_mriqc_0.10.4-2018-03-23-8fb5f5e9184f.img", + "mriqc_options": { + "anat": { + "mod": { + "T1w": { + "mriqc_settings": { + } + } + } + }, + "func": { + "task": { + "mid": { + "mriqc_settings": { + "fd_thres": 0.6 + } + }, + "nback": { + "mriqc_settings": { + "fd_thres": 0.6 + } + }, + "rest": { + "mriqc_settings": { + "fd_thres": 0.3 + } + } + } + } + }, + "fmriprep": "", + "email": "miriedel@fiu.edu masuther@fiu.edu jflan008@fiu.edu nhidmi@fiu.edu" +} From d408eee4d1accfc8f3d669de8bffef48e58165ca Mon Sep 17 00:00:00 2001 From: Michael Riedel Date: Tue, 7 May 2019 12:04:00 -0400 Subject: [PATCH 5/6] Create Sutherland_ACE.json --- protocol/Sutherland_ACE.json | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 protocol/Sutherland_ACE.json diff --git a/protocol/Sutherland_ACE.json b/protocol/Sutherland_ACE.json new file mode 100644 index 0000000..83080c0 --- /dev/null +++ b/protocol/Sutherland_ACE.json @@ -0,0 +1,28 @@ +{ + "project": "Sutherland_ACE", + "T1w_MPR_vNav": { + "num": 2, + "dicoms": 176 + }, + "fMRI_DistortionMap_PA": { + "num": 3, + "dicoms": 60 + }, + "fMRI_DistortionMap_AP": { + "num": 3, + "dicoms": 60 + }, + "fMRI_task_MID": { + "num": 4, + "dicoms": 642 + }, + "fMRI_rest": { + "num": 1, + "dicoms": 750 + }, + "fMRI_task_n_back": { + "num": 3, + "dicoms": 345 + }, + "email": "miriedel@fiu.edu masuther@fiu.edu jflan008@fiu.edu nhidmi@fiu.edu" +} From 1c2f910b37c142bb0e7cd44e857a8c203d476981 Mon Sep 17 00:00:00 2001 From: Michael Riedel Date: Thu, 16 May 2019 12:34:58 -0400 Subject: [PATCH 6/6] introduced scans.tsv in raw To be able to include behavioral files in the raw subject/session directories, individual folders for each sub/ses were created, and .tar files placed in there. these were then catalogued in a scans.tsv file in the raw directory along with creation time/date. so, rather than generating the *-processed.txt from files in the raw directory, the *-processed.txt file is created from the column 'file' in scans.tsv --- cis.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/cis.py b/cis.py index 65bb3f4..317b51d 100644 --- a/cis.py +++ b/cis.py @@ -118,11 +118,15 @@ def main(argv=None): if not op.isdir(raw_dir): os.makedirs(raw_dir) - tar_list = os.listdir(raw_dir) - with open(op.join(proj_work_dir, '{0}-processed.txt'.format(config_options['project'])), 'a') as fo: - for tmp_tar_list in tar_list: - fo.write(tmp_tar_list + "\n") - + #tar_list = os.listdir(raw_dir) + #with open(op.join(proj_work_dir, '{0}-processed.txt'.format(config_options['project'])), 'a') as fo: + # for tmp_tar_list in tar_list: + # fo.write(tmp_tar_list + "\n") + + scans_df = pd.read_csv(op.join(raw_dir, 'scans.tsv'), sep='\t') + scans_df = scans_df['file'] + scans_df.to_csv(op.join(proj_work_dir, '{0}-processed.txt'.format(config_options['project'])), index=False) + # Copy singularity images to scratch scratch_xnatdownload = op.join(args.work_dir, op.basename(xnatdownload_file)) if not op.isfile(scratch_xnatdownload): @@ -163,10 +167,22 @@ def main(argv=None): run(cmd) #tar the subject and session directory and copy to raw dir - with tarfile.open(op.join(raw_dir, '{sub}-{ses}.tar'.format(sub=tmp_sub, ses=tmp_ses)), "w") as tar: + with tarfile.open(op.join(raw_dir, tmp_sub, tmp_ses, '{sub}-{ses}.tar'.format(sub=tmp_sub, ses=tmp_ses)), "w") as tar: tar.add(op.join(raw_work_dir, '{sub}'.format(sub=tmp_sub)), arcname=os.path.basename(op.join(raw_work_dir, '{sub}'.format(sub=tmp_sub)))) shutil.rmtree(op.join(raw_work_dir, '{sub}'.format(sub=tmp_sub))) + scans_df = pd.read_csv(op.join(raw_dir, 'scans.tsv'), sep='\t') + cols = scans_df.columns + tmp_df = pd.DataFrame() + tmp_df = tmp_df.append({'sub': tmp_sub}, ignore_index=True) + tmp_df['ses'] = tmp_ses + tmp_df['file'] = '{sub}-{ses}.tar'.format(sub=tmp_sub, ses=tmp_ses) + moddate = os.path.getmtime(op.join(raw_dir, tmp_sub, tmp_ses, '{sub}-{ses}.tar'.format(sub=tmp_sub, ses=tmp_ses))) + timedateobj = datetime.datetime.fromtimestamp(moddate) + tmp_df['creation'] = datetime.datetime.strftime(datetimeob, "%m/%d/%Y, %H:%M") + scans_df = scans_df.append(tmp_df) + scans_df.to_csv(op.join(raw_dir, 'scans.tsv'), sep='\t', index=False) + # run cis_proc.py #cmd = ('python cis_proc.py -t {tarfile} -b {bidsdir} -w {work} --config {config} --sub {sub} --ses {ses} --n_procs {nprocs}'. format(tarfile=op.join(raw_dir, '{sub}-{ses}.tar'.format(sub=tmp_sub, ses=tmp_ses)), bidsdir=args.bids_dir, work=proj_work_dir, config=args.config, sub=tmp_sub.strip('sub-'), ses=tmp_ses.strip('ses-'), nprocs=args.n_procs)) #cmd = ('srun -J cis_proc-{proj}-{sub}-{ses} -e {err_file_loc} -o {out_file_loc} -c {nprocs} -q {hpc_queue} -p investor python /home/data/cis/cis-processing/cis_proc.py -t {tarfile} -b {bidsdir} -w {work} --config {config} --sub {sub} --ses {ses} --n_procs {nprocs}'. format(hpc_queue=config_options['hpc_queue'], proj=config_options['project'], err_file_loc = op.join('/home/data/cis/cis-processing/err', config_options['project'], 'cis_proc-{sub}-{ses}'.format(sub=tmp_sub, ses=tmp_ses)), out_file_loc= op.join('/home/data/cis/cis-processing/out', config_options['project'], 'cis_proc-{sub}-{ses}'.format(sub=tmp_sub, ses=tmp_ses)), tarfile=op.join(raw_dir, '{sub}-{ses}.tar'.format(sub=tmp_sub, ses=tmp_ses)), bidsdir=args.bids_dir, work=proj_work_dir, config=args.config, sub=tmp_sub.strip('sub-'), ses=tmp_ses.strip('ses-'), nprocs=args.n_procs))