diff --git a/bin/desi_night_qa b/bin/desi_night_qa index 35c4d0f68..194e39d3a 100755 --- a/bin/desi_night_qa +++ b/bin/desi_night_qa @@ -42,27 +42,34 @@ steps_all = list(get_nightqa_outfns("", 0).keys()) def parse(options=None): parser = argparse.ArgumentParser( description="Generate $DESI_ROOT/spectro/redux/nightqa/{NIGHT}/nightqa-{NIGHT}.html page, and related products") - parser.add_argument("-p", "--prod", type = str, default = None, required = False, - help = "Path to input reduction, e.g. /global/cfs/cdirs/desi/spectro/redux/blanc, or simply prod version, like blanc, but requires env. variable DESI_SPECTRO_REDUX. Default is $DESI_SPECTRO_REDUX/$SPECPROD.") parser.add_argument("-n","--night", type = int, default = None, required = True, help = "night to process. ex: 20211128") + parser.add_argument("-p", "--prod", type = str, default = None, required = False, + help = "Path to input reduction, e.g. /global/cfs/cdirs/desi/spectro/redux/blanc, or simply prod version, like blanc, but requires env. variable DESI_SPECTRO_REDUX. Default is $DESI_SPECTRO_REDUX/$SPECPROD.") parser.add_argument("-g","--group", type = str, default = "cumulative", required = False, help = 'tile group "cumulative" or "pernight"') parser.add_argument("-o", "--outdir", type = str, default = None, required = False, help = "Path to ouput folder, default is the input prod directory. Files written in {prod}/nightqa/{night}; several files will be created there") parser.add_argument("--css", type = str, default = None, required = False, help = "html formatting css file; default to importlib.resources.files('desispec').joinpath('data/qa/nightqa.css')") - parser.add_argument("--recompute", action = "store_true", - help = "recompute (i.e. overwrite args.outfile if already existing") - parser.add_argument("--compute_missing_only", action = "store_true", - help = "compute missing files only; overrides args.steps") parser.add_argument("--steps", type = str, default = ",".join(steps_all), required = False, help = "comma-separated list of steps to execute (default={})".format(",".join(steps_all))) parser.add_argument("--nproc", type = int, default = 1, required = False, help="number of parallel processes for create_dark_pdf(), create_ctedet_pdf(), create_ctedet_rowbyrow_pdf() and create_sframesky_pdf() (default=1)") parser.add_argument("--dark-bkgsub-science-cameras", type = str, default = "b", required = False, help="for the dark/morningdark, comma-separated list of the cameras to be additionally processed with the --bkgsub-for-science argument (default=b)") - + ## flags + parser.add_argument("--recompute", action = "store_true", + help="recompute (i.e. overwrite args.outfile if already existing") + parser.add_argument("--compute-missing-only", action = "store_true", + help="compute missing files only; overrides args.steps") + parser.add_argument("--skip-dark-steps", action = "store_true", + help="skips the dark, morningdark, and badcol steps even " + + "if supplied in --steps; overrides args.steps") + parser.add_argument("--skip-cal-steps", action = "store_true", + help="skips the dark, morningdark, badcol, ctedet, " + + "and ctedetrowbyrow steps even if supplied " + + "in --steps; overrides args.steps") args = None if options is None: args = parser.parse_args() @@ -71,6 +78,7 @@ def parse(options=None): # AR handle None... if args.dark_bkgsub_science_cameras.lower() == "none": args.dark_bkgsub_science_cameras = None + return args @@ -117,9 +125,20 @@ def main(): log.info("args.compute_missing_only set: will override args.steps={}".format(args.steps)) steps_tbd = [] for step in steps_all: - if not os.path.isfile(outfns[step]): + ## recompute html since recomputing other steps + if not os.path.isfile(outfns[step]) or step == 'html': steps_tbd.append(step) - log.info("add {} to steps_tbd, as args.compute_missing_only set and no {} present".format(step, outfns[step])) + log.info(f"add {step} to steps_tbd, as args.compute_missing_only " + + f"set and no {outfns[step]} present") + if args.skip_dark_steps or args.skip_cal_steps: + for step in ['dark', 'morningdark', 'badcol']: + if step in steps_tbd: + steps_tbd.remove(step) + if args.skip_cal_steps: + for step in ['ctedet', 'ctedetrowbyrow']: + if step in steps_tbd: + steps_tbd.remove(step) + if len(steps_tbd) == 0: log.info("no steps to be done") else: @@ -131,10 +150,11 @@ def main(): if os.path.isfile(fn): if args.recompute: log.warning("\texisting {} will be overwritten".format(fn)) + elif step == 'html': + log.warning("\texisting {} will be overwritten even though args.recompute = False".format(fn)) else: - msg = "\t{} already exists, and args.recompute = False".format(fn) - log.error(msg) - raise ValueError(msg) + log.warning(f"\t{fn} already exists and args.recompute = False," + + f" so skipping this step") # AR expids, tileids, surveys if np.in1d(["sframesky", "tileqa", "skyzfiber", "petalnz", "html"], steps_tbd).sum() > 0: @@ -152,45 +172,50 @@ def main(): ctedet_expid = get_ctedet_night_expid(args.night, args.prod) # AR dark - if "dark" in steps_tbd: - if dark_expid is not None: - create_dark_pdf(outfns["dark"], args.night, args.prod, dark_expid, args.nproc, bkgsub_science_cameras=args.dark_bkgsub_science_cameras) + if "dark" in steps_tbd and dark_expid is not None \ + and (args.recompute or not os.path.exists(outfns["dark"])): + create_dark_pdf(outfns["dark"], args.night, args.prod, dark_expid, args.nproc, + bkgsub_science_cameras_str=args.dark_bkgsub_science_cameras) # AR morning dark expid - if "morningdark" in steps_tbd: - if morningdark_expid is not None: - create_dark_pdf(outfns["morningdark"], args.night, args.prod, morningdark_expid, args.nproc, bkgsub_science_cameras=args.dark_bkgsub_science_cameras) + if "morningdark" in steps_tbd and morningdark_expid is not None \ + and (args.recompute or not os.path.exists(outfns["morningdark"])): + create_dark_pdf(outfns["morningdark"], args.night, args.prod, morningdark_expid, + args.nproc, bkgsub_science_cameras_str=args.dark_bkgsub_science_cameras) # AR badcolumn - if "badcol" in steps_tbd: - if dark_expid is not None: - create_badcol_png(outfns["badcol"], args.night, args.prod) + if "badcol" in steps_tbd and dark_expid is not None \ + and (args.recompute or not os.path.exists(outfns["badcol"])): + create_badcol_png(outfns["badcol"], args.night, args.prod) # AR CTE detector - if np.in1d(["ctedet", "ctedetrowbyrow"], steps_tbd).sum() > 0: - if ctedet_expid is not None: - create_ctedet_pdf(outfns["ctedet"], args.night, args.prod, ctedet_expid, args.nproc) - create_ctedet_rowbyrow_pdf(outfns["ctedetrowbyrow"], args.night, args.prod, ctedet_expid, args.nproc) + if "ctedet" in steps_tbd and ctedet_expid is not None \ + and (args.recompute or not os.path.exists(outfns["ctedet"])): + create_ctedet_pdf(outfns["ctedet"], args.night, args.prod, ctedet_expid, args.nproc) + + if "ctedetrowbyrow" in steps_tbd and ctedet_expid is not None \ + and (args.recompute or not os.path.exists(outfns["ctedetrowbyrow"])): + create_ctedet_rowbyrow_pdf(outfns["ctedetrowbyrow"], args.night, args.prod, ctedet_expid, args.nproc) # AR sframesky - if "sframesky" in steps_tbd: + if "sframesky" in steps_tbd and (args.recompute or not os.path.exists(outfns["sframesky"])): create_sframesky_pdf(outfns["sframesky"], args.night, args.prod, expids, args.nproc) # AR tileqa - if "tileqa" in steps_tbd: + if "tileqa" in steps_tbd and (args.recompute or not os.path.exists(outfns["tileqa"])): create_tileqa_pdf(outfns["tileqa"], args.night, args.prod, expids, tileids, group=args.group) # AR skyzfiber - if "skyzfiber" in steps_tbd: - create_skyzfiber_png(outfns["skyzfiber"], args.night, args.prod, np.unique(tileids), dchi2_threshold=9, - group=args.group) + if "skyzfiber" in steps_tbd and (args.recompute or not os.path.exists(outfns["skyzfiber"])): + create_skyzfiber_png(outfns["skyzfiber"], args.night, args.prod, + np.unique(tileids), dchi2_threshold=9, group=args.group) # AR per-petal n(z) - if "petalnz" in steps_tbd: + if "petalnz" in steps_tbd and (args.recompute or not os.path.exists(outfns["petalnz"])): unq_tileids, ii = np.unique(tileids, return_index=True) unq_surveys = surveys[ii] - create_petalnz_pdf(outfns["petalnz"], args.night, args.prod, unq_tileids, unq_surveys, dchi2_threshold=25, - group=args.group) + create_petalnz_pdf(outfns["petalnz"], args.night, args.prod, unq_tileids, + unq_surveys, dchi2_threshold=25, group=args.group) # AR create index.html # AR we first copy the args.css file to args.outdir @@ -200,6 +225,7 @@ def main(): outfns, args.night, args.prod, os.path.basename(args.css), expids, tileids, surveys, ) + duration_seconds = time.time() - start_time minutes = int(duration_seconds // 60) seconds = int(duration_seconds % 60) diff --git a/py/desispec/night_qa.py b/py/desispec/night_qa.py index af6337311..e1239e8e1 100644 --- a/py/desispec/night_qa.py +++ b/py/desispec/night_qa.py @@ -8,6 +8,7 @@ import os from glob import glob import tempfile +import shutil import textwrap from desiutil.log import get_logger import multiprocessing @@ -168,11 +169,7 @@ def get_dark_night_expid(night, prod): """ # expid = None - proctable_fn = os.path.join( - prod, - "processing_tables", - "processing_table_{}-{}.csv".format(os.path.basename(prod), night), - ) + proctable_fn = findfile('processing_table', night=night, specprod_dir=prod) log.info("proctable_fn = {}".format(proctable_fn)) if not os.path.isfile(proctable_fn): log.warning("no {} found; returning None".format(proctable_fn)) @@ -225,12 +222,7 @@ def get_morning_dark_night_expid(night, prod, exptime=1200): """ # expid = None - exptable_fn = os.path.join( - prod, - "exposure_tables", - str(night // 100), - "exposure_table_{}.csv".format(night), - ) + exptable_fn = findfile('exposure_table', night=night, specprod_dir=prod) log.info("exptable_fn = {}".format(exptable_fn)) if not os.path.isfile(exptable_fn): log.warning("no {} found; returning None".format(exptable_fn)) @@ -468,7 +460,7 @@ def _read_dark(fn, night, prod, dark_expid, petal, camera, binning=4): return None -def create_dark_pdf(outpdf, night, prod, dark_expid, nproc, binning=4, bkgsub_science_cameras=None, run_preproc=None): +def create_dark_pdf(outpdf, night, prod, dark_expid, nproc, binning=4, bkgsub_science_cameras_str=None, run_preproc=None): """ For a given night, create a pdf with the 300s binned dark. @@ -479,7 +471,7 @@ def create_dark_pdf(outpdf, night, prod, dark_expid, nproc, binning=4, bkgsub_sc dark_expid: EXPID of the 300s or 1200s DARK exposure to display (int) nproc: number of processes running at the same time (int) binning (optional, defaults to 4): binning of the image (which will be beforehand trimmed) (int) - bkgsub_science_cameras (optional, defaults to None): comma-separated list of the + bkgsub_science_cameras_str (optional, defaults to None): comma-separated list of the cameras to be additionally processed with the --bkgsub-for-science argument (e.g. "b") (string) run_preproc (optional, defaults to None): if None, autoderive if preproc should be run, otherwise @@ -501,9 +493,10 @@ def create_dark_pdf(outpdf, night, prod, dark_expid, nproc, binning=4, bkgsub_sc raise ValueError(msg) # AR sanity check - if bkgsub_science_cameras is not None: - if not np.all(np.in1d(bkgsub_science_cameras.split(","), cameras)): - raise ValueError("cameras_bkgsub_science={} not in b,r,z".format(bkgsub_science_cameras)) + if bkgsub_science_cameras_str is not None: + bkgsub_science_cameras = bkgsub_science_cameras_str.split(",") + if not np.all(np.in1d(bkgsub_science_cameras_str.split(","), cameras)): + raise ValueError("cameras_bkgsub_science={} not in b,r,z".format(bkgsub_science_cameras_str)) # AR get existing campets h = fits.open(rawfn) @@ -515,15 +508,10 @@ def create_dark_pdf(outpdf, night, prod, dark_expid, nproc, binning=4, bkgsub_sc campet = "{}{}".format(camera, petal) if campet.upper() in extnames: campets.append(campet) - campets = ",".join(campets) log.info("existing campets: {}".format(campets)) # AR first check if we need to process this dark image - proctable_fn = os.path.join( - prod, - "processing_tables", - "processing_table_{}-{}.csv".format(os.path.basename(prod), night), - ) + proctable_fn = findfile('processing_table', night=night, specprod_dir=prod) # if set to None will judge necessity for preprocessing according to proctable # but allows manual override e.g. for cases where no proctable should be there if run_preproc is None: @@ -548,17 +536,24 @@ def create_dark_pdf(outpdf, night, prod, dark_expid, nproc, binning=4, bkgsub_sc run_preproc = False # AR run preproc? + temp_dir_loc = None if run_preproc: - specprod_dir = tempfile.mkdtemp() - outdir = os.path.join(specprod_dir, "preproc", str(night), "{:08d}".format(dark_expid)) + cmds = [] + temp_dir_loc = tempfile.mkdtemp() + specprod_dir = temp_dir_loc + outdir = os.path.dirname(findfile("preproc", night, dark_expid, + 'r1', specprod_dir=specprod_dir)) os.makedirs(outdir, exist_ok=True) - cmd = "desi_preproc -n {} -e {} --outdir {} --ncpu {} --cameras {}".format( - night, dark_expid, outdir, nproc, campets, - ) - log.info("run: {}".format(cmd)) + for campet in campets: + cmd = "desi_preproc -n {} -e {} --outdir {} --ncpu 1 --cameras {}".format( + night, dark_expid, outdir, campet, + ) + log.info("run: {}".format(cmd)) - # like os.system(cmd), but avoids system call for MPI compatibility - preproc.main(cmd.split()[1:]) + # like os.system(cmd), but avoids system call for MPI compatibility + cmds.append(cmd.split()[1:]) + with multiprocessing.Pool(processes=nproc) as pool: + pool.map(preproc.main, cmds) else: specprod_dir = prod @@ -578,19 +573,23 @@ def create_dark_pdf(outpdf, night, prod, dark_expid, nproc, binning=4, bkgsub_sc ] ) # AR launching pool - pool = multiprocessing.Pool(processes=nproc) - with pool: + with multiprocessing.Pool(processes=nproc) as pool: mydicts = pool.starmap(_read_dark, myargs) + ## Remove the temporary directory + if temp_dir_loc is not None and os.path.exists(temp_dir_loc): + shutil.rmtree(temp_dir_loc) + log.info(f"Removed temporary directory and its contents: {temp_dir_loc}") + # AR campets to be additionally processed with --bkgsub-for-science # AR besides, check the very unlikely case where a camera is missing # AR for all petals - print(repr(bkgsub_science_cameras)) - if bkgsub_science_cameras is not None: + print(repr(bkgsub_science_cameras_str)) + if bkgsub_science_cameras_str is not None: missing_cameras = [] - for camera in bkgsub_science_cameras.split(","): - _ = [campet for campet in campets.split(",") if campet[0] == camera] - if len(_) == 0: + for camera in bkgsub_science_cameras: + testlist = [campet for campet in campets if campet[0] == camera] + if len(testlist) == 0: missing_cameras.append(camera) if len(missing_cameras) > 0: log.warning( @@ -598,30 +597,33 @@ def create_dark_pdf(outpdf, night, prod, dark_expid, nproc, binning=4, bkgsub_sc missing_cameras ) ) - bkgsub_science_cameras = ",".join([camera for camera in bkgsub_science_cameras.split(",") if camera not in missing_cameras]) - bkgsub_science_campets = ",".join( - [ - campet for campet in campets.split(",") - if campet[0] in bkgsub_science_cameras.split(",") - ] - ) + bkgsub_science_cameras = [camera for camera in bkgsub_science_cameras if camera not in missing_cameras] + bkgsub_science_campets = [campet for campet in campets if campet[0] in bkgsub_science_cameras] + bkgsub_science_cameras_str = ",".join(bkgsub_science_cameras) log.info("campets to be additionally processed with --bkgsub-for-science: {}".format(bkgsub_science_campets)) # + bkgsub_specprod_dir = None if len(bkgsub_science_campets) > 0: bkgsub_specprod_dir = tempfile.mkdtemp() - outdir = os.path.join(bkgsub_specprod_dir, "preproc", str(night), "{:08d}".format(dark_expid)) + outdir = os.path.dirname(findfile("preproc", night, dark_expid, + 'r1', specprod_dir=bkgsub_specprod_dir)) os.makedirs(outdir, exist_ok=True) - cmd = "desi_preproc -n {} -e {} --outdir {} --ncpu {} --cameras {} --bkgsub-for-science".format( - night, dark_expid, outdir, nproc, bkgsub_science_campets, - ) - log.info("run: {}".format(cmd)) - # like os.system(cmd), but avoids system call for MPI compatibility - preproc.main(cmd.split()[1:]) + cmds = [] + for campet in bkgsub_science_campets: + cmd = "desi_preproc -n {} -e {} --outdir {} --ncpu 1 --cameras {} --bkgsub-for-science".format( + night, dark_expid, outdir, campet, + ) + log.info("run: {}".format(cmd)) + + # like os.system(cmd), but avoids system call for MPI compatibility + cmds.append(cmd.split()[1:]) + with multiprocessing.Pool(processes=nproc) as pool: + pool.map(preproc.main, cmds) # AR read the bkgsub dark bkgsub_myargs = [] for petal in petals: - for camera in bkgsub_science_cameras.split(","): + for camera in bkgsub_science_cameras: bkgsub_myargs.append( [ findfile("preproc", night, dark_expid, camera+str(petal), specprod_dir=bkgsub_specprod_dir), @@ -634,10 +636,14 @@ def create_dark_pdf(outpdf, night, prod, dark_expid, nproc, binning=4, bkgsub_sc ] ) # AR launching pool - pool = multiprocessing.Pool(processes=nproc) - with pool: + with multiprocessing.Pool(processes=nproc) as pool: bkgsub_mydicts = pool.starmap(_read_dark, bkgsub_myargs) + ## Remove the temporary directory + if bkgsub_specprod_dir is not None and os.path.exists(bkgsub_specprod_dir): + shutil.rmtree(bkgsub_specprod_dir) + log.info(f"Removed temporary directory and its contents: {bkgsub_specprod_dir}") + # AR plotting # AR remarks: # AR - the (x,y) conversions for the side panels @@ -650,8 +656,8 @@ def create_dark_pdf(outpdf, night, prod, dark_expid, nproc, binning=4, bkgsub_sc cmap.set_bad(color="r") # AR ncol = 2 * len(cameras) - if bkgsub_science_cameras is not None: - ncol += 2 * len(bkgsub_science_cameras) + if bkgsub_science_cameras_str is not None: + ncol += 2 * len(bkgsub_science_cameras_str) width_ratios = 0.1 + np.zeros(ncol) width_ratios[::2] = 1 tmpcols = plt.rcParams["axes.prop_cycle"].by_key()["color"] @@ -668,8 +674,8 @@ def create_dark_pdf(outpdf, night, prod, dark_expid, nproc, binning=4, bkgsub_sc campet_mydicts = [mydicts[ii[0]]] campet_titles = [title] # AR bkgsub-for-science case - if bkgsub_science_cameras is not None: - if camera in bkgsub_science_cameras.split(","): + if bkgsub_science_cameras_str is not None: + if camera in bkgsub_science_cameras: bkgsub_ii = [i for i in range(len(bkgsub_myargs)) if bkgsub_myargs[i][4] == petal and bkgsub_myargs[i][5] == camera] assert(len(bkgsub_ii) == 1) campet_mydicts.append(bkgsub_mydicts[bkgsub_ii[0]]) @@ -898,8 +904,7 @@ def _read_ctedet(night, prod, ctedet_expid, nproc): ] ) # AR launching pool - pool = multiprocessing.Pool(processes=nproc) - with pool: + with multiprocessing.Pool(processes=nproc) as pool: mydicts = pool.starmap(_read_ctedet_campet, myargs) return mydicts @@ -1042,8 +1047,7 @@ def _compute_rowbyrow(ims, nproc): else: tmp_ims = [ims[i] for i in ii] - pool = multiprocessing.Pool(nproc) - with pool: + with multiprocessing.Pool(nproc) as pool: tmp_mods = pool.map(get_rowbyrow_image_model, tmp_ims) # AR mods with mods[i]=None if ims[i]=None @@ -1290,8 +1294,7 @@ def create_sframesky_pdf(outpdf, night, prod, expids, nproc): ] ) # AR launching pool - pool = multiprocessing.Pool(processes=nproc) - with pool: + with multiprocessing.Pool(processes=nproc) as pool: mydicts = pool.starmap(_read_sframesky, myargs) # AR creating pdf (+ removing temporary files) cmap = matplotlib.cm.Greys_r diff --git a/py/desispec/scripts/preproc.py b/py/desispec/scripts/preproc.py index 78b9a5318..ed71b39f7 100644 --- a/py/desispec/scripts/preproc.py +++ b/py/desispec/scripts/preproc.py @@ -219,11 +219,9 @@ def main(args=None): if args.ncpu > 1 and num_cameras>1: n = min(args.ncpu, num_cameras) log.info(f'Processing {num_cameras} cameras with {n} multiprocessing processes') - pool = mp.Pool(n) - failed = pool.map(_preproc_file_kwargs_wrapper, opts_array) + with mp.Pool(n) as pool: + failed = pool.map(_preproc_file_kwargs_wrapper, opts_array) num_failed = np.sum(failed) - pool.close() - pool.join() else: log.info(f'Not using multiprocessing for {num_cameras} cameras') num_failed = 0