Merge branch 'refs/heads/main' into zcat_maker

# Conflicts:
#	py/desispec/io/meta.py

bin/desi_find_file

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+
+"""
+Utility for returning the DESI NERSC location of a specified file
+"""
+
+from desispec.io.meta import findfile, get_findfile_argparser
+
+if __name__ == "__main__":
+    parser = get_findfile_argparser()
+    args = parser.parse_args()
+    print(findfile(**args.__dict__))

bin/desi_make_dark_tables

@@ -47,6 +47,8 @@ for files,outfile in zip([dark_files,bias_files],
         files_new = [f for f in files if f.replace(f"{args.indir}/","") not in outtab['FILENAME']]
     else:
         files_new = files
+    if len(files_new)==0:
+        raise ValueError(f"No new files found in {args.indir} to add to {outfile}")
     for fi in files_new:
         with fits.open(fi, "readonly") as f:
             det = f[0].header["DETECTOR"]

doc/changes.rst

@@ -2,21 +2,43 @@
 desispec Change Log
 ===================
 
-0.68.0 (unreleased)
+0.68.2 (unreleased)
 -------------------
 
+* No changes yet.
+
+0.68.1 (2024-11-08)
+-------------------
+
+Used for rerunning the QuasarNet afterburner on Loa. Compatible with
+0.68.0 tag for files that succeeded with that tag.
+
+* Fix QuasarNet afterburner indexing bug when all input spectra are masked
+  (PR `#2409`_).
+
+.. _`#2409`: https://github.com/desihub/desispec/pull/2409
+
+0.68.0 (2024-11-07)
+-------------------
+
+Used for rerunning the QuasarNet afterburner on Loa.
+
 * Allow `desi_proc_night` to restart midway in calibrations (PR `#2392`_).
-* Remove E(B-V) factor in backup tile EFFTIME calculations (PR `#2395`_).
+* Remove E(B-V) factor in backup tile EFFTIME calculations (PR `#2395`_, `#2406`_).
 * Fix undefined variables reported by static checker (PR `#2400`_).
 * QuasarNet afterburner runs LOZ and HIZ templates together;
   Add input provenance to output files including $QN_MODEL_FILE (PR `#2402`_).
 * Use slurm job status to color and inform processing dashboards (PR `#2403`_).
+* Update QuasarNet afterburner outputs - add columns, don't run SKY fibers
+  (PR `#2407`_).
 
 .. _`#2392`: https://github.com/desihub/desispec/pull/2392
 .. _`#2395`: https://github.com/desihub/desispec/pull/2395
 .. _`#2400`: https://github.com/desihub/desispec/pull/2400
 .. _`#2402`: https://github.com/desihub/desispec/pull/2402
 .. _`#2403`: https://github.com/desihub/desispec/pull/2403
+.. _`#2406`: https://github.com/desihub/desispec/pull/2406
+.. _`#2407`: https://github.com/desihub/desispec/pull/2407
 
 0.67.0 (2024-10-11)
 -------------------
@@ -117,6 +139,7 @@ First tag used for Kibo/Y3 run.
   desi_spectro_calib yaml files (PR `#2296`_).
 * desi_group_spectra header propagation cleanup (PR `#2302`_).
 * zproc requires exposure-qa files for tileqa step (PR `#2306`_).
+* Re-add support for custom tile groups like 1x_depth (PR `#2309`_)
 * Don't set envs in desispec.module that are now set in desimodules
   (PR `#2310`_).
 * New FIBERSTATUS NEARCHARGETRAP and VARIABLETHRU set in
@@ -137,6 +160,7 @@ First tag used for Kibo/Y3 run.
 .. _`#2296`: https://github.com/desihub/desispec/pull/2296
 .. _`#2302`: https://github.com/desihub/desispec/pull/2302
 .. _`#2306`: https://github.com/desihub/desispec/pull/2306
+.. _`#2309`: https://github.com/desihub/desispec/pull/2309
 .. _`#2310`: https://github.com/desihub/desispec/pull/2310
 .. _`#2313`: https://github.com/desihub/desispec/pull/2313
 .. _`#2314`: https://github.com/desihub/desispec/pull/2314

py/desispec/_version.py

@@ -1 +1 @@
-__version__ = '0.67.0.dev8946'
+__version__ = '0.68.1.dev8961'

py/desispec/efftime.py

@@ -64,7 +64,8 @@ def compute_efftime(table,
 
     # flux in fiber artificially divided by fiber_area_arcsec2  because the sky flux is per arcsec2
     fflux_bright = flux_bright_nom * transparency * fiber_fracflux_bgs / airfac / ebvfac / fiber_area_arcsec2
-    fflux_backup = flux_backup_nom * transparency * fiber_fracflux_psf / airfac / ebvfac / fiber_area_arcsec2
+    fflux_backup = flux_backup_nom * transparency * fiber_fracflux_psf / airfac / fiber_area_arcsec2
+    # Note we are omitting the ebvfac for backup
 
     # AR effective sky
     effsky_dark = (sky + sky_rdn * exptime_nom / exptime) / (1.0 + sky_rdn / sky_nom)
@@ -93,7 +94,7 @@ def compute_efftime(table,
         * (sky_nom / effsky_backup)
     )
     # Note that the backup calculation is on purpose
-    # ommitting the E(B-V) term
+    # omitting the E(B-V) term
 
     # set to -1 values with incorrect inputs
     bad=table["AIRMASS"]<0.99

py/desispec/io/meta.py

@@ -68,22 +68,81 @@ def get_readonly_filepath(filepath):
 
     return filepath
 
+def get_findfile_argparser():
+    """
+    Defines an argparser for a command line tool to call findfile
+    """
+    import argparse
+    parser = argparse.ArgumentParser(description="Returns the location where a file should be.")
+
+    ## Required argument
+    parser.add_argument("filetype", type=str, help="File type, typically the prefix, e.g., 'frame' or 'psf'.")
+
+    ## Optional arguments depending on filetype
+    parser.add_argument("-n", "--night", type=str,
+                        help="YEARMMDD string or int.")
+    parser.add_argument("-e", "--expid", type=int,
+                        help="Integer exposure ID.")
+    parser.add_argument("-c", "--camera", type=str,
+                        help="'b0', 'r1', ..., 'z9'.")
+    parser.add_argument("-t", "--tile", type=int,
+                        help="Integer tile (pointing) number.")
+    parser.add_argument("-g", "--groupname", type=str,
+                        help="Spectral grouping name (e.g., 'healpix', 'cumulative', 'pernight').")
+    parser.add_argument("--subgroup", type=str,
+                        help="Subgrouping name for non-standard group names.")
+    parser.add_argument("--healpix", type=int,
+                        help="Healpix pixel number.")
+    parser.add_argument("--nside", type=int, default=64,
+                        help="Healpix nside (default: 64).")
+    parser.add_argument("--band", type=str, choices=['b', 'r', 'z'],
+                        help="Camera band ('b', 'r', or 'z').")
+    parser.add_argument("--spectrograph", type=int,
+                        help="Integer spectrograph number (0-9).")
+    parser.add_argument("--survey", type=str,
+                        help="Survey, e.g., sv1, sv3, main, special.")
+    parser.add_argument("--faprogram", type=str,
+                        help="Fiberassign program, e.g., dark, bright.")
+
+    ## General options
+    parser.add_argument("--rawdata_dir", type=str,
+                        help="Overrides $DESI_SPECTRO_DATA.")
+    parser.add_argument("--specprod_dir", type=str,
+                        help="Overrides $DESI_SPECTRO_REDUX/$SPECPROD/.")
+    parser.add_argument("--specprod", type=str,
+                        help="Production name or full path to production.")
+    parser.add_argument("--qaprod_dir", type=str,
+                        help="Defaults to $DESI_SPECTRO_REDUX/$SPECPROD/QA/ if not provided.")
+    parser.add_argument("--tiles_dir", type=str,
+                        help="Defaults to $FIBER_ASSIGN_DIR if not provided.")
+    parser.add_argument("--outdir", type=str,
+                        help="Use this directory for output instead of canonical location.")
+    parser.add_argument("--download", action="store_true",
+                        help="If not found locally, try to fetch remotely.")
+    parser.add_argument("--return_exists", action="store_true",
+                        help="Also return whether the file exists.")
+    parser.add_argument("--readonly", action="store_true",
+                        help="Return read-only version of path if possible.")
+    parser.add_argument("--logfile", action="store_true",
+                        help="Return the pathname of the log instead of the data product itself.")
+
+    return parser
+
 def findfile(filetype, night=None, expid=None, camera=None,
         tile=None, groupname=None, subgroup=None,
-        healpix=None, nside=64,
-        band=None, spectrograph=None,
+        healpix=None, nside=64, band=None, spectrograph=None,
         survey=None, faprogram=None, version=None,
-        rawdata_dir=None, specprod_dir=None, specprod=None, qaprod_dir=None,
-        tiles_dir=None, download=False, outdir=None,
-        return_exists=False,
+        rawdata_dir=None, specprod_dir=None, specprod=None,
+        qaprod_dir=None, tiles_dir=None, outdir=None,
+        download=False, return_exists=False,
         readonly=False, logfile=False):
     """Returns location where file should be
 
     Args:
         filetype : file type, typically the prefix, e.g. "frame" or "psf"
 
     Args depending upon filetype:
-        night : YEARMMDD string
+        night : YEARMMDD string or int
         expid : integer exposure id
         camera : 'b0' 'r1' .. 'z9'
         tile : integer tile (pointing) number