diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 2b20981..796ef92 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,22 +1,11 @@ name: lint on: - - push - - pull_request + push: + branches: + - main + pull_request: jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - - name: Install - run: pip install -r <(curl https://raw.githubusercontent.com/lsst/linting/main/requirements.txt) - - - name: Run linter - run: flake8 + call-workflow: + uses: lsst/rubin_workflows/.github/workflows/lint.yaml@main diff --git a/.gitignore b/.gitignore index bb98353..37b3b3a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .sconsign.dblite config.log .sconf_temp +.scons* *.o *.os *.so diff --git a/README.md b/README.md new file mode 100644 index 0000000..60b2797 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# obs_fiberspectrograph + +``obs_fiberspectrograph`` is a package in the `LSST Science Pipelines `_. + +Package to ingest fiber spectrograph data. diff --git a/README.rst b/README.rst deleted file mode 100644 index 1dbfb6c..0000000 --- a/README.rst +++ /dev/null @@ -1,7 +0,0 @@ -##################### -obs_fiberspectrograph -##################### - -``obs_fiberspectrograph`` is a package in the `LSST Science Pipelines `_. - -.. Add a brief (few sentence) description of what this package provides. diff --git a/pipelines/fiberspectrograph/ISR.yaml b/pipelines/fiberspectrograph/ISR.yaml new file mode 100644 index 0000000..29b3926 --- /dev/null +++ b/pipelines/fiberspectrograph/ISR.yaml @@ -0,0 +1,24 @@ +description: ISR for Rubin fiber spectrographs +instrument: lsst.obs.fiberspectrograph.FiberSpectrograph + +tasks: + isr: + class: lsst.obs.fiberspectrograph.isrTask.IsrTask + config: + doBias: false + doCrosstalk: false + doVariance: false + doLinearize: false + doDefect: false + doDark: false + doFlat: false + doFringe: false + doAssembleCcd: false + doNanMasking: false + doSaturation: true + doWidenSaturationTrails: false + doCameraSpecificMasking: false + doSetBadRegions: false + doInterpolate: false + doMeasureBackground: false + doStandardStatistics: false diff --git a/policy/fiberSpectrograph.yaml b/policy/fiberSpectrograph.yaml new file mode 100644 index 0000000..49cdaef --- /dev/null +++ b/policy/fiberSpectrograph.yaml @@ -0,0 +1,84 @@ +# This file is part of obs_fiberspectrograph. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This product includes software developed by the +# LSST Project (http://www.lsst.org/). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the LSST License Statement and +# the GNU General Public License along with this program. If not, +# see . +# +# + +name : "FiberSpec" +plateScale : 1.0 + +# Provide transformations *from* the nativeSys *to* the specified system (e.g. FieldAngle) +transforms : + nativeSys : FocalPlane + FieldAngle : + transformType : radial + coeffs : [0.0, 1.0, 0.0] # radial distortion coefficients (c_0 + c_1 r + c_2 r^2 + ...) + +# +# A list of detectors in the camera; we only have one +# +CCDs : &CCDs + "ccd0" : + detectorType : 0 + id : 0 + serial : "1606191U1" + offset : [0, 0] + refpos : [0, 0] + # + # [[x0, y0], [xSize, ySize]] + bbox : &bbox [[ 0, 0], [ 2048, 1]] # total bbox of trimmed detector + pixelSize : [1, 1] # in mm + transformDict : {nativeSys : 'Pixels', transforms : None} + transposeDetector : False + pitch : 0.0 # (degrees) + yaw : 0.0 # rotation in plane of camera (degrees) + roll : 0.0 # (degrees) + + amplifiers: # only 1 amplifier + "0": + hdu : 1 # Only one HDU in the file + + ixy : [0, 0] + readCorner : LL + flipXY : [False, False] + perAmpData : False # is the amp data split across multiple HDUs/Files? + + # [[x0, y0], [xSize, ySize]] + rawBBox : *bbox + rawDataBBox : *bbox + rawSerialPrescanBBox : [[0, 0], [0, 0]] # serial prescan + rawSerialOverscanBBox : [[0, 0], [0, 0]] # serial overscan + rawParallelPrescanBBox : [[0, 0], [0, 0]] # pixels digitised before first parallel + rawParallelOverscanBBox : [[0, 0], [0, 0]] # parallel overscan + + saturation : 16383 # saturation level, DN XXX Should this be in electrons? + + # Linearity correction is still under discussion, so this is a placeholder. + linearityType : PROPORTIONAL + linearityThreshold : 0 + linearityMax : 65535 + linearityCoeffs : [0, 65535] # == [linearityThreshold, linearityMax] + + gain : 1.00 + readNoise : 10 diff --git a/python/lsst/obs/__init__.py b/python/lsst/obs/__init__.py new file mode 100644 index 0000000..bb61062 --- /dev/null +++ b/python/lsst/obs/__init__.py @@ -0,0 +1,2 @@ +import pkgutil +__path__ = pkgutil.extend_path(__path__, __name__) diff --git a/python/lsst/obs/fiberspectrograph/__init__.py b/python/lsst/obs/fiberspectrograph/__init__.py index 05676d2..4c46211 100644 --- a/python/lsst/obs/fiberspectrograph/__init__.py +++ b/python/lsst/obs/fiberspectrograph/__init__.py @@ -20,3 +20,5 @@ # along with this program. If not, see . from .version import * # Generated by sconsUtils +from ._instrument import * +from .spectrum import * diff --git a/python/lsst/obs/fiberspectrograph/_instrument.py b/python/lsst/obs/fiberspectrograph/_instrument.py new file mode 100644 index 0000000..9c81646 --- /dev/null +++ b/python/lsst/obs/fiberspectrograph/_instrument.py @@ -0,0 +1,78 @@ +# This file is part of obs_fiberspectrograph +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +__all__ = ("FiberSpectrograph", ) + +import os.path + +import lsst.obs.base.yamlCamera as yamlCamera +from lsst.utils import getPackageDir +from lsst.obs.base import VisitSystem +from lsst.obs.lsst import LsstCam +from .filters import FIBER_SPECTROGRAPH_FILTER_DEFINITIONS +from .translator import FiberSpectrographTranslator + +PACKAGE_DIR = getPackageDir("obs_fiberspectrograph") + + +class FiberSpectrograph(LsstCam): + """Gen3 instrument for the Rubin fiber spectrographs + + Parameters + ---------- + camera : `lsst.cameraGeom.Camera` + Camera object from which to extract detector information. + filters : `list` of `FilterDefinition` + An ordered list of filters to define the set of PhysicalFilters + associated with this instrument in the registry. + """ + filterDefinitions = FIBER_SPECTROGRAPH_FILTER_DEFINITIONS + instrument = "FiberSpec" + policyName = "fiberSpectrograph" + translatorClass = FiberSpectrographTranslator + visitSystem = VisitSystem.BY_SEQ_START_END + raw_definition = ("rawSpectrum", + ("instrument", "exposure", "detector"), + "FiberSpectrum") + + @classmethod + def getCamera(cls): + # Constructing a YAML camera takes a long time but we rely on + # yamlCamera to cache for us. + # N.b. can't inherit as PACKAGE_DIR isn't in the class + cameraYamlFile = os.path.join(PACKAGE_DIR, "policy", f"{cls.policyName}.yaml") + return yamlCamera.makeCamera(cameraYamlFile) + + def getRawFormatter(self, dataId): + # Docstring inherited from Instrument.getRawFormatter + # local import to prevent circular dependency + from .rawFormatter import FiberSpectrographRawFormatter + return FiberSpectrographRawFormatter + + def extractDetectorRecord(self, camGeomDetector): + """Create a Gen3 Detector entry dict from a cameraGeom.Detector. + """ + return dict( + instrument=self.getName(), + id=camGeomDetector.getId(), + full_name=camGeomDetector.getName(), + purpose=str(camGeomDetector.getType()).split(".")[-1] + ) diff --git a/python/lsst/obs/fiberspectrograph/data_manager.py b/python/lsst/obs/fiberspectrograph/data_manager.py new file mode 100644 index 0000000..389fcf6 --- /dev/null +++ b/python/lsst/obs/fiberspectrograph/data_manager.py @@ -0,0 +1,129 @@ +# This file is part of obs_fiberspectrograph +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import astropy.io.fits + + +class DataManager: + """A data packager for `Spectrum` objects + that comes from the ts_fiberspectrograph package + """ + + wcs_table_name = "WCS-TAB" + """Name of the table containing the wavelength WCS (EXTNAME).""" + wcs_table_ver = 1 + """WCS table version (EXTVER).""" + wcs_column_name = "wavelength" + """Name of the table column containing the wavelength information.""" + + # The version of the FITS file format produced by this class. + FORMAT_VERSION = 1 + + def __init__(self, spectrum): + self.spectrum = spectrum + + def make_hdulist(self): + """Generate a FITS hdulist built from SpectrographData. + Parameters + ---------- + spec : `SpectrographData` + The data from which to build the FITS hdulist. + Returns + ------- + hdulist : `astropy.io.fits.HDUList` + The FITS hdulist. + """ + hdu1 = self.make_primary_hdu() + hdu2 = self.make_wavelength_hdu() + hdu3, hdu4 = self.make_maskvariance_hdu() + return astropy.io.fits.HDUList([hdu1, hdu2, hdu3, hdu4]) + + def make_fits_header(self): + """Return a FITS header built from a Spectrum""" + hdr = astropy.io.fits.Header() + + hdr["FORMAT_V"] = self.FORMAT_VERSION + hdr.update(self.spectrum.metadata) + + # WCS headers - Use -TAB WCS definition + wcs_cards = [ + "WCSAXES = 1 / Number of WCS axes", + "CRPIX1 = 0.0 / Reference pixel on axis 1", + "CRVAL1 = 0.0 / Value at ref. pixel on axis 1", + "CNAME1 = 'Wavelength' / Axis name for labeling purposes", + "CTYPE1 = 'WAVE-TAB' / Wavelength axis by lookup table", + "CDELT1 = 1.0 / Pixel size on axis 1", + f"CUNIT1 = '{self.spectrum.wavelength.unit.name:8s}' / Units for axis 1", + f"PV1_1 = {self.wcs_table_ver:20d} / EXTVER of bintable extension for -TAB arrays", + f"PS1_0 = '{self.wcs_table_name:8s}' / EXTNAME of bintable extension for -TAB arrays", + f"PS1_1 = '{self.wcs_column_name:8s}' / Wavelength coordinate array", + ] + for c in wcs_cards: + hdr.append(astropy.io.fits.Card.fromstring(c)) + + return hdr + + def make_primary_hdu(self): + """Return the primary HDU built from a Spectrum.""" + + hdu = astropy.io.fits.PrimaryHDU( + data=self.spectrum.flux, header=self.make_fits_header() + ) + return hdu + + def make_maskvariance_hdu(self): + """Return the HDU for the mask and variance plane.""" + hdr_mask = astropy.io.fits.Header() + hdr_mask["EXTTYPE"] = 'MASK ' + hdr_mask["EXTNAME"] = 'MASK ' + hdu_mask = astropy.io.fits.ImageHDU( + data=self.spectrum.mask, header=hdr_mask + ) + + hdr_variance = astropy.io.fits.Header() + hdr_variance["EXTTYPE"] = 'VARIANCE' + hdr_variance["EXTNAME"] = 'VARIANCE' + hdu_variance = astropy.io.fits.ImageHDU( + data=self.spectrum.variance, header=hdr_variance + ) + return hdu_mask, hdu_variance + + def make_wavelength_hdu(self): + """Return the wavelength HDU built from a Spectrum.""" + + # The wavelength array must be 2D (N, 1) in numpy but (1, N) in FITS + wavelength = self.spectrum.wavelength.reshape([self.spectrum.wavelength.size, 1]) + + # Create a Table. It will be a single element table + table = astropy.table.Table() + + # Create the wavelength column + # Create the column explicitly since it is easier to ensure the + # shape this way. + wavecol = astropy.table.Column([wavelength], unit=wavelength.unit.name) + + # The column name must match the PS1_1 entry from the primary HDU + table[self.wcs_column_name] = wavecol + + # The name MUST match the value of PS1_0 and the version MUST + # match the value of PV1_1 + hdu = astropy.io.fits.BinTableHDU(table, name=self.wcs_table_name, ver=1) + return hdu diff --git a/python/lsst/obs/fiberspectrograph/filters.py b/python/lsst/obs/fiberspectrograph/filters.py new file mode 100644 index 0000000..b009d57 --- /dev/null +++ b/python/lsst/obs/fiberspectrograph/filters.py @@ -0,0 +1,5 @@ +from lsst.obs.base import FilterDefinition, FilterDefinitionCollection + +FIBER_SPECTROGRAPH_FILTER_DEFINITIONS = FilterDefinitionCollection( + FilterDefinition(band="white", physical_filter="empty"), +) diff --git a/python/lsst/obs/fiberspectrograph/isrTask.py b/python/lsst/obs/fiberspectrograph/isrTask.py new file mode 100644 index 0000000..a4cccf4 --- /dev/null +++ b/python/lsst/obs/fiberspectrograph/isrTask.py @@ -0,0 +1,117 @@ +# This file is part of obs_fiberspectrograph +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +__all__ = ["IsrTask", "IsrTaskConfig"] + +import numpy as np + +import lsst.geom +import lsst.pex.config as pexConfig +import lsst.pipe.base.connectionTypes as cT + +import lsst.ip.isr + + +class IsrTaskConnections(lsst.ip.isr.isrTask.IsrTaskConnections): + ccdExposure = cT.Input( + name="rawSpectrum", + doc="Input spectrum to process.", + storageClass="FiberSpectrum", + dimensions=["instrument", "exposure", "detector"], + ) + bias = cT.PrerequisiteInput( + name="bias", + doc="Input bias calibration.", + storageClass="FiberSpectrum", + dimensions=["instrument", "detector"], + isCalibration=True, + ) + outputExposure = cT.Output( + name="spectrum", + doc="Corrected spectrum.", + storageClass="FiberSpectrum", + dimensions=["instrument", "exposure", "detector"], + ) + + def __init__(self, *, config=None): + super().__init__(config=config) + + +class IsrTaskConfig(lsst.ip.isr.IsrTaskConfig, pipelineConnections=IsrTaskConnections): + """Configuration parameters for IsrTask. + + Items are grouped in the order in which they are executed by the task. + """ + datasetType = pexConfig.Field( + dtype=str, + doc="Dataset type for input data; users will typically leave this alone, " + "but camera-specific ISR tasks will override it", + default="rawSpectrum", + ) + + +class IsrTask(lsst.ip.isr.IsrTask): + """Apply common instrument signature correction algorithms to a raw frame. + + The process for correcting imaging data is very similar from + camera to camera. This task provides a vanilla implementation of + doing these corrections, including the ability to turn certain + corrections off if they are not needed. The inputs to the primary + method, `run()`, are a raw exposure to be corrected and the + calibration data products. The raw input is a single chip sized + mosaic of all amps including overscans and other non-science + pixels. + + The __init__ method sets up the subtasks for ISR processing, using + the defaults from `lsst.ip.isr`. + + Parameters + ---------- + args : `list` + Positional arguments passed to the Task constructor. + None used at this time. + kwargs : `dict`, optional + Keyword arguments passed on to the Task constructor. + None used at this time. + """ + ConfigClass = IsrTaskConfig + _DefaultName = "isr" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def ensureExposure(self, inputExposure, *args, **kwargs): + return inputExposure + + def convertIntToFloat(self, ccdExposure): + return ccdExposure + + def maskAmplifier(self, ccdExposure, amp, defects): + flux = ccdExposure.flux + + saturated = flux > amp.getSaturation() + flux[saturated] = np.NaN + ccdExposure.mask[saturated] |= ccdExposure.getPlaneBitMask(self.config.saturatedMaskName) + + return False + + def roughZeroPoint(self, exposure): + pass diff --git a/python/lsst/obs/fiberspectrograph/rawFormatter.py b/python/lsst/obs/fiberspectrograph/rawFormatter.py new file mode 100644 index 0000000..260b828 --- /dev/null +++ b/python/lsst/obs/fiberspectrograph/rawFormatter.py @@ -0,0 +1,35 @@ +__all__ = [] + +from lsst.daf.butler import Formatter +from .filters import FIBER_SPECTROGRAPH_FILTER_DEFINITIONS +from ._instrument import FiberSpectrograph +from .translator import FiberSpectrographTranslator +from .spectrum import FiberSpectrum + + +class FiberSpectrographRawFormatter(Formatter): + cameraClass = FiberSpectrograph + translatorClass = FiberSpectrographTranslator + fiberSpectrumClass = FiberSpectrum + filterDefinitions = FIBER_SPECTROGRAPH_FILTER_DEFINITIONS + extension = ".fits" + + def getDetector(self, id): + return self.cameraClass().getCamera()[id] + + def read(self, component=None): + """Read fiberspectrograph data. + + Returns + ------- + FiberSpectrum: `~lsst.obs.fiberspectrograph.FiberSpectrum` + In-memory spectrum. + """ + path = self.fileDescriptor.location.path + + return self.fiberSpectrumClass.readFits(path) + + def write(self): + path = self.fileDescriptor.location.path + + return self.fiberSpectrumClass.writeFits(path) diff --git a/python/lsst/obs/fiberspectrograph/spectrum.py b/python/lsst/obs/fiberspectrograph/spectrum.py new file mode 100644 index 0000000..cb1c536 --- /dev/null +++ b/python/lsst/obs/fiberspectrograph/spectrum.py @@ -0,0 +1,132 @@ +# This file is part of obs_fiberspectrograph +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (http://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +__all__ = ("FiberSpectrum",) + +import numpy as np +import astropy.io.fits +import astropy.units as u +from ._instrument import FiberSpectrograph +from .data_manager import DataManager +import lsst.afw.image as afwImage +from astro_metadata_translator import ObservationInfo + + +class FiberSpectrum: + """Define a spectrum from a fiber spectrograph. + + Parameters + ---------- + wavelength : `numpy.ndarray` + Spectrum wavelength in units provided by the spectrum file. + flux: `numpy.ndarray` + Spectrum flux. + md: `dict` + Dictionary of the spectrum headers. + detectorId : `int` + Optional Detector ID for this data. + """ + + def __init__(self, wavelength, flux, md=None, detectorId=0, mask=None, variance=None): + self.wavelength = wavelength + self.flux = flux + self.metadata = md + + self.info = ObservationInfo(md) + self.detector = FiberSpectrograph().getCamera()[detectorId] + + mask_temp = afwImage.MaskX(1, 1) + self.getPlaneBitMask = mask_temp.getPlaneBitMask # ughh, awful Mask API + self.mask = mask + self.variance = variance + + def getDetector(self): + """Get fiber spectrograph detector." + """ + return self.detector + + def getInfo(self): + """Get observation information." + """ + return self.info + + def getMetadata(self): + """Get the spectrum metadata." + """ + return self.metadata + + def getFilter(self): + """Get filter label." + """ + return FiberSpectrograph.filterDefinitions[0].makeFilterLabel() + + def getBBox(self): + """Get bounding box." + """ + return self.detector.getBBox() + + @classmethod + def readFits(cls, path): + """Read a Spectrum from disk." + + Parameters + ---------- + path : `str` + The file to read + + Returns + ------- + spectrum : `~lsst.obs.fiberspectrograph.FiberSpectrum` + In-memory spectrum. + """ + + fitsfile = astropy.io.fits.open(path) + md = dict(fitsfile[0].header) + format_v = md["FORMAT_V"] + + if format_v == 1: + flux = fitsfile[0].data + wavelength = fitsfile[md["PS1_0"]].data[md["PS1_1"]].flatten() + + wavelength = u.Quantity(wavelength, u.Unit(md["CUNIT1"]), copy=False) + + mask_temp = afwImage.MaskX(1, 1) + mask = np.zeros(flux.shape, dtype=mask_temp.array.dtype) + variance = np.zeros_like(flux) + if len(fitsfile) == 4: + mask = fitsfile[2].data + variance = fitsfile[3].data + else: + raise ValueError(f"FORMAT_V has changed from 1 to {format_v}") + + return cls(wavelength, flux, md=md, mask=mask, variance=variance) + + def writeFits(self, path): + """Write a Spectrum to disk. + + Parameters + ---------- + path : `str` + The file to write + """ + hdl = DataManager(self).make_hdulist() + + hdl.writeto(path) diff --git a/python/lsst/obs/fiberspectrograph/translator.py b/python/lsst/obs/fiberspectrograph/translator.py new file mode 100644 index 0000000..6621227 --- /dev/null +++ b/python/lsst/obs/fiberspectrograph/translator.py @@ -0,0 +1,165 @@ +import logging +import os + +import astropy.units as u +from astropy.time import Time + +from astro_metadata_translator import cache_translation +from lsst.obs.lsst.translators.lsst import SIMONYI_TELESCOPE, LsstBaseTranslator + +from lsst.utils import getPackageDir + +__all__ = ["FiberSpectrographTranslator", ] + +log = logging.getLogger(__name__) + + +class FiberSpectrographTranslator(LsstBaseTranslator): + """Metadata translator for Rubin calibration fibre spectrographs headers""" + + name = "FiberSpectrograph" + """Name of this translation class""" + + supported_instrument = "FiberSpec" + """Supports the Rubin calibration fiber spectrographs.""" + + default_search_path = os.path.join(getPackageDir("obs_fiberspectrograph"), "corrections") + """Default search path to use to locate header correction files.""" + + default_resource_root = os.path.join(getPackageDir("obs_fiberspectrograph"), "corrections") + """Default resource path root to use to locate header correction files.""" + + DETECTOR_MAX = 1 + + _const_map = { + # TODO: DM-43041 DATE, detector name and controller should be put + # in file header and add to mapping + "detector_num": 0, + "detector_name": "ccd0", + "object": None, + "physical_filter": "empty", + "detector_group": "None", + "relative_humidity": None, + "pressure": None, + "temperature": None, + "focus_z": None, + "boresight_airmass": None, + "boresight_rotation_angle": None, + "tracking_radec": None, + "telescope": SIMONYI_TELESCOPE, + "observation_type": "spectrum", # IMGTYPE is '' + } + """Constant mappings""" + + _trivial_map = { + "observation_id": "OBSID", + "science_program": ("PROGRAM", dict(default="unknown")), + "detector_serial": "SERIAL", + } + """One-to-one mappings""" + + @classmethod + def can_translate(cls, header, filename=None): + """Indicate whether this translation class can translate the + supplied header. + + Parameters + ---------- + header : `dict`-like + Header to convert to standardized form. + filename : `str`, optional + Name of file being translated. + + Returns + ------- + can : `bool` + `True` if the header is recognized by this class. `False` + otherwise. + """ + + # TODO: DM-43041 need to be updated with new fiber spec + return "INSTRUME" in header and header["INSTRUME"] in ["FiberSpectrograph.Broad"] + + @cache_translation + def to_instrument(self): + return "FiberSpec" + + @cache_translation + def to_datetime_begin(self): + self._used_these_cards("DATE-BEG") + return Time(self._header["DATE-BEG"], scale="tai", format="isot") + + @cache_translation + def to_exposure_time(self): + # Docstring will be inherited. Property defined in properties.py + # Some data is missing a value for EXPTIME. + # Have to be careful we do not have circular logic when trying to + # guess + if self.is_key_ok("EXPTIME"): + return self.quantity_from_card("EXPTIME", u.s) + + # A missing or undefined EXPTIME is problematic. Set to -1 + # to indicate that none was found. + log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s", + self._log_prefix) + return -1.0 * u.s + + @cache_translation + def to_dark_time(self): # N.b. defining this suppresses a warning re setting from exptime + if "DARKTIME" in self._header: + darkTime = self._header["DARKTIME"] + self._used_these_cards("DARKTIME") + return (darkTime, dict(unit=u.s)) + return self.to_exposure_time() + + @staticmethod + def compute_exposure_id(dayobs, seqnum, controller=None): + """Helper method to calculate the exposure_id. + + Parameters + ---------- + dayobs : `str` + Day of observation in either YYYYMMDD or YYYY-MM-DD format. + If the string looks like ISO format it will be truncated before the + ``T`` before being handled. + seqnum : `int` or `str` + Sequence number. + controller : `str`, optional + Controller to use. If this is "O", no change is made to the + exposure ID. If it is "C" a 1000 is added to the year component + of the exposure ID. If it is "H" a 2000 is added to the year + component. This sequence continues with "P" and "Q" controllers. + `None` indicates that the controller is not relevant to the + exposure ID calculation (generally this is the case for test + stand data). + + Returns + ------- + exposure_id : `int` + Exposure ID in form YYYYMMDDnnnnn form. + """ + if not isinstance(dayobs, int): + if "T" in dayobs: + dayobs = dayobs[:dayobs.find("T")] + + dayobs = dayobs.replace("-", "") + + if len(dayobs) != 8: + raise ValueError(f"Malformed dayobs: {dayobs}") + + # Expect no more than 99,999 exposures in a day + maxdigits = 5 + if seqnum >= 10**maxdigits: + raise ValueError(f"Sequence number ({seqnum}) exceeds limit") + + # Form the number as a string zero padding the sequence number + idstr = f"{dayobs}{seqnum:0{maxdigits}d}" + + # Exposure ID has to be an integer + return int(idstr) + + @cache_translation + def to_visit_id(self): + """Calculate the visit associated with this exposure. + """ + return self.to_exposure_id() diff --git a/setup.cfg b/setup.cfg index 53fa4eb..df45691 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,5 +10,3 @@ exclude = tests/.tests [tool:pytest] -addopts = --flake8 -flake8-ignore = E133 E226 E228 N802 N803 N806 N812 N813 N815 N816 W503 diff --git a/tests/SConscript b/tests/SConscript index 5437cb4..23d3252 100644 --- a/tests/SConscript +++ b/tests/SConscript @@ -1,3 +1,8 @@ # -*- python -*- -from lsst.sconsUtils import scripts +import os + +from lsst.sconsUtils import env, scripts scripts.BasicSConscript.tests(pyList=[]) + +if "DAF_BUTLER_PLUGINS" in os.environ: + env["ENV"]["DAF_BUTLER_PLUGINS"] = os.environ["DAF_BUTLER_PLUGINS"] diff --git a/tests/data/Broad_fiberSpecBroad_2024-01-09T17:41:34.996.fits b/tests/data/Broad_fiberSpecBroad_2024-01-09T17:41:34.996.fits new file mode 100644 index 0000000..59229ce Binary files /dev/null and b/tests/data/Broad_fiberSpecBroad_2024-01-09T17:41:34.996.fits differ diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py new file mode 100644 index 0000000..4595f21 --- /dev/null +++ b/tests/test_ingestion.py @@ -0,0 +1,46 @@ +"""Unit tests for Gen3 fiberspectrograph raw data ingest. +""" + +import unittest +import os +import lsst.utils.tests + +from lsst.obs.base.ingest_tests import IngestTestBase +from lsst.obs.fiberspectrograph import FiberSpectrograph +from lsst.obs.fiberspectrograph.filters import FIBER_SPECTROGRAPH_FILTER_DEFINITIONS + +# TODO DM 42620 +# testDataPackage = "testdata_fiberSpectrograph" +# try: +# testDataDirectory = lsst.utils.getPackageDir(testDataPackage) +# except (LookupError, lsst.pex.exceptions.NotFoundError): +# testDataDirectory = None +testDataDirectory = os.path.join(os.path.dirname(__file__), "data") + + +class FiberSpectrographIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase): + instrumentClassName = "lsst.obs.fiberspectrograph.FiberSpectrograph" + visits = None # we don't have a definition of visits + ingestDatasetTypeName = "rawSpectrum" + + def setUp(self): + self.ingestdir = os.path.dirname(__file__) + self.instrument = FiberSpectrograph() + self.file = os.path.join(testDataDirectory, + "Broad_fiberSpecBroad_2024-01-09T17:41:34.996.fits") + + day_obs = 20240109 + seq_num = 4 + self.dataIds = [dict(instrument="FiberSpec", exposure=100000 * day_obs + seq_num, detector=0)] + self.filterLabel = FIBER_SPECTROGRAPH_FILTER_DEFINITIONS[0].makeFilterLabel() + + super().setUp() + + +def setup_module(module): + lsst.utils.tests.init() + + +if __name__ == "__main__": + lsst.utils.tests.init() + unittest.main() diff --git a/tests/test_instrument.py b/tests/test_instrument.py new file mode 100644 index 0000000..b7bbb6b --- /dev/null +++ b/tests/test_instrument.py @@ -0,0 +1,24 @@ +"""Tests of the FiberSpectrograph instrument class. +""" + +import unittest + +import lsst.utils.tests +import lsst.obs.fiberspectrograph +from lsst.obs.base.instrument_tests import InstrumentTests, InstrumentTestData + + +class TestStarTracker(InstrumentTests, lsst.utils.tests.TestCase): + def setUp(self): + physical_filters = set(["empty"]) + + self.data = InstrumentTestData(name="FiberSpec", + nDetectors=1, + firstDetectorName="ccd0", + physical_filters=physical_filters) + self.instrument = lsst.obs.fiberspectrograph.FiberSpectrograph() + + +if __name__ == '__main__': + lsst.utils.tests.init() + unittest.main() diff --git a/ups/obs_fiberspectrograph.table b/ups/obs_fiberspectrograph.table index 848450f..af78a75 100644 --- a/ups/obs_fiberspectrograph.table +++ b/ups/obs_fiberspectrograph.table @@ -1,9 +1,8 @@ -# List EUPS dependencies of this package here. -# - Any package whose API is used directly should be listed explicitly. -# - Common third-party packages can be assumed to be recursively included by -# the "base" package. -setupRequired(base) +setupRequired(obs_base) +setupRequired(obs_lsst) +setupRequired(astro_metadata_translator) +setupRequired(daf_butler) +setupRequired(ip_isr) +setupRequired(afw) -# The following is boilerplate for all packages. -# See https://dmtn-001.lsst.io for details on LSST_LIBRARY_PATH. envPrepend(PYTHONPATH, ${PRODUCT_DIR}/python)