Merge pull request #1 from lsst/tickets/DM-40740

DM-40740: Create an obs_package to handle the fiber spectrographs

.github/workflows/lint.yaml

@@ -1,22 +1,11 @@
 name: lint
 
 on:
-  - push
-  - pull_request
+  push:
+    branches:
+      - main
+  pull_request:
 
 jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.7
-
-      - name: Install
-        run: pip install -r <(curl https://raw.githubusercontent.com/lsst/linting/main/requirements.txt)
-
-      - name: Run linter
-        run: flake8
+  call-workflow:
+    uses: lsst/rubin_workflows/.github/workflows/lint.yaml@main

.gitignore

@@ -1,6 +1,7 @@
 .sconsign.dblite
 config.log
 .sconf_temp
+.scons*
 *.o
 *.os
 *.so

README.md

@@ -0,0 +1,5 @@
+# obs_fiberspectrograph
+
+``obs_fiberspectrograph`` is a package in the `LSST Science Pipelines <https://pipelines.lsst.io>`_.
+
+Package to ingest fiber spectrograph data.

pipelines/fiberspectrograph/ISR.yaml

@@ -0,0 +1,24 @@
+description: ISR for Rubin fiber spectrographs
+instrument: lsst.obs.fiberspectrograph.FiberSpectrograph
+
+tasks:
+  isr:
+    class: lsst.obs.fiberspectrograph.isrTask.IsrTask
+    config:
+      doBias: false
+      doCrosstalk: false
+      doVariance: false
+      doLinearize: false
+      doDefect: false
+      doDark: false
+      doFlat: false
+      doFringe: false
+      doAssembleCcd: false
+      doNanMasking: false
+      doSaturation: true
+      doWidenSaturationTrails: false
+      doCameraSpecificMasking: false
+      doSetBadRegions: false
+      doInterpolate: false
+      doMeasureBackground: false
+      doStandardStatistics: false

policy/fiberSpectrograph.yaml

@@ -0,0 +1,84 @@
+# This file is part of obs_fiberspectrograph.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This product includes software developed by the
+# LSST Project (http://www.lsst.org/).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the LSST License Statement and
+# the GNU General Public License along with this program.  If not,
+# see <http://www.lsstcorp.org/LegalNotices/>.
+#
+#
+
+name : "FiberSpec"
+plateScale : 1.0
+
+# Provide transformations *from* the nativeSys *to* the specified system (e.g. FieldAngle)
+transforms :
+  nativeSys : FocalPlane
+  FieldAngle :
+    transformType : radial
+    coeffs :        [0.0, 1.0, 0.0]     # radial distortion coefficients (c_0 + c_1 r + c_2 r^2 + ...)
+
+#
+# A list of detectors in the camera; we only have one
+#
+CCDs : &CCDs
+    "ccd0" :
+        detectorType : 0
+        id : 0
+        serial : "1606191U1"
+        offset : [0, 0]
+        refpos : [0, 0]
+        #
+        #            [[x0,  y0], [xSize, ySize]]
+        bbox : &bbox [[ 0,   0], [ 2048,     1]] # total bbox of trimmed detector
+        pixelSize : [1, 1]       # in mm
+        transformDict : {nativeSys : 'Pixels', transforms : None}
+        transposeDetector : False
+        pitch : 0.0                     # (degrees)
+        yaw : 0.0                       # rotation in plane of camera (degrees)
+        roll : 0.0                      # (degrees)
+
+        amplifiers:                     # only 1 amplifier
+            "0":
+              hdu : 1                   # Only one HDU in the file
+
+              ixy : [0, 0]
+              readCorner : LL
+              flipXY :     [False, False]
+              perAmpData : False                  # is the amp data split across multiple HDUs/Files?
+
+              #                         [[x0,  y0], [xSize, ySize]]
+              rawBBox                 : *bbox
+              rawDataBBox             : *bbox
+              rawSerialPrescanBBox    : [[0,    0], [0,         0]] # serial prescan
+              rawSerialOverscanBBox   : [[0,    0], [0,         0]] # serial overscan
+              rawParallelPrescanBBox  : [[0,    0], [0,         0]] # pixels digitised before first parallel
+              rawParallelOverscanBBox : [[0,    0], [0,         0]] # parallel overscan
+
+              saturation : 16383        # saturation level, DN XXX Should this be in electrons?
+
+              # Linearity correction is still under discussion, so this is a placeholder.
+              linearityType : PROPORTIONAL
+              linearityThreshold : 0
+              linearityMax : 65535
+              linearityCoeffs : [0, 65535]        # == [linearityThreshold, linearityMax]
+
+              gain : 1.00
+              readNoise : 10

python/lsst/obs/__init__.py

@@ -0,0 +1,2 @@
+import pkgutil
+__path__ = pkgutil.extend_path(__path__, __name__)

python/lsst/obs/fiberspectrograph/__init__.py

@@ -20,3 +20,5 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 from .version import *  # Generated by sconsUtils
+from ._instrument import *
+from .spectrum import *

python/lsst/obs/fiberspectrograph/_instrument.py

@@ -0,0 +1,78 @@
+# This file is part of obs_fiberspectrograph
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__all__ = ("FiberSpectrograph", )
+
+import os.path
+
+import lsst.obs.base.yamlCamera as yamlCamera
+from lsst.utils import getPackageDir
+from lsst.obs.base import VisitSystem
+from lsst.obs.lsst import LsstCam
+from .filters import FIBER_SPECTROGRAPH_FILTER_DEFINITIONS
+from .translator import FiberSpectrographTranslator
+
+PACKAGE_DIR = getPackageDir("obs_fiberspectrograph")
+
+
+class FiberSpectrograph(LsstCam):
+    """Gen3 instrument for the Rubin fiber spectrographs
+
+    Parameters
+    ----------
+    camera : `lsst.cameraGeom.Camera`
+        Camera object from which to extract detector information.
+    filters : `list` of `FilterDefinition`
+        An ordered list of filters to define the set of PhysicalFilters
+        associated with this instrument in the registry.
+    """
+    filterDefinitions = FIBER_SPECTROGRAPH_FILTER_DEFINITIONS
+    instrument = "FiberSpec"
+    policyName = "fiberSpectrograph"
+    translatorClass = FiberSpectrographTranslator
+    visitSystem = VisitSystem.BY_SEQ_START_END
+    raw_definition = ("rawSpectrum",
+                      ("instrument", "exposure", "detector"),
+                      "FiberSpectrum")
+
+    @classmethod
+    def getCamera(cls):
+        # Constructing a YAML camera takes a long time but we rely on
+        # yamlCamera to cache for us.
+        # N.b. can't inherit as PACKAGE_DIR isn't in the class
+        cameraYamlFile = os.path.join(PACKAGE_DIR, "policy", f"{cls.policyName}.yaml")
+        return yamlCamera.makeCamera(cameraYamlFile)
+
+    def getRawFormatter(self, dataId):
+        # Docstring inherited from Instrument.getRawFormatter
+        # local import to prevent circular dependency
+        from .rawFormatter import FiberSpectrographRawFormatter
+        return FiberSpectrographRawFormatter
+
+    def extractDetectorRecord(self, camGeomDetector):
+        """Create a Gen3 Detector entry dict from a cameraGeom.Detector.
+        """
+        return dict(
+            instrument=self.getName(),
+            id=camGeomDetector.getId(),
+            full_name=camGeomDetector.getName(),
+            purpose=str(camGeomDetector.getType()).split(".")[-1]
+        )

python/lsst/obs/fiberspectrograph/data_manager.py

@@ -0,0 +1,129 @@
+# This file is part of obs_fiberspectrograph
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import astropy.io.fits
+
+
+class DataManager:
+    """A data packager for `Spectrum` objects
+    that comes from the ts_fiberspectrograph package
+    """
+
+    wcs_table_name = "WCS-TAB"
+    """Name of the table containing the wavelength WCS (EXTNAME)."""
+    wcs_table_ver = 1
+    """WCS table version (EXTVER)."""
+    wcs_column_name = "wavelength"
+    """Name of the table column containing the wavelength information."""
+
+    # The version of the FITS file format produced by this class.
+    FORMAT_VERSION = 1
+
+    def __init__(self, spectrum):
+        self.spectrum = spectrum
+
+    def make_hdulist(self):
+        """Generate a FITS hdulist built from SpectrographData.
+        Parameters
+        ----------
+        spec : `SpectrographData`
+            The data from which to build the FITS hdulist.
+        Returns
+        -------
+        hdulist : `astropy.io.fits.HDUList`
+            The FITS hdulist.
+        """
+        hdu1 = self.make_primary_hdu()
+        hdu2 = self.make_wavelength_hdu()
+        hdu3, hdu4 = self.make_maskvariance_hdu()
+        return astropy.io.fits.HDUList([hdu1, hdu2, hdu3, hdu4])
+
+    def make_fits_header(self):
+        """Return a FITS header built from a Spectrum"""
+        hdr = astropy.io.fits.Header()
+
+        hdr["FORMAT_V"] = self.FORMAT_VERSION
+        hdr.update(self.spectrum.metadata)
+
+        # WCS headers - Use -TAB WCS definition
+        wcs_cards = [
+            "WCSAXES =                    1 / Number of WCS axes",
+            "CRPIX1  =                  0.0 / Reference pixel on axis 1",
+            "CRVAL1  =                  0.0 / Value at ref. pixel on axis 1",
+            "CNAME1  = 'Wavelength'         / Axis name for labeling purposes",
+            "CTYPE1  = 'WAVE-TAB'           / Wavelength axis by lookup table",
+            "CDELT1  =                  1.0 / Pixel size on axis 1",
+            f"CUNIT1  = '{self.spectrum.wavelength.unit.name:8s}'           / Units for axis 1",
+            f"PV1_1   = {self.wcs_table_ver:20d} / EXTVER  of bintable extension for -TAB arrays",
+            f"PS1_0   = '{self.wcs_table_name:8s}'           / EXTNAME of bintable extension for -TAB arrays",
+            f"PS1_1   = '{self.wcs_column_name:8s}'         / Wavelength coordinate array",
+        ]
+        for c in wcs_cards:
+            hdr.append(astropy.io.fits.Card.fromstring(c))
+
+        return hdr
+
+    def make_primary_hdu(self):
+        """Return the primary HDU built from a Spectrum."""
+
+        hdu = astropy.io.fits.PrimaryHDU(
+            data=self.spectrum.flux, header=self.make_fits_header()
+        )
+        return hdu
+
+    def make_maskvariance_hdu(self):
+        """Return the HDU for the mask and variance plane."""
+        hdr_mask = astropy.io.fits.Header()
+        hdr_mask["EXTTYPE"] = 'MASK    '
+        hdr_mask["EXTNAME"] = 'MASK    '
+        hdu_mask = astropy.io.fits.ImageHDU(
+            data=self.spectrum.mask, header=hdr_mask
+        )
+
+        hdr_variance = astropy.io.fits.Header()
+        hdr_variance["EXTTYPE"] = 'VARIANCE'
+        hdr_variance["EXTNAME"] = 'VARIANCE'
+        hdu_variance = astropy.io.fits.ImageHDU(
+            data=self.spectrum.variance, header=hdr_variance
+        )
+        return hdu_mask, hdu_variance
+
+    def make_wavelength_hdu(self):
+        """Return the wavelength HDU built from a Spectrum."""
+
+        # The wavelength array must be 2D (N, 1) in numpy but (1, N) in FITS
+        wavelength = self.spectrum.wavelength.reshape([self.spectrum.wavelength.size, 1])
+
+        # Create a Table. It will be a single element table
+        table = astropy.table.Table()
+
+        # Create the wavelength column
+        # Create the column explicitly since it is easier to ensure the
+        # shape this way.
+        wavecol = astropy.table.Column([wavelength], unit=wavelength.unit.name)
+
+        # The column name must match the PS1_1 entry from the primary HDU
+        table[self.wcs_column_name] = wavecol
+
+        # The name MUST match the value of PS1_0 and the version MUST
+        # match the value of PV1_1
+        hdu = astropy.io.fits.BinTableHDU(table, name=self.wcs_table_name, ver=1)
+        return hdu

python/lsst/obs/fiberspectrograph/filters.py

@@ -0,0 +1,5 @@
+from lsst.obs.base import FilterDefinition, FilterDefinitionCollection
+
+FIBER_SPECTROGRAPH_FILTER_DEFINITIONS = FilterDefinitionCollection(
+    FilterDefinition(band="white", physical_filter="empty"),
+)