Merge pull request #845 from lsst/tickets/DM-32769

DM-32769: Implement streak detection and masking in single frame processing

python/lsst/pipe/tasks/calibrate.py

@@ -311,6 +311,10 @@ def setDefaults(self):
         # The photoRefCat connection is the name to use for the colorterms.
         self.photoCal.photoCatName = self.connections.photoRefCat
 
+        # Keep track of which footprints contain streaks
+        self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK']
+        self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK']
+
 
 class CalibrateTask(pipeBase.PipelineTask):
     """Calibrate an exposure: measure sources and perform astrometric and

python/lsst/pipe/tasks/calibrateImage.py

@@ -33,7 +33,7 @@
 from lsst.pipe.base import connectionTypes
 from lsst.utils.timer import timeMethod
 
-from . import measurePsf, repair, setPrimaryFlags, photoCal, computeExposureSummaryStats
+from . import measurePsf, repair, setPrimaryFlags, photoCal, computeExposureSummaryStats, maskStreaks
 
 
 class CalibrateImageConnections(pipeBase.PipelineTaskConnections,
@@ -181,6 +181,10 @@ class CalibrateImageConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Cali
         target=lsst.meas.algorithms.SourceDetectionTask,
         doc="Task to detect stars to return in the output catalog."
     )
+    star_mask_streaks = pexConfig.ConfigurableField(
+        target=maskStreaks.MaskStreaksTask,
+        doc="Task for masking streaks. Adds a STREAK mask plane to an exposure.",
+    )
     star_deblend = pexConfig.ConfigurableField(
         target=lsst.meas.deblender.SourceDeblendTask,
         doc="Split blended sources into their components"
@@ -292,6 +296,10 @@ def setDefaults(self):
         # Only measure the apertures we need for star selection.
         self.star_measurement.plugins["base_CircularApertureFlux"].radii = [12.0]
 
+        # Keep track of which footprints contain streaks
+        self.star_measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK']
+        self.star_measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK']
+
         # Select isolated stars with reliable measurements and no bad flags.
         self.star_selector["science"].doFlags = True
         self.star_selector["science"].doUnresolved = True
@@ -344,6 +352,7 @@ def __init__(self, initial_stars_schema=None, **kwargs):
             initial_stars_schema = afwTable.SourceTable.makeMinimalSchema()
         afwTable.CoordKey.addErrorFields(initial_stars_schema)
         self.makeSubtask("star_detection", schema=initial_stars_schema)
+        self.makeSubtask("star_mask_streaks")
         self.makeSubtask("star_deblend", schema=initial_stars_schema)
         self.makeSubtask("star_measurement", schema=initial_stars_schema)
         self.makeSubtask("star_apply_aperture_correction", schema=initial_stars_schema)
@@ -536,6 +545,10 @@ def _find_stars(self, exposure, background):
         # measurement uses the most accurate background-subtraction.
         detections = self.star_detection.run(table=table, exposure=exposure, background=background)
         sources = detections.sources
+
+        # Mask streaks
+        self.star_mask_streaks.run(exposure)
+
         # TODO investigation: Could this deblender throw away blends of non-PSF sources?
         self.star_deblend.run(exposure=exposure, sources=sources)
         # The deblender may not produce a contiguous catalog; ensure

python/lsst/pipe/tasks/characterizeImage.py

@@ -51,6 +51,7 @@
 import lsst.meas.extensions.shapeHSM  # noqa: F401 needed for default shape plugin
 from .measurePsf import MeasurePsfTask
 from .repair import RepairTask
+from .maskStreaks import MaskStreaksTask
 from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask
 from lsst.pex.exceptions import LengthError
 from lsst.utils.timer import timeMethod
@@ -224,6 +225,16 @@ class CharacterizeImageConfig(pipeBase.PipelineTaskConfig,
         dtype=str,
         default="raise",
     )
+    doMaskStreaks = pexConfig.Field(
+        doc="Mask streaks",
+        default=True,
+        dtype=bool,
+    )
+    maskStreaks = pexConfig.ConfigurableField(
+        target=MaskStreaksTask,
+        doc="Subtask for masking streaks. Only used if doMaskStreaks is True. "
+            "Adds a mask plane to an exposure, with the mask plane name set by streakMaskName.",
+    )
     idGenerator = DetectorVisitIdGeneratorConfig.make_field()
 
     def setDefaults(self):
@@ -271,6 +282,7 @@ class CharacterizeImageTask(pipeBase.PipelineTask):
     e.g. as output by `~lsst.ip.isr.IsrTask`):
     - detect and measure bright sources
     - repair cosmic rays
+    - detect and mask streaks
     - measure and subtract background
     - measure PSF
 
@@ -318,6 +330,8 @@ def __init__(self, refObjLoader=None, schema=None, **kwargs):
         self.makeSubtask("background")
         self.makeSubtask("installSimplePsf")
         self.makeSubtask("repair")
+        if self.config.doMaskStreaks:
+            self.makeSubtask("maskStreaks")
         self.makeSubtask("measurePsf", schema=self.schema)
         # TODO DM-34769: remove this `if` block
         if self.config.doMeasurePsf and self.measurePsf.usesMatches:
@@ -430,6 +444,10 @@ def run(self, exposure, exposureIdInfo=None, background=None, idGenerator=None):
         self.repair.run(exposure=dmeRes.exposure)
         self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
 
+        # mask streaks
+        if self.config.doMaskStreaks:
+            _ = self.maskStreaks.run(dmeRes.exposure)
+
         # perform final measurement with final PSF, including measuring and applying aperture correction,
         # if wanted
         self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,

python/lsst/pipe/tasks/finalizeCharacterization.py

@@ -215,6 +215,11 @@ def setDefaults(self):
         self.measurement.slots.shape = 'ext_shapeHSM_HsmSourceMoments'
         self.measurement.slots.psfShape = 'ext_shapeHSM_HsmPsfMoments'
         self.measurement.plugins['ext_shapeHSM_HsmShapeRegauss'].deblendNChild = ""
+
+        # Keep track of which footprints contain streaks
+        self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK']
+        self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK']
+
         # Turn off slot setting for measurement for centroid and shape
         # (for which we use the input src catalog measurements)
         self.measurement.slots.centroid = None

python/lsst/pipe/tasks/multiBand.py

@@ -448,9 +448,9 @@ def setDefaults(self):
                                            'base_LocalPhotoCalib',
                                            'base_LocalWcs']
         self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
-                                                                       'INEXACT_PSF']
+                                                                       'INEXACT_PSF', 'STREAK']
         self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
-                                                                     'INEXACT_PSF']
+                                                                     'INEXACT_PSF', 'STREAK']
 
 
 class MeasureMergedCoaddSourcesTask(PipelineTask):

schemas/ForcedSource.yaml

@@ -1,4 +1,4 @@
-# This file defines the mapping between the columns in a forced source table 
+# This file defines the mapping between the columns in a forced source table
 # and their respective DPDD-style column names, as used by
 # `lsst.pipe.tasks.postprocess.TransformForcedSourceTableTask`.
 # See the DPDD for more information about the output: https://lse-163.lsst.io
@@ -91,6 +91,8 @@ calexpFlags:
     - base_PixelFlags_flag_saturatedCenter
     - base_PixelFlags_flag_crCenter
     - base_PixelFlags_flag_suspectCenter
+    - base_PixelFlags_flag_streak
+    - base_PixelFlags_flag_streakCenter
 flag_rename_rules:
     - ['base_PixelFlags_flag', 'pixelFlags']
     - ['base_Local', 'local']

schemas/Object.yaml

@@ -659,6 +659,8 @@ flags:
     - base_PixelFlags_flag_sensor_edgeCenter
     - base_PixelFlags_flag_suspect
     - base_PixelFlags_flag_suspectCenter
+    - base_PixelFlags_flag_streak
+    - base_PixelFlags_flag_streakCenter
     - base_ClassificationExtendedness_flag
     - calib_astrometry_used
     - calib_photometry_reserved

schemas/Source.yaml

@@ -44,13 +44,13 @@ funcs:
    # compatibility. To be removed after September 2023.
     decl:
         functor: DecColumn
-    
+
     raErr:
         functor: RAErrColumn
     decErr:
         functor: DecErrColumn
     ra_dec_Cov:
-        functor: RADecCovColumn    
+        functor: RADecCovColumn
     # One calibrated Calib flux is important:
     calibFlux:
         functor: LocalNanojansky
@@ -369,6 +369,8 @@ flags:
    - base_PixelFlags_flag_saturatedCenter
    - base_PixelFlags_flag_suspect
    - base_PixelFlags_flag_suspectCenter
+   - base_PixelFlags_flag_streak
+   - base_PixelFlags_flag_streakCenter
    - base_PsfFlux_apCorr
    - base_PsfFlux_apCorrErr
    - base_PsfFlux_area