From 7a1440fa4c3a5b73f2de3d5348bdaf6f5feaf7b1 Mon Sep 17 00:00:00 2001 From: Meredith Rawls Date: Fri, 18 Aug 2023 16:48:45 -0700 Subject: [PATCH 1/2] Add streak masking to single frame processing. This implementation is in both characterizeImage -> calibrate as well as the newer calibrateImage. It adds streak and streakCenter flag columns to the Source, Object, and ForcedSource schemas. --- python/lsst/pipe/tasks/calibrate.py | 4 ++++ python/lsst/pipe/tasks/calibrateImage.py | 15 ++++++++++++++- python/lsst/pipe/tasks/characterizeImage.py | 18 ++++++++++++++++++ schemas/ForcedSource.yaml | 4 +++- schemas/Object.yaml | 2 ++ schemas/Source.yaml | 6 ++++-- 6 files changed, 45 insertions(+), 4 deletions(-) diff --git a/python/lsst/pipe/tasks/calibrate.py b/python/lsst/pipe/tasks/calibrate.py index 58201066e..ac2d6b2b8 100644 --- a/python/lsst/pipe/tasks/calibrate.py +++ b/python/lsst/pipe/tasks/calibrate.py @@ -311,6 +311,10 @@ def setDefaults(self): # The photoRefCat connection is the name to use for the colorterms. self.photoCal.photoCatName = self.connections.photoRefCat + # Keep track of which footprints contain streaks + self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK'] + self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK'] + class CalibrateTask(pipeBase.PipelineTask): """Calibrate an exposure: measure sources and perform astrometric and diff --git a/python/lsst/pipe/tasks/calibrateImage.py b/python/lsst/pipe/tasks/calibrateImage.py index 8a2f0f55d..082c11597 100644 --- a/python/lsst/pipe/tasks/calibrateImage.py +++ b/python/lsst/pipe/tasks/calibrateImage.py @@ -33,7 +33,7 @@ from lsst.pipe.base import connectionTypes from lsst.utils.timer import timeMethod -from . import measurePsf, repair, setPrimaryFlags, photoCal, computeExposureSummaryStats +from . import measurePsf, repair, setPrimaryFlags, photoCal, computeExposureSummaryStats, maskStreaks class CalibrateImageConnections(pipeBase.PipelineTaskConnections, @@ -181,6 +181,10 @@ class CalibrateImageConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Cali target=lsst.meas.algorithms.SourceDetectionTask, doc="Task to detect stars to return in the output catalog." ) + star_mask_streaks = pexConfig.ConfigurableField( + target=maskStreaks.MaskStreaksTask, + doc="Task for masking streaks. Adds a STREAK mask plane to an exposure.", + ) star_deblend = pexConfig.ConfigurableField( target=lsst.meas.deblender.SourceDeblendTask, doc="Split blended sources into their components" @@ -292,6 +296,10 @@ def setDefaults(self): # Only measure the apertures we need for star selection. self.star_measurement.plugins["base_CircularApertureFlux"].radii = [12.0] + # Keep track of which footprints contain streaks + self.star_measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK'] + self.star_measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK'] + # Select isolated stars with reliable measurements and no bad flags. self.star_selector["science"].doFlags = True self.star_selector["science"].doUnresolved = True @@ -344,6 +352,7 @@ def __init__(self, initial_stars_schema=None, **kwargs): initial_stars_schema = afwTable.SourceTable.makeMinimalSchema() afwTable.CoordKey.addErrorFields(initial_stars_schema) self.makeSubtask("star_detection", schema=initial_stars_schema) + self.makeSubtask("star_mask_streaks") self.makeSubtask("star_deblend", schema=initial_stars_schema) self.makeSubtask("star_measurement", schema=initial_stars_schema) self.makeSubtask("star_apply_aperture_correction", schema=initial_stars_schema) @@ -536,6 +545,10 @@ def _find_stars(self, exposure, background): # measurement uses the most accurate background-subtraction. detections = self.star_detection.run(table=table, exposure=exposure, background=background) sources = detections.sources + + # Mask streaks + self.star_mask_streaks.run(exposure) + # TODO investigation: Could this deblender throw away blends of non-PSF sources? self.star_deblend.run(exposure=exposure, sources=sources) # The deblender may not produce a contiguous catalog; ensure diff --git a/python/lsst/pipe/tasks/characterizeImage.py b/python/lsst/pipe/tasks/characterizeImage.py index f9e0da29c..1095b74ac 100644 --- a/python/lsst/pipe/tasks/characterizeImage.py +++ b/python/lsst/pipe/tasks/characterizeImage.py @@ -51,6 +51,7 @@ import lsst.meas.extensions.shapeHSM # noqa: F401 needed for default shape plugin from .measurePsf import MeasurePsfTask from .repair import RepairTask +from .maskStreaks import MaskStreaksTask from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask from lsst.pex.exceptions import LengthError from lsst.utils.timer import timeMethod @@ -224,6 +225,16 @@ class CharacterizeImageConfig(pipeBase.PipelineTaskConfig, dtype=str, default="raise", ) + doMaskStreaks = pexConfig.Field( + doc="Mask streaks", + default=True, + dtype=bool, + ) + maskStreaks = pexConfig.ConfigurableField( + target=MaskStreaksTask, + doc="Subtask for masking streaks. Only used if doMaskStreaks is True. " + "Adds a mask plane to an exposure, with the mask plane name set by streakMaskName.", + ) idGenerator = DetectorVisitIdGeneratorConfig.make_field() def setDefaults(self): @@ -271,6 +282,7 @@ class CharacterizeImageTask(pipeBase.PipelineTask): e.g. as output by `~lsst.ip.isr.IsrTask`): - detect and measure bright sources - repair cosmic rays + - detect and mask streaks - measure and subtract background - measure PSF @@ -318,6 +330,8 @@ def __init__(self, refObjLoader=None, schema=None, **kwargs): self.makeSubtask("background") self.makeSubtask("installSimplePsf") self.makeSubtask("repair") + if self.config.doMaskStreaks: + self.makeSubtask("maskStreaks") self.makeSubtask("measurePsf", schema=self.schema) # TODO DM-34769: remove this `if` block if self.config.doMeasurePsf and self.measurePsf.usesMatches: @@ -430,6 +444,10 @@ def run(self, exposure, exposureIdInfo=None, background=None, idGenerator=None): self.repair.run(exposure=dmeRes.exposure) self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat) + # mask streaks + if self.config.doMaskStreaks: + _ = self.maskStreaks.run(dmeRes.exposure) + # perform final measurement with final PSF, including measuring and applying aperture correction, # if wanted self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure, diff --git a/schemas/ForcedSource.yaml b/schemas/ForcedSource.yaml index a04535951..6b709edac 100644 --- a/schemas/ForcedSource.yaml +++ b/schemas/ForcedSource.yaml @@ -1,4 +1,4 @@ -# This file defines the mapping between the columns in a forced source table +# This file defines the mapping between the columns in a forced source table # and their respective DPDD-style column names, as used by # `lsst.pipe.tasks.postprocess.TransformForcedSourceTableTask`. # See the DPDD for more information about the output: https://lse-163.lsst.io @@ -91,6 +91,8 @@ calexpFlags: - base_PixelFlags_flag_saturatedCenter - base_PixelFlags_flag_crCenter - base_PixelFlags_flag_suspectCenter + - base_PixelFlags_flag_streak + - base_PixelFlags_flag_streakCenter flag_rename_rules: - ['base_PixelFlags_flag', 'pixelFlags'] - ['base_Local', 'local'] diff --git a/schemas/Object.yaml b/schemas/Object.yaml index 36f4a8f4c..89be9c6f3 100644 --- a/schemas/Object.yaml +++ b/schemas/Object.yaml @@ -659,6 +659,8 @@ flags: - base_PixelFlags_flag_sensor_edgeCenter - base_PixelFlags_flag_suspect - base_PixelFlags_flag_suspectCenter + - base_PixelFlags_flag_streak + - base_PixelFlags_flag_streakCenter - base_ClassificationExtendedness_flag - calib_astrometry_used - calib_photometry_reserved diff --git a/schemas/Source.yaml b/schemas/Source.yaml index fb50da57e..df3198c0b 100644 --- a/schemas/Source.yaml +++ b/schemas/Source.yaml @@ -44,13 +44,13 @@ funcs: # compatibility. To be removed after September 2023. decl: functor: DecColumn - + raErr: functor: RAErrColumn decErr: functor: DecErrColumn ra_dec_Cov: - functor: RADecCovColumn + functor: RADecCovColumn # One calibrated Calib flux is important: calibFlux: functor: LocalNanojansky @@ -369,6 +369,8 @@ flags: - base_PixelFlags_flag_saturatedCenter - base_PixelFlags_flag_suspect - base_PixelFlags_flag_suspectCenter + - base_PixelFlags_flag_streak + - base_PixelFlags_flag_streakCenter - base_PsfFlux_apCorr - base_PsfFlux_apCorrErr - base_PsfFlux_area From 47cee018ec436f3f55d4c00bc163d633092b4196 Mon Sep 17 00:00:00 2001 From: Meredith Rawls Date: Tue, 24 Oct 2023 23:07:19 -0700 Subject: [PATCH 2/2] Add streak mask propagation to DRP tasks --- python/lsst/pipe/tasks/finalizeCharacterization.py | 5 +++++ python/lsst/pipe/tasks/multiBand.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/python/lsst/pipe/tasks/finalizeCharacterization.py b/python/lsst/pipe/tasks/finalizeCharacterization.py index 444ce22d8..49625b0a4 100644 --- a/python/lsst/pipe/tasks/finalizeCharacterization.py +++ b/python/lsst/pipe/tasks/finalizeCharacterization.py @@ -215,6 +215,11 @@ def setDefaults(self): self.measurement.slots.shape = 'ext_shapeHSM_HsmSourceMoments' self.measurement.slots.psfShape = 'ext_shapeHSM_HsmPsfMoments' self.measurement.plugins['ext_shapeHSM_HsmShapeRegauss'].deblendNChild = "" + + # Keep track of which footprints contain streaks + self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['STREAK'] + self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['STREAK'] + # Turn off slot setting for measurement for centroid and shape # (for which we use the input src catalog measurements) self.measurement.slots.centroid = None diff --git a/python/lsst/pipe/tasks/multiBand.py b/python/lsst/pipe/tasks/multiBand.py index 0b18c1f44..f0c1d99dc 100644 --- a/python/lsst/pipe/tasks/multiBand.py +++ b/python/lsst/pipe/tasks/multiBand.py @@ -448,9 +448,9 @@ def setDefaults(self): 'base_LocalPhotoCalib', 'base_LocalWcs'] self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE', - 'INEXACT_PSF'] + 'INEXACT_PSF', 'STREAK'] self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE', - 'INEXACT_PSF'] + 'INEXACT_PSF', 'STREAK'] class MeasureMergedCoaddSourcesTask(PipelineTask):