diff --git a/python/lsst/pipe/tasks/finalizeCharacterization.py b/python/lsst/pipe/tasks/finalizeCharacterization.py index 956251681..5ae15f077 100644 --- a/python/lsst/pipe/tasks/finalizeCharacterization.py +++ b/python/lsst/pipe/tasks/finalizeCharacterization.py @@ -26,6 +26,8 @@ 'FinalizeCharacterizationConfig', 'FinalizeCharacterizationTask'] +import logging + import numpy as np import esutil import pandas as pd @@ -43,17 +45,20 @@ from .reserveIsolatedStars import ReserveIsolatedStarsTask +_LOG = logging.getLogger(__name__) + + class FinalizeCharacterizationConnections(pipeBase.PipelineTaskConnections, dimensions=('instrument', 'visit',), defaultTemplates={}): src_schema = pipeBase.connectionTypes.InitInput( doc='Input schema used for src catalogs.', - name='src_schema', + name='initial_stars_schema', storageClass='SourceCatalog', ) srcs = pipeBase.connectionTypes.Input( doc='Source catalogs for the visit', - name='src', + name='initial_stars_footprints_detector', storageClass='SourceCatalog', dimensions=('instrument', 'visit', 'detector'), deferLoad=True, @@ -61,7 +66,7 @@ class FinalizeCharacterizationConnections(pipeBase.PipelineTaskConnections, ) calexps = pipeBase.connectionTypes.Input( doc='Calexps for the visit', - name='calexp', + name='initial_pvi', storageClass='ExposureF', dimensions=('instrument', 'visit', 'detector'), deferLoad=True, @@ -85,6 +90,17 @@ class FinalizeCharacterizationConnections(pipeBase.PipelineTaskConnections, deferLoad=True, multiple=True, ) + initial_photo_calibs = pipeBase.connectionTypes.Input( + doc=("Initial photometric calibration that was already applied to " + "calexps, to be removed prior to measurement in order to recover " + "instrumental fluxes."), + name="initial_photoCalib_detector", + storageClass="PhotoCalib", + dimensions=("instrument", "visit", "detector"), + multiple=True, + deferLoad=True, + minimum=0, + ) finalized_psf_ap_corr_cat = pipeBase.connectionTypes.Output( doc=('Per-visit finalized psf models and aperture corrections. This ' 'catalog uses detector id for the id and are sorted for fast ' @@ -100,6 +116,28 @@ class FinalizeCharacterizationConnections(pipeBase.PipelineTaskConnections, dimensions=('instrument', 'visit'), ) + def adjustQuantum(self, inputs, outputs, label, data_id): + if self.config.remove_initial_photo_calib and not inputs["initial_photo_calibs"]: + _LOG.warning( + "Dropping %s quantum %s because initial photo calibs are needed and none were present " + "this may be an upstream partial-outputs error covering an entire visit (which is why this " + "is not an error), but it may mean that 'config.remove_initial_photo_calib' should be " + "False.", + label, + data_id, + ) + raise pipeBase.NoWorkFound("No initial photo calibs.") + elif not self.config.remove_initial_photo_calib and inputs["initial_photo_calibs"]: + _LOG.warning( + "For %s quantum %s, input collections have initial photo calib datasets but " + "'config.remove_initial_photo_calib=False'. This is either a very unusual collection " + "search path or (more likely) a bad configuration. Not that this config option should " + "be true when using images produced by CalibrateImageTask.", + label, + data_id, + ) + return super().adjustQuantum(inputs, outputs, label, data_id) + class FinalizeCharacterizationConfig(pipeBase.PipelineTaskConfig, pipelineConnections=FinalizeCharacterizationConnections): @@ -113,6 +151,12 @@ class FinalizeCharacterizationConfig(pipeBase.PipelineTaskConfig, dtype=str, default='sourceId', ) + remove_initial_photo_calib = pexConfig.Field( + doc=("Expect an initial photo calib input to be present, and use it ", + "to restore the image to instrumental units."), + dtype=bool, + default=True, + ) reserve_selection = pexConfig.ConfigurableField( target=ReserveIsolatedStarsTask, doc='Task to select reserved stars', @@ -269,6 +313,8 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): for handle in input_handle_dict['srcs']} calexp_dict_temp = {handle.dataId['detector']: handle for handle in input_handle_dict['calexps']} + initial_photo_calib_dict_temp = {handle.dataId['detector']: handle + for handle in input_handle_dict['initial_photo_calibs']} isolated_star_cat_dict_temp = {handle.dataId['tract']: handle for handle in input_handle_dict['isolated_star_cats']} isolated_star_source_dict_temp = {handle.dataId['tract']: handle @@ -279,6 +325,8 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): detector in sorted(src_dict_temp.keys())} calexp_dict = {detector: calexp_dict_temp[detector] for detector in sorted(calexp_dict_temp.keys())} + initial_photo_calib_dict = {detector: initial_photo_calib_dict_temp[detector] + for detector in sorted(initial_photo_calib_dict_temp.keys())} isolated_star_cat_dict = {tract: isolated_star_cat_dict_temp[tract] for tract in sorted(isolated_star_cat_dict_temp.keys())} isolated_star_source_dict = {tract: isolated_star_source_dict_temp[tract] for @@ -289,14 +337,24 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): isolated_star_cat_dict, isolated_star_source_dict, src_dict, - calexp_dict) + calexp_dict, + initial_photo_calib_dict) butlerQC.put(struct.psf_ap_corr_cat, outputRefs.finalized_psf_ap_corr_cat) butlerQC.put(pd.DataFrame(struct.output_table), outputRefs.finalized_src_table) - def run(self, visit, band, isolated_star_cat_dict, isolated_star_source_dict, src_dict, calexp_dict): + def run( + self, + visit, + band, + isolated_star_cat_dict, + isolated_star_source_dict, + src_dict, + calexp_dict, + initial_photo_calib_dict, + ): """ Run the FinalizeCharacterizationTask. @@ -314,6 +372,8 @@ def run(self, visit, band, isolated_star_cat_dict, isolated_star_source_dict, sr Per-detector dict of src catalog handles. calexp_dict : `dict` Per-detector dict of calibrated exposure handles. + initial_photo_calib_dict : `dict` + Per-detector dict of initial photometric calibration handles Returns ------- @@ -349,13 +409,18 @@ def run(self, visit, band, isolated_star_cat_dict, isolated_star_source_dict, sr for detector in src_dict: src = src_dict[detector].get() exposure = calexp_dict[detector].get() + if detector in initial_photo_calib_dict: + initial_photo_calib = initial_photo_calib_dict[detector].get() + else: + initial_photo_calib = None psf, ap_corr_map, measured_src = self.compute_psf_and_ap_corr_map( visit, detector, exposure, src, - isolated_source_table + isolated_source_table, + initial_photo_calib ) # And now we package it together... @@ -591,7 +656,15 @@ def concat_isolated_star_cats(self, band, isolated_star_cat_dict, isolated_star_ return isolated_table, isolated_source_table - def compute_psf_and_ap_corr_map(self, visit, detector, exposure, src, isolated_source_table): + def compute_psf_and_ap_corr_map( + self, + visit, + detector, + exposure, + src, + isolated_source_table, + initial_photo_calib, + ): """Compute psf model and aperture correction map for a single exposure. Parameters @@ -603,6 +676,8 @@ def compute_psf_and_ap_corr_map(self, visit, detector, exposure, src, isolated_s exposure : `lsst.afw.image.ExposureF` src : `lsst.afw.table.SourceCatalog` isolated_source_table : `np.ndarray` + initial_photo_calib : `lsst.afw.image.PhotoCalib` or `None` + Initial photometric calibration to remove from the image. Returns ------- @@ -613,6 +688,17 @@ def compute_psf_and_ap_corr_map(self, visit, detector, exposure, src, isolated_s measured_src : `lsst.afw.table.SourceCatalog` Updated source catalog with measurements, flags and aperture corrections. """ + if self.config.remove_initial_photo_calib: + if initial_photo_calib is None: + self.log.warning("No initial photo calib found for visit %d, detector %d", visit, detector) + return None, None, None + if not initial_photo_calib._isConstant: + # TODO DM-46720: remove this limitation and usage of private (why?!) property. + raise NotImplementedError( + "removeInitialPhotoCalib=True can only work when the initialPhotoCalib is constant." + ) + exposure.maskedImage /= initial_photo_calib.getCalibrationMean() + # Extract footprints from the input src catalog for noise replacement. footprints = SingleFrameMeasurementTask.getFootprintsFromCatalog(src) diff --git a/python/lsst/pipe/tasks/makeWarp.py b/python/lsst/pipe/tasks/makeWarp.py index 64469a867..d867e09ad 100644 --- a/python/lsst/pipe/tasks/makeWarp.py +++ b/python/lsst/pipe/tasks/makeWarp.py @@ -50,21 +50,21 @@ class MakeWarpConnections(pipeBase.PipelineTaskConnections, "calexpType": ""}): calExpList = connectionTypes.Input( doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch", - name="{calexpType}calexp", + name="{calexpType}initial_pvi", storageClass="ExposureF", dimensions=("instrument", "visit", "detector"), multiple=True, deferLoad=True, ) backgroundList = connectionTypes.Input( - doc="Input backgrounds to be added back into the calexp if bgSubtracted=False", - name="calexpBackground", + doc="Input backgrounds to be added back into the exposure if bgSubtracted=False", + name="initial_pvi_background", storageClass="Background", dimensions=("instrument", "visit", "detector"), multiple=True, ) skyCorrList = connectionTypes.Input( - doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True", + doc="Input Sky Correction to be subtracted from the exposure if doApplySkyCorr=True", name="skyCorr", storageClass="Background", dimensions=("instrument", "visit", "detector"), @@ -77,14 +77,14 @@ class MakeWarpConnections(pipeBase.PipelineTaskConnections, dimensions=("skymap",), ) direct = connectionTypes.Output( - doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ", + doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling " "calexps onto the skyMap patch geometry."), name="{coaddName}Coadd_directWarp", storageClass="ExposureF", dimensions=("tract", "patch", "skymap", "visit", "instrument"), ) psfMatched = connectionTypes.Output( - doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ", + doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling " "calexps onto the skyMap patch geometry and PSF-matching to a model PSF."), name="{coaddName}Coadd_psfMatchedWarp", storageClass="ExposureF", @@ -96,6 +96,15 @@ class MakeWarpConnections(pipeBase.PipelineTaskConnections, storageClass="ExposureCatalog", dimensions=("instrument", "visit",), ) + initialPhotoCalibList = pipeBase.connectionTypes.Input( + doc=("Initial photometric calibration that was already applied to calExpList, " + "to be removed prior to applying the final photometric calibration."), + name="initial_photoCalib_detector", + storageClass="PhotoCalib", + dimensions=("instrument", "visit", "detector"), + multiple=True, + minimum=0, + ) def __init__(self, *, config=None): if config.bgSubtracted: @@ -107,6 +116,33 @@ def __init__(self, *, config=None): if not config.makePsfMatched: del self.psfMatched + def adjustQuantum(self, inputs, outputs, label, data_id): + # Instead of disabling the initial photo calibs connection when + # config.removeInitialPhotoCalib is False, check here (in QG + # generation) that the configuration is consistent with what's + # available. This gives us a chance at least warn if somebody + # runs CalibrateImageTask but doesn't configure MakeWarpTask to + # use it properly. + if self.config.removeInitialPhotoCalib and not inputs["initialPhotoCalibList"]: + log.warning( + "Dropping %s quantum %s because initial photo calibs are needed and none were present " + "this may be an upstream partial-outputs error covering an entire visit (which is why this " + "is not an error), but it may mean that 'config.removeInitialPhotoCalib' should be False.", + label, + data_id, + ) + raise pipeBase.NoWorkFound("No initial photo calibs.") + elif not self.config.removeInitialPhotoCalib and inputs["initialPhotoCalibList"]: + log.warning( + "For %s quantum %s, input collections have initial photo calib datasets but " + "'config.removeInitialPhotoCalib=False'. This is either a very unusual collection " + "search path or (more likely) a bad configuration. Not that this config option should " + "be true when using images produced by CalibrateImageTask.", + label, + data_id, + ) + return super().adjustQuantum(inputs, outputs, label, data_id) + class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass, pipelineConnections=MakeWarpConnections): @@ -164,6 +200,12 @@ class MakeWarpConfig(pipeBase.PipelineTaskConfig, CoaddBaseTask.ConfigClass, default=False, doc="Apply sky correction?", ) + removeInitialPhotoCalib = pexConfig.Field( + doc=("Expect an initial photo calib input to be present, and use it to restore the image ", + "to instrumental units before applying the final photometric calibration."), + dtype=bool, + default=True, + ) idGenerator = DetectorVisitIdGeneratorConfig.make_field() def validate(self): @@ -225,7 +267,7 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs): detector order (to ensure reproducibility). Then ensure all input lists are in the same sorted detector order. """ - detectorOrder = [ref.datasetRef.dataId['detector'] for ref in inputRefs.calExpList] + detectorOrder = [handle.datasetRef.dataId['detector'] for handle in inputRefs.calExpList] detectorOrder.sort() inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') @@ -299,9 +341,9 @@ def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwar (violates LSST algorithms group policy), but will be fixed up by interpolating after the coaddition. - calexpRefList : `list` - List of data references for calexps that (may) - overlap the patch of interest. + calExpList : `list` [ `lsst.afw.image.Exposure` ] + List of single-detector input images that (may) overlap the patch + of interest. skyInfo : `lsst.pipe.base.Struct` Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with geometric information about the patch. @@ -425,7 +467,8 @@ def filterInputs(self, indices, inputs): return inputs def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=None, - backgroundList=None, skyCorrList=None, **kwargs): + backgroundList=None, skyCorrList=None, initialPhotoCalibList=None, + **kwargs): """Calibrate and add backgrounds to input calExpList in place. Parameters @@ -435,16 +478,22 @@ def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=No to `None` are ignored. calExpList : `list` [`lsst.afw.image.Exposure` or `lsst.daf.butler.DeferredDatasetHandle`] - Sequence of calexps to be modified in place. + Sequence of single-epoch images (or deferred load handles for + images) to be modified in place. On return this always has images, + not handles. wcsList : `list` [`lsst.afw.geom.SkyWcs`] The WCSs of the calexps in ``calExpList``. These will be used to determine if the calexp should be used in the warp. The list is dynamically updated with the WCSs from the visitSummary. - backgroundList : `list` [`lsst.afw.math.backgroundList`], optional + backgroundList : `list` [`lsst.afw.math.BackgroundList`], optional Sequence of backgrounds to be added back in if bgSubtracted=False. - skyCorrList : `list` [`lsst.afw.math.backgroundList`], optional + skyCorrList : `list` [`lsst.afw.math.BackgroundList`], optional Sequence of background corrections to be subtracted if doApplySkyCorr=True. + initialPhotoCalibList : `list [`lsst.afw.image.PhotoCalib`], optional + Initial photometric calibrations that were already applied to the + images in `calExpList` and must be removed before applying the + final photometric calibration. **kwargs Additional keyword arguments. @@ -457,13 +506,16 @@ def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=No wcsList = len(calExpList)*[None] if wcsList is None else wcsList backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList + initialPhotoCalibList = ( + len(calExpList)*[None] if initialPhotoCalibList is None else initialPhotoCalibList + ) includeCalibVar = self.config.includeCalibVar indices = [] - for index, (calexp, background, skyCorr) in enumerate(zip(calExpList, - backgroundList, - skyCorrList)): + for index, (calexp, background, skyCorr, initialPhotoCalib) in enumerate( + zip(calExpList, backgroundList, skyCorrList, initialPhotoCalibList) + ): if isinstance(calexp, DeferredDatasetHandle): calexp = calexp.get() @@ -526,6 +578,26 @@ def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=No ) continue + # Apply skycorr. We assume it has the same units as the calExpList + # image. + if self.config.doApplySkyCorr: + calexp.maskedImage -= skyCorr.getImage() + + # Remove initial photometric calibration, if we expect one. + if self.config.removeInitialPhotoCalib: + if initialPhotoCalib is None: + self.log.warning( + "Detector id %d for visit %d has no initialPhotoCalib and will " + "not be used in the warp", detectorId, row["visit"], + ) + continue + if not initialPhotoCalib._isConstant: + # TODO DM-46720: remove this limitation and usage of private (why?!) property. + raise NotImplementedError( + "removeInitialPhotoCalib=True can only work when the initialPhotoCalib is constant." + ) + calexp.maskedImage /= initialPhotoCalib.getCalibrationMean() + # Calibrate the image. calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage, includeScaleUncertainty=includeCalibVar) @@ -534,10 +606,6 @@ def _prepareCalibratedExposures(self, *, visitSummary, calExpList=[], wcsList=No # RFC-545 is implemented. # exposure.setCalib(afwImage.Calib(1.0)) - # Apply skycorr - if self.config.doApplySkyCorr: - calexp.maskedImage -= skyCorr.getImage() - indices.append(index) calExpList[index] = calexp @@ -606,7 +674,7 @@ def reorderRefs(inputRefs, outputSortKeyOrder, dataIdKey): if hasattr(refs[0], "dataId"): inputSortKeyOrder = [ref.dataId[dataIdKey] for ref in refs] else: - inputSortKeyOrder = [ref.datasetRef.dataId[dataIdKey] for ref in refs] + inputSortKeyOrder = [handle.datasetRef.dataId[dataIdKey] for handle in refs] if inputSortKeyOrder != outputSortKeyOrder: setattr(inputRefs, connectionName, reorderAndPadList(refs, inputSortKeyOrder, outputSortKeyOrder)) diff --git a/python/lsst/pipe/tasks/make_direct_warp.py b/python/lsst/pipe/tasks/make_direct_warp.py index 5bc35f2b4..cec1ea2cc 100644 --- a/python/lsst/pipe/tasks/make_direct_warp.py +++ b/python/lsst/pipe/tasks/make_direct_warp.py @@ -78,7 +78,7 @@ class MakeDirectWarpConnections( calexp_list = Input( doc="Input exposures to be interpolated and resampled onto a SkyMap " "projection/patch.", - name="{calexpType}calexp", + name="{calexpType}initial_pvi", storageClass="ExposureF", dimensions=("instrument", "visit", "detector"), multiple=True, @@ -87,7 +87,7 @@ class MakeDirectWarpConnections( background_revert_list = Input( doc="Background to be reverted (i.e., added back to the calexp). " "This connection is used only if doRevertOldBackground=False.", - name="calexpBackground", + name="initial_pvi_background", storageClass="Background", dimensions=("instrument", "visit", "detector"), multiple=True, diff --git a/python/lsst/pipe/tasks/metrics.py b/python/lsst/pipe/tasks/metrics.py index d6ef47a3c..9713b0dac 100644 --- a/python/lsst/pipe/tasks/metrics.py +++ b/python/lsst/pipe/tasks/metrics.py @@ -41,7 +41,7 @@ class NumberDeblendedSourcesMetricConnections( ): sources = connectionTypes.Input( doc="The catalog of science sources.", - name="src", + name="initial_stars_footprints_detector", storageClass="SourceCatalog", dimensions={"instrument", "visit", "detector"}, ) @@ -120,7 +120,7 @@ class NumberDeblendChildSourcesMetricConnections( ): sources = connectionTypes.Input( doc="The catalog of science sources.", - name="src", + name="initial_stars_footprints_detector", storageClass="SourceCatalog", dimensions={"instrument", "visit", "detector"}, ) diff --git a/python/lsst/pipe/tasks/multiBand.py b/python/lsst/pipe/tasks/multiBand.py index 33e4d729e..e99de63fb 100644 --- a/python/lsst/pipe/tasks/multiBand.py +++ b/python/lsst/pipe/tasks/multiBand.py @@ -260,7 +260,7 @@ class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections, doc="Source catalogs for visits which overlap input tract, patch, band. Will be " "further filtered in the task for the purpose of propagating flags from image calibration " "and characterization to coadd objects. Only used in legacy PropagateVisitFlagsTask.", - name="src", + name="initial_stars_footprints_detector", dimensions=("instrument", "visit", "detector"), storageClass="SourceCatalog", multiple=True @@ -449,11 +449,13 @@ def setDefaults(self): 'base_LocalPhotoCalib', 'base_LocalWcs'] - # TODO: Remove STREAK in DM-44658, streak masking to happen only in ip_diffim + # TODO: Remove STREAK in DM-44658, streak masking to happen only in + # ip_diffim; if we can propagate the streak mask from diffim, we can + # still set flags with it here. self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE', - 'INEXACT_PSF', 'STREAK'] + 'INEXACT_PSF'] self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE', - 'INEXACT_PSF', 'STREAK'] + 'INEXACT_PSF'] def validate(self): super().validate() diff --git a/python/lsst/pipe/tasks/postprocess.py b/python/lsst/pipe/tasks/postprocess.py index 2418056fa..3be60d31e 100644 --- a/python/lsst/pipe/tasks/postprocess.py +++ b/python/lsst/pipe/tasks/postprocess.py @@ -188,30 +188,33 @@ def run(self, catalogs, tract, patch): return catalog +# TODO: should deprecate this? class WriteSourceTableConnections(pipeBase.PipelineTaskConnections, defaultTemplates={"catalogType": ""}, dimensions=("instrument", "visit", "detector")): catalog = connectionTypes.Input( doc="Input full-depth catalog of sources produced by CalibrateTask", - name="{catalogType}src", + name="{catalogType}initial_stars_footprints_detector", storageClass="SourceCatalog", dimensions=("instrument", "visit", "detector") ) outputCatalog = connectionTypes.Output( doc="Catalog of sources, `src` in DataFrame/Parquet format. The 'id' column is " "replaced with an index; all other columns are unchanged.", - name="{catalogType}source", + name="{catalogType}initial_stars_detector", storageClass="DataFrame", dimensions=("instrument", "visit", "detector") ) +# TODO: should deprecate this! class WriteSourceTableConfig(pipeBase.PipelineTaskConfig, pipelineConnections=WriteSourceTableConnections): pass +# TODO: should deprecate this! class WriteSourceTableTask(pipeBase.PipelineTask): """Write source table to DataFrame Parquet format. """ @@ -281,6 +284,7 @@ class WriteRecalibratedSourceTableConfig(WriteSourceTableConfig, ) +# TODO: deprecate, since reprocessVisitImage does this more thoroughly? class WriteRecalibratedSourceTableTask(WriteSourceTableTask): """Write source table to DataFrame Parquet format. """ @@ -959,8 +963,8 @@ class TransformSourceTableConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector")): inputCatalog = connectionTypes.Input( - doc="Wide input catalog of sources produced by WriteSourceTableTask", - name="{catalogType}source", + doc="Wide input catalog of sources produced by WriteSourceTableTask or CalibrateImage.", + name="{catalogType}sources_detector", storageClass="DataFrame", dimensions=("instrument", "visit", "detector"), deferLoad=True @@ -996,7 +1000,7 @@ class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections, defaultTemplates={"calexpType": ""}): calexp = connectionTypes.Input( doc="Processed exposures used for metadata", - name="calexp", + name="initial_pvi", storageClass="ExposureF", dimensions=("instrument", "visit", "detector"), deferLoad=True, @@ -1122,6 +1126,7 @@ def _combineExposureMetadata(self, visit, dataRefs): class ConsolidateSourceTableConnections(pipeBase.PipelineTaskConnections, defaultTemplates={"catalogType": ""}, dimensions=("instrument", "visit")): + # TODO: Deprecate the dataframe connection? inputCatalogs = connectionTypes.Input( doc="Input per-detector Source Tables", name="{catalogType}sourceTable", diff --git a/python/lsst/pipe/tasks/skyCorrection.py b/python/lsst/pipe/tasks/skyCorrection.py index df6f00f1c..0f7454b76 100644 --- a/python/lsst/pipe/tasks/skyCorrection.py +++ b/python/lsst/pipe/tasks/skyCorrection.py @@ -109,7 +109,7 @@ class SkyCorrectionConnections(PipelineTaskConnections, dimensions=("instrument" ) calExps = cT.Input( doc="Background-subtracted calibrated exposures.", - name="calexp", + name="initial_pvi", multiple=True, storageClass="ExposureF", dimensions=["instrument", "visit", "detector"], @@ -117,7 +117,7 @@ class SkyCorrectionConnections(PipelineTaskConnections, dimensions=("instrument" calBkgs = cT.Input( doc="Subtracted backgrounds for input calibrated exposures.", multiple=True, - name="calexpBackground", + name="initial_pvi_background", storageClass="Background", dimensions=["instrument", "visit", "detector"], ) diff --git a/schemas/ForcedSource.yaml b/schemas/ForcedSource.yaml index 70c501ab7..5fae0fe01 100644 --- a/schemas/ForcedSource.yaml +++ b/schemas/ForcedSource.yaml @@ -95,8 +95,9 @@ calexpFlags: - base_PixelFlags_flag_saturatedCenter - base_PixelFlags_flag_crCenter - base_PixelFlags_flag_suspectCenter - - base_PixelFlags_flag_streak - - base_PixelFlags_flag_streakCenter + # Streak flags not yet propagated from difference imaging. + # - base_PixelFlags_flag_streak + # - base_PixelFlags_flag_streakCenter - base_InvalidPsf_flag flag_rename_rules: - ['base_PixelFlags_flag', 'pixelFlags'] diff --git a/schemas/Object.yaml b/schemas/Object.yaml index 8bac141fd..1165c2e69 100644 --- a/schemas/Object.yaml +++ b/schemas/Object.yaml @@ -769,8 +769,9 @@ flags: - base_PixelFlags_flag_sensor_edgeCenter - base_PixelFlags_flag_suspect - base_PixelFlags_flag_suspectCenter - - base_PixelFlags_flag_streak - - base_PixelFlags_flag_streakCenter + # Streak flags not yet propagated from difference imaging. + # - base_PixelFlags_flag_streak + # - base_PixelFlags_flag_streakCenter - base_ClassificationExtendedness_flag - base_ClassificationSizeExtendedness_flag - base_InvalidPsf_flag diff --git a/schemas/Source.yaml b/schemas/Source.yaml index f2a7dfb8f..a860bf384 100644 --- a/schemas/Source.yaml +++ b/schemas/Source.yaml @@ -4,7 +4,8 @@ # See the DPDD for more information about the output: https://lse-163.lsst.io funcs: sourceId: - functor: Index + functor: Column + args: id coord_ra: # reference position required by db. Not in DPDD functor: CoordColumn @@ -380,8 +381,9 @@ flags: - base_PixelFlags_flag_saturatedCenter - base_PixelFlags_flag_suspect - base_PixelFlags_flag_suspectCenter - - base_PixelFlags_flag_streak - - base_PixelFlags_flag_streakCenter + # Streak flags not yet propagated from difference imaging. + # - base_PixelFlags_flag_streak + # - base_PixelFlags_flag_streakCenter - base_PsfFlux_apCorr - base_PsfFlux_apCorrErr - base_PsfFlux_area @@ -401,7 +403,6 @@ flags: - base_Variance_flag_emptyFootprint - base_Variance_value - calib_astrometry_used - - calib_detected - calib_photometry_reserved - calib_photometry_used - calib_psf_candidate diff --git a/schemas/initial_stars_detector_standardized.yaml b/schemas/initial_stars_detector_standardized.yaml new file mode 100644 index 000000000..6e83d5aa1 --- /dev/null +++ b/schemas/initial_stars_detector_standardized.yaml @@ -0,0 +1,196 @@ +# This file defines the mapping between the columns in a single visit+detector +# initial_stars table and their respective DPDD-style column names, as used by +# `lsst.pipe.tasks.postprocess.TransformSourceTableTask`. +# See the DPDD for more information about the output: https://lse-163.lsst.io +funcs: + sourceId: + functor: Column + args: id + coord_ra: + # reference position required by db. Not in DPDD + functor: CoordColumn + args: coord_ra + coord_dec: + # Reference position required by db. Not in DPDD + functor: CoordColumn + args: coord_dec + # objectId: not avaliable + # ssObjectId: not avaliable + parentSourceId: + functor: Column + args: parent + # htmId20: not avaliable + x: + functor: Column + args: slot_Centroid_x + y: + functor: Column + args: slot_Centroid_y + xErr: + functor: Column + args: slot_Centroid_xErr + yErr: + functor: Column + args: slot_Centroid_yErr + # x_y_Cov: not available + ra: + functor: RAColumn + dec: + functor: DecColumn + + # RFC-924: Temporarily keep a duplicate "decl" entry for backwards + # compatibility. To be removed after September 2023. + decl: + functor: DecColumn + raErr: + functor: RAErrColumn + decErr: + functor: DecErrColumn + ra_dec_Cov: + functor: RADecCovColumn + # One calibrated Calib flux is important: + calibFlux: + functor: Column + args: slot_CalibFlux_instFlux + calibFluxErr: + functor: Column + args: slot_CalibFlux_instFluxErr + ap12Flux: + functor: Column + args: base_CircularApertureFlux_12_0_instFlux + ap12FluxErr: + functor: Column + args: base_CircularApertureFlux_12_0_instFluxErr + ap12Flux_flag: + functor: Column + args: base_CircularApertureFlux_12_0_flag + ap17Flux: + functor: Column + args: base_CircularApertureFlux_17_0_instFlux + ap17FluxErr: + functor: Column + args: base_CircularApertureFlux_17_0_instFluxErr + ap17Flux_flag: + functor: Column + args: base_CircularApertureFlux_17_0_flag + psfFlux: + functor: Column + args: slot_PsfFlux_instFlux + psfFluxErr: + functor: Column + args: slot_PsfFlux_instFluxErr + ixx: + functor: Column + args: slot_Shape_xx + iyy: + functor: Column + args: slot_Shape_yy + ixy: + functor: Column + args: slot_Shape_xy + # DPDD should include Psf Shape + ixxPSF: + functor: Column + args: slot_PsfShape_xx + iyyPSF: + functor: Column + args: slot_PsfShape_yy + ixyPSF: + functor: Column + args: slot_PsfShape_xy + # apNann: Replaced by raw Aperture instFluxes in flags section below + # apMeanSb: Replaced by raw Aperture instFluxes in flags section below + # apMeanSbErr: Replaced by raw Aperture instFluxes in flags section below + + # DPDD does not include gaussianFluxes, however they are used for + # the single frame extendedness column which is used for QA. + gaussianFlux: + functor: Column + args: base_GaussianFlux_instFlux + gaussianFluxErr: + functor: Column + args: base_GaussianFlux_instFluxErr + extendedness: + functor: Column + args: base_ClassificationExtendedness_value + sizeExtendedness: + functor: Column + args: base_ClassificationSizeExtendedness_value +flags: + - base_ClassificationExtendedness_flag + - base_ClassificationSizeExtendedness_flag + - base_LocalBackground_instFlux # needed by isolatedStarAssociation + - base_LocalBackground_instFluxErr # needed by isolatedStarAssociation + - base_LocalBackground_flag # needed by isolatedStarAssociation + - base_NormalizedCompensatedTophatFlux_flag + - base_NormalizedCompensatedTophatFlux_instFlux + - base_NormalizedCompensatedTophatFlux_instFluxErr + - base_CircularApertureFlux_12_0_flag + - base_CircularApertureFlux_12_0_flag_apertureTruncated + - base_CircularApertureFlux_12_0_instFlux + - base_CircularApertureFlux_12_0_instFluxErr + - base_CircularApertureFlux_17_0_flag + - base_CircularApertureFlux_17_0_instFlux + - base_CircularApertureFlux_17_0_instFluxErr + - base_FootprintArea_value + - base_PixelFlags_flag_bad + - base_PixelFlags_flag_cr + - base_PixelFlags_flag_crCenter + - base_PixelFlags_flag_edge + - base_PixelFlags_flag_interpolated + - base_PixelFlags_flag_interpolatedCenter + - base_PixelFlags_flag_offimage + - base_PixelFlags_flag_saturated + - base_PixelFlags_flag_saturatedCenter + - base_PixelFlags_flag_suspect + - base_PixelFlags_flag_suspectCenter + - base_PsfFlux_apCorr + - base_PsfFlux_apCorrErr + - base_PsfFlux_area + - base_PsfFlux_flag + - base_PsfFlux_flag_apCorr + - base_PsfFlux_flag_edge + - base_PsfFlux_flag_noGoodPixels + - base_GaussianFlux_flag + - base_SdssCentroid_flag + - base_SdssCentroid_flag_almostNoSecondDerivative + - base_SdssCentroid_flag_badError + - base_SdssCentroid_flag_edge + - base_SdssCentroid_flag_noSecondDerivative + - base_SdssCentroid_flag_notAtMaximum + - base_SdssCentroid_flag_resetToPeak + - ext_shapeHSM_HsmPsfMoments_flag + - ext_shapeHSM_HsmPsfMoments_flag_no_pixels + - ext_shapeHSM_HsmPsfMoments_flag_not_contained + - ext_shapeHSM_HsmPsfMoments_flag_parent_source + - calib_astrometry_used + - calib_photometry_reserved + - calib_photometry_used + - calib_psf_candidate + - calib_psf_reserved + - calib_psf_used + - deblend_deblendedAsPsf + - deblend_hasStrayFlux + - deblend_masked + - deblend_nChild + - deblend_parentTooBig + - deblend_patchedTemplate + - deblend_rampedTemplate + - deblend_skipped + - deblend_tooManyPeaks + - sky_source + - detect_isPrimary + +flag_rename_rules: + # Taken from db-meas-forced + - ['base_Local', 'local'] + - ['base_PixelFlags_flag', 'pixelFlags'] + - ['base_SdssCentroid', 'centroid'] + - ['base_Psf', 'psf'] + - ['base_GaussianFlux', 'gaussianFlux'] + - ['base_CircularApertureFlux', 'apFlux'] + - ['base_NormalizedCompensatedTophatFlux', 'normCompTophatFlux'] + - ['ext_shapeHSM_Hsm', 'hsm'] + - ['base_', ''] + - ['slot_', ''] + diff --git a/tests/test_finalizeCharacterization.py b/tests/test_finalizeCharacterization.py index 174b4383d..ca0c0be6a 100644 --- a/tests/test_finalizeCharacterization.py +++ b/tests/test_finalizeCharacterization.py @@ -56,6 +56,7 @@ class FinalizeCharacterizationTestCase(lsst.utils.tests.TestCase): """ def setUp(self): config = FinalizeCharacterizationConfig() + config.remove_initial_photo_calib = False self.finalizeCharacterizationTask = TestFinalizeCharacterizationTask( config=config, @@ -232,7 +233,8 @@ def test_compute_psf_and_ap_corr_map_no_sources(self): detector, exposure, src, - isolated_source_table + isolated_source_table, + initial_photo_calib=None, ) self.assertIn( "No good sources remain after cuts for visit {}, detector {}".format(visit, detector),