Merge pull request #748 from lsst/tickets/DM-35207-v25

DM-35207-v25: backport UpdateVisitSummaryTask and usage of its finalVisitSummary output

doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.healSparseMapping.HealSparsePropertyMapTask.rst

@@ -10,7 +10,7 @@ One map is created per map type per tract per band.
 The resolution of the maps is set by the resolution configured for :doc:`lsst.pipe.tasks.healSparseMapping.HealSparseInputMapTask` and run during the coadd assembly.
 
 The maps to be run are configured through the "property map registry", and new maps can be defined via the ``lsst.pipe.tasks.healSparseMappingProperties.register_property_map`` decorator.
-Maps can do computations with any values that are available via the visit summaries produced by :doc:`lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask`.
+Maps can do computations with any values that are available via the visit summary datasets.
 
 Each map type can be configured to compute the minimum value at each position (``do_min``, dataset type ``{name}_map_min``); the maximum value (``do_max``, dataset type ``{name}_map_max``); the mean value (``do_mean``, dataset type ``{name}_map_mean``); the weighted mean, using the coadd weights (``do_weighted_mean``, dataset type ``{name}_map_weighted_mean``); and the sum (``do_sum``, dataset type ``{name}_map_sum``).
 In each case ``{name}`` refers to the registered name of the map.

doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask.rst

@@ -54,16 +54,5 @@ Output datasets
 
 ``visitSummary``
     Per-visit summary catalog of ccd/visit metadata.
-
-.. _lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask-subtasks:
-
-Examples
-========
-
-The following command shows an example of how to run the task on an example HSC repository.
-
-.. code-block:: bash
-
-    consolidateVisitSummary.py /datasets/hsc/repo --rerun <rerun name> --id visit=30504
-
-.. _lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask-debug:
+``visitSummary_schema``
+    Catalog with no rows with the same schema as ``visitSummary``.

python/lsst/pipe/tasks/coaddBase.py

@@ -98,7 +98,7 @@ class CoaddBaseConfig(pexConfig.Config):
     )
     useGlobalExternalSkyWcs = pexConfig.Field(
         dtype=bool,
-        default=False,
+        default=True,
         doc=("When using doApplyExternalSkyWcs, use 'global' calibrations "
              "that are not run per-tract.  When False, use per-tract wcs "
              "files.")

python/lsst/pipe/tasks/computeExposureSummaryStats.py

@@ -62,12 +62,12 @@ class ComputeExposureSummaryStatsConfig(pexConfig.Config):
     starShape = pexConfig.Field(
         doc="Base name of columns to use for the source shape in the PSF statistics computation.",
         dtype=str,
-        default="base_SdssShape"
+        default="slot_Shape"
     )
     psfShape = pexConfig.Field(
         doc="Base name of columns to use for the PSF shape in the PSF statistics computation.",
         dtype=str,
-        default="base_SdssShape_psf"
+        default="slot_PsfShape"
     )
     psfSampling = pexConfig.Field(
         dtype=int,
@@ -171,7 +171,7 @@ def run(self, exposure, sources, background):
 
         return summary
 
-    def update_psf_stats(self, summary, psf, bbox, sources=None, image_mask=None, sources_columns=None):
+    def update_psf_stats(self, summary, psf, bbox, sources=None, image_mask=None, sources_is_astropy=False):
         """Compute all summary-statistic fields that depend on the PSF model.
 
         Parameters
@@ -184,17 +184,18 @@ def update_psf_stats(self, summary, psf, bbox, sources=None, image_mask=None, so
         bbox : `lsst.geom.Box2I`
             Bounding box of the image for which summary stats are being
             computed.
-        sources : `lsst.afw.table.SourceCatalog`, optional
+        sources : `lsst.afw.table.SourceCatalog` or `astropy.table.Table`
             Catalog for quantities that are computed from source table columns.
             If `None`, these quantities will be reset (generally to NaN).
+            The type of this table must correspond to the
+            ``sources_is_astropy`` argument.
         image_mask : `lsst.afw.image.Mask`, optional
             Mask image that may be used to compute distance-to-nearest-star
             metrics.
-        sources_columns : `collections.abc.Set` [ `str` ], optional
-            Set of all column names in ``sources``.  If provided, ``sources``
-            may be any table type for which string indexes yield column arrays.
-            If not provided, ``sources`` is assumed to be an
-            `lsst.afw.table.SourceCatalog`.
+        sources_is_astropy : `bool`, optional
+            Whether ``sources`` is an `astropy.table.Table` instance instead
+            of an `lsst.afw.table.Catalog` instance.  Default is `False` (the
+            latter).
         """
         nan = float("nan")
         summary.psfSigma = nan
@@ -240,24 +241,18 @@ def update_psf_stats(self, summary, psf, bbox, sources=None, image_mask=None, so
             # No sources are available (as in some tests)
             return
 
-        if sources_columns is None:
-            sources_columns = sources.schema.getNames()
-        if (
-            self.config.starSelection not in sources_columns
-            or self.config.starShape + '_flag' not in sources_columns
-        ):
-            # The source catalog does not have the necessary fields (as in some
-            # tests).
-            return
-
         psf_mask = sources[self.config.starSelection] & (~sources[self.config.starShape + '_flag'])
         nPsfStar = psf_mask.sum()
 
         if nPsfStar == 0:
             # No stars to measure statistics, so we must return the defaults
             # of 0 stars and NaN values.
             return
-        psf_cat = sources[psf_mask].copy(deep=True)
+
+        if sources_is_astropy:
+            psf_cat = sources[psf_mask]
+        else:
+            psf_cat = sources[psf_mask].copy(deep=True)
 
         starXX = psf_cat[self.config.starShape + '_xx']
         starYY = psf_cat[self.config.starShape + '_yy']
@@ -449,8 +444,8 @@ def maximum_nearest_psf_distance(
     Parameters
     ----------
     image_mask : `lsst.afw.image.Mask`
-        The mask plane assosiated with the exposure.
-    psf_cat : `lsst.afw.table.SourceCatalog`
+        The mask plane associated with the exposure.
+    psf_cat : `lsst.afw.table.SourceCatalog` or `astropy.table.Table`
         Catalog containing only the stars used in the PSF modeling.
     sampling : `int`
         Sampling rate in each dimension to create the grid of points on which
@@ -476,8 +471,8 @@ def maximum_nearest_psf_distance(
 
     dist_to_nearest_psf = np.full(good.shape, np.inf)
     for psf in psf_cat:
-        x_psf = psf.getX()
-        y_psf = psf.getY()
+        x_psf = psf["slot_Centroid_x"]
+        y_psf = psf["slot_Centroid_y"]
         dist_to_nearest_psf = np.minimum(dist_to_nearest_psf, np.hypot(xx - x_psf, yy - y_psf))
         unmasked_dists = dist_to_nearest_psf * good
         max_dist_to_nearest_psf = np.max(unmasked_dists)
@@ -498,9 +493,9 @@ def psf_trace_radius_delta(
     Parameters
     ----------
     image_mask : `lsst.afw.image.Mask`
-        The mask plane assosiated with the exposure.
+        The mask plane associated with the exposure.
     image_psf : `lsst.afw.detection.Psf`
-        The PSF model assosiated with the exposure.
+        The PSF model associated with the exposure.
     sampling : `int`
         Sampling rate in each dimension to create the grid of points at which
         to evaluate ``image_psf``s trace radius value. The tradeoff is between

python/lsst/pipe/tasks/healSparseMapping.py

@@ -342,7 +342,7 @@ class HealSparsePropertyMapConnections(pipeBase.PipelineTaskConnections,
     )
     visit_summaries = pipeBase.connectionTypes.Input(
         doc="Visit summary tables with aggregated statistics",
-        name="{calexpType}visitSummary",
+        name="finalVisitSummary",
         storageClass="ExposureCatalog",
         dimensions=("instrument", "visit"),
         multiple=True,

python/lsst/pipe/tasks/imageDifference.py

@@ -104,7 +104,7 @@ class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections,
         doc=("Per-visit finalized psf models and aperture correction maps. "
              "These catalogs use the detector id for the catalog id, "
              "sorted on id for fast lookup."),
-        name="finalized_psf_ap_corr_catalog",
+        name="finalVisitSummary",
         storageClass="ExposureCatalog",
         dimensions=("instrument", "visit"),
     )

python/lsst/pipe/tasks/makeWarp.py

@@ -88,7 +88,7 @@ class MakeWarpConnections(pipeBase.PipelineTaskConnections,
         doc=("Per-visit wcs calibrations computed globally (with no tract information). "
              "These catalogs use the detector id for the catalog id, sorted on id for "
              "fast lookup."),
-        name="{skyWcsName}SkyWcsCatalog",
+        name="finalVisitSummary",
         storageClass="ExposureCatalog",
         dimensions=("instrument", "visit"),
     )
@@ -103,15 +103,15 @@ class MakeWarpConnections(pipeBase.PipelineTaskConnections,
         doc=("Per-visit photometric calibrations computed globally (with no tract "
              "information).  These catalogs use the detector id for the catalog id, "
              "sorted on id for fast lookup."),
-        name="{photoCalibName}PhotoCalibCatalog",
+        name="finalVisitSummary",
         storageClass="ExposureCatalog",
         dimensions=("instrument", "visit"),
     )
     finalizedPsfApCorrCatalog = connectionTypes.Input(
         doc=("Per-visit finalized psf models and aperture correction maps. "
              "These catalogs use the detector id for the catalog id, "
              "sorted on id for fast lookup."),
-        name="finalized_psf_ap_corr_catalog",
+        name="finalVisitSummary",
         storageClass="ExposureCatalog",
         dimensions=("instrument", "visit"),
     )
@@ -146,8 +146,8 @@ class MakeWarpConnections(pipeBase.PipelineTaskConnections,
         multiple=True,
     )
     visitSummary = connectionTypes.Input(
-        doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask",
-        name="{calexpType}visitSummary",
+        doc="Consolidated exposure metadata",
+        name="finalVisitSummary",
         storageClass="ExposureCatalog",
         dimensions=("instrument", "visit",),
     )

python/lsst/pipe/tasks/postprocess.py

@@ -285,7 +285,7 @@ class WriteRecalibratedSourceTableConnections(WriteSourceTableConnections,
         doc=("Per-visit wcs calibrations computed globally (with no tract information). "
              "These catalogs use the detector id for the catalog id, sorted on id for "
              "fast lookup."),
-        name="{skyWcsName}SkyWcsCatalog",
+        name="finalVisitSummary",
         storageClass="ExposureCatalog",
         dimensions=["instrument", "visit"],
     )
@@ -301,7 +301,7 @@ class WriteRecalibratedSourceTableConnections(WriteSourceTableConnections,
         doc=("Per-visit photometric calibrations computed globally (with no tract "
              "information).  These catalogs use the detector id for the catalog id, "
              "sorted on id for fast lookup."),
-        name="{photoCalibName}PhotoCalibCatalog",
+        name="finalVisitSummary",
         storageClass="ExposureCatalog",
         dimensions=["instrument", "visit"],
     )
@@ -362,7 +362,7 @@ class WriteRecalibratedSourceTableConfig(WriteSourceTableConfig,
     )
     useGlobalExternalSkyWcs = pexConfig.Field(
         dtype=bool,
-        default=False,
+        default=True,
         doc=("When using doApplyExternalSkyWcs, use 'global' calibrations "
              "that are not run per-tract.  When False, use per-tract wcs "
              "files.")
@@ -1212,7 +1212,7 @@ class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections,
                                          defaultTemplates={"calexpType": ""}):
     calexp = connectionTypes.Input(
         doc="Processed exposures used for metadata",
-        name="{calexpType}calexp",
+        name="calexp",
         storageClass="ExposureF",
         dimensions=("instrument", "visit", "detector"),
         deferLoad=True,
@@ -1222,10 +1222,15 @@ class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections,
         doc=("Per-visit consolidated exposure metadata.  These catalogs use "
              "detector id for the id and are sorted for fast lookups of a "
              "detector."),
-        name="{calexpType}visitSummary",
+        name="visitSummary",
         storageClass="ExposureCatalog",
         dimensions=("instrument", "visit"),
     )
+    visitSummarySchema = connectionTypes.InitOutput(
+        doc="Schema of the visitSummary catalog",
+        name="visitSummary_schema",
+        storageClass="ExposureCatalog",
+    )
 
 
 class ConsolidateVisitSummaryConfig(pipeBase.PipelineTaskConfig,
@@ -1255,6 +1260,15 @@ class ConsolidateVisitSummaryTask(pipeBase.PipelineTask):
     _DefaultName = "consolidateVisitSummary"
     ConfigClass = ConsolidateVisitSummaryConfig
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.schema = afwTable.ExposureTable.makeMinimalSchema()
+        self.schema.addField('visit', type='L', doc='Visit number')
+        self.schema.addField('physical_filter', type='String', size=32, doc='Physical filter')
+        self.schema.addField('band', type='String', size=32, doc='Name of band')
+        ExposureSummaryStats.update_schema(self.schema)
+        self.visitSummarySchema = afwTable.ExposureCatalog(self.schema)
+
     def runQuantum(self, butlerQC, inputRefs, outputRefs):
         dataRefs = butlerQC.get(inputRefs.calexp)
         visit = dataRefs[0].dataId.byName()['visit']
@@ -1283,8 +1297,7 @@ def _combineExposureMetadata(self, visit, dataRefs):
         visitSummary : `lsst.afw.table.ExposureCatalog`
             Exposure catalog with per-detector summary information.
         """
-        schema = self._makeVisitSummarySchema()
-        cat = afwTable.ExposureCatalog(schema)
+        cat = afwTable.ExposureCatalog(self.schema)
         cat.resize(len(dataRefs))
 
         cat['visit'] = visit
@@ -1321,15 +1334,6 @@ def _combineExposureMetadata(self, visit, dataRefs):
         cat.sort()
         return cat
 
-    def _makeVisitSummarySchema(self):
-        """Make the schema for the visitSummary catalog."""
-        schema = afwTable.ExposureTable.makeMinimalSchema()
-        schema.addField('visit', type='L', doc='Visit number')
-        schema.addField('physical_filter', type='String', size=32, doc='Physical filter')
-        schema.addField('band', type='String', size=32, doc='Name of band')
-        ExposureSummaryStats.update_schema(schema)
-        return schema
-
 
 class ConsolidateSourceTableConnections(pipeBase.PipelineTaskConnections,
                                         defaultTemplates={"catalogType": ""},
@@ -1380,16 +1384,16 @@ class MakeCcdVisitTableConnections(pipeBase.PipelineTaskConnections,
                                    dimensions=("instrument",),
                                    defaultTemplates={"calexpType": ""}):
     visitSummaryRefs = connectionTypes.Input(
-        doc="Data references for per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
-        name="{calexpType}visitSummary",
+        doc="Data references for per-visit consolidated exposure metadata",
+        name="finalVisitSummary",
         storageClass="ExposureCatalog",
         dimensions=("instrument", "visit"),
         multiple=True,
         deferLoad=True,
     )
     outputCatalog = connectionTypes.Output(
         doc="CCD and Visit metadata table",
-        name="{calexpType}ccdVisitTable",
+        name="ccdVisitTable",
         storageClass="DataFrame",
         dimensions=("instrument",)
     )
@@ -1401,13 +1405,13 @@ class MakeCcdVisitTableConfig(pipeBase.PipelineTaskConfig,
 
 
 class MakeCcdVisitTableTask(pipeBase.PipelineTask):
-    """Produce a `ccdVisitTable` from the `visitSummary` exposure catalogs.
+    """Produce a `ccdVisitTable` from the visit summary exposure catalogs.
     """
     _DefaultName = 'makeCcdVisitTable'
     ConfigClass = MakeCcdVisitTableConfig
 
     def run(self, visitSummaryRefs):
-        """Make a table of ccd information from the `visitSummary` catalogs.
+        """Make a table of ccd information from the visit summary catalogs.
 
         Parameters
         ----------
@@ -1487,16 +1491,16 @@ class MakeVisitTableConnections(pipeBase.PipelineTaskConnections,
                                 dimensions=("instrument",),
                                 defaultTemplates={"calexpType": ""}):
     visitSummaries = connectionTypes.Input(
-        doc="Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
-        name="{calexpType}visitSummary",
+        doc="Per-visit consolidated exposure metadata",
+        name="finalVisitSummary",
         storageClass="ExposureCatalog",
         dimensions=("instrument", "visit",),
         multiple=True,
         deferLoad=True,
     )
     outputCatalog = connectionTypes.Output(
         doc="Visit metadata table",
-        name="{calexpType}visitTable",
+        name="visitTable",
         storageClass="DataFrame",
         dimensions=("instrument",)
     )
@@ -1508,13 +1512,13 @@ class MakeVisitTableConfig(pipeBase.PipelineTaskConfig,
 
 
 class MakeVisitTableTask(pipeBase.PipelineTask):
-    """Produce a `visitTable` from the `visitSummary` exposure catalogs.
+    """Produce a `visitTable` from the visit summary exposure catalogs.
     """
     _DefaultName = 'makeVisitTable'
     ConfigClass = MakeVisitTableConfig
 
     def run(self, visitSummaries):
-        """Make a table of visit information from the `visitSummary` catalogs.
+        """Make a table of visit information from the visit summary catalogs.
 
         Parameters
         ----------

python/lsst/pipe/tasks/processCcdWithFakes.py

@@ -88,7 +88,7 @@ class ProcessCcdWithFakesConnections(PipelineTaskConnections,
         doc=("Per-visit wcs calibrations computed globally (with no tract information). "
              "These catalogs use the detector id for the catalog id, sorted on id for "
              "fast lookup."),
-        name="{wcsName}SkyWcsCatalog",
+        name="finalVisitSummary",
         storageClass="ExposureCatalog",
         dimensions=("instrument", "visit"),
     )
@@ -106,7 +106,7 @@ class ProcessCcdWithFakesConnections(PipelineTaskConnections,
     externalPhotoCalibGlobalCatalog = cT.Input(
         doc=("Per-visit photometric calibrations. These catalogs use the "
              "detector id for the catalog id, sorted on id for fast lookup."),
-        name="{photoCalibName}PhotoCalibCatalog",
+        name="finalVisitSummary",
         storageClass="ExposureCatalog",
         dimensions=("instrument", "visit"),
     )
@@ -254,6 +254,7 @@ def setDefaults(self):
         self.calibrate.doAstrometry = False
         self.calibrate.doWriteMatches = False
         self.calibrate.doPhotoCal = False
+        self.calibrate.doComputeSummaryStats = False
         self.calibrate.detection.reEstimateBackground = False