diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.healSparseMapping.HealSparsePropertyMapTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.healSparseMapping.HealSparsePropertyMapTask.rst index f545069b4..85afd6e44 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.healSparseMapping.HealSparsePropertyMapTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.healSparseMapping.HealSparsePropertyMapTask.rst @@ -10,7 +10,7 @@ One map is created per map type per tract per band. The resolution of the maps is set by the resolution configured for :doc:`lsst.pipe.tasks.healSparseMapping.HealSparseInputMapTask` and run during the coadd assembly. The maps to be run are configured through the "property map registry", and new maps can be defined via the ``lsst.pipe.tasks.healSparseMappingProperties.register_property_map`` decorator. -Maps can do computations with any values that are available via the visit summaries produced by :doc:`lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask`. +Maps can do computations with any values that are available via the visit summary datasets. Each map type can be configured to compute the minimum value at each position (``do_min``, dataset type ``{name}_map_min``); the maximum value (``do_max``, dataset type ``{name}_map_max``); the mean value (``do_mean``, dataset type ``{name}_map_mean``); the weighted mean, using the coadd weights (``do_weighted_mean``, dataset type ``{name}_map_weighted_mean``); and the sum (``do_sum``, dataset type ``{name}_map_sum``). In each case ``{name}`` refers to the registered name of the map. diff --git a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask.rst b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask.rst index a06ea5145..23f3a8dbc 100644 --- a/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask.rst +++ b/doc/lsst.pipe.tasks/tasks/lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask.rst @@ -54,16 +54,5 @@ Output datasets ``visitSummary`` Per-visit summary catalog of ccd/visit metadata. - -.. _lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask-subtasks: - -Examples -======== - -The following command shows an example of how to run the task on an example HSC repository. - -.. code-block:: bash - - consolidateVisitSummary.py /datasets/hsc/repo --rerun --id visit=30504 - -.. _lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask-debug: +``visitSummary_schema`` + Catalog with no rows with the same schema as ``visitSummary``. diff --git a/python/lsst/pipe/tasks/coaddBase.py b/python/lsst/pipe/tasks/coaddBase.py index 8147515cc..cb2810df4 100644 --- a/python/lsst/pipe/tasks/coaddBase.py +++ b/python/lsst/pipe/tasks/coaddBase.py @@ -98,7 +98,7 @@ class CoaddBaseConfig(pexConfig.Config): ) useGlobalExternalSkyWcs = pexConfig.Field( dtype=bool, - default=False, + default=True, doc=("When using doApplyExternalSkyWcs, use 'global' calibrations " "that are not run per-tract. When False, use per-tract wcs " "files.") diff --git a/python/lsst/pipe/tasks/computeExposureSummaryStats.py b/python/lsst/pipe/tasks/computeExposureSummaryStats.py index 604c8c23c..5072f3c8b 100644 --- a/python/lsst/pipe/tasks/computeExposureSummaryStats.py +++ b/python/lsst/pipe/tasks/computeExposureSummaryStats.py @@ -62,12 +62,12 @@ class ComputeExposureSummaryStatsConfig(pexConfig.Config): starShape = pexConfig.Field( doc="Base name of columns to use for the source shape in the PSF statistics computation.", dtype=str, - default="base_SdssShape" + default="slot_Shape" ) psfShape = pexConfig.Field( doc="Base name of columns to use for the PSF shape in the PSF statistics computation.", dtype=str, - default="base_SdssShape_psf" + default="slot_PsfShape" ) psfSampling = pexConfig.Field( dtype=int, @@ -171,7 +171,7 @@ def run(self, exposure, sources, background): return summary - def update_psf_stats(self, summary, psf, bbox, sources=None, image_mask=None, sources_columns=None): + def update_psf_stats(self, summary, psf, bbox, sources=None, image_mask=None, sources_is_astropy=False): """Compute all summary-statistic fields that depend on the PSF model. Parameters @@ -184,17 +184,18 @@ def update_psf_stats(self, summary, psf, bbox, sources=None, image_mask=None, so bbox : `lsst.geom.Box2I` Bounding box of the image for which summary stats are being computed. - sources : `lsst.afw.table.SourceCatalog`, optional + sources : `lsst.afw.table.SourceCatalog` or `astropy.table.Table` Catalog for quantities that are computed from source table columns. If `None`, these quantities will be reset (generally to NaN). + The type of this table must correspond to the + ``sources_is_astropy`` argument. image_mask : `lsst.afw.image.Mask`, optional Mask image that may be used to compute distance-to-nearest-star metrics. - sources_columns : `collections.abc.Set` [ `str` ], optional - Set of all column names in ``sources``. If provided, ``sources`` - may be any table type for which string indexes yield column arrays. - If not provided, ``sources`` is assumed to be an - `lsst.afw.table.SourceCatalog`. + sources_is_astropy : `bool`, optional + Whether ``sources`` is an `astropy.table.Table` instance instead + of an `lsst.afw.table.Catalog` instance. Default is `False` (the + latter). """ nan = float("nan") summary.psfSigma = nan @@ -240,16 +241,6 @@ def update_psf_stats(self, summary, psf, bbox, sources=None, image_mask=None, so # No sources are available (as in some tests) return - if sources_columns is None: - sources_columns = sources.schema.getNames() - if ( - self.config.starSelection not in sources_columns - or self.config.starShape + '_flag' not in sources_columns - ): - # The source catalog does not have the necessary fields (as in some - # tests). - return - psf_mask = sources[self.config.starSelection] & (~sources[self.config.starShape + '_flag']) nPsfStar = psf_mask.sum() @@ -257,7 +248,11 @@ def update_psf_stats(self, summary, psf, bbox, sources=None, image_mask=None, so # No stars to measure statistics, so we must return the defaults # of 0 stars and NaN values. return - psf_cat = sources[psf_mask].copy(deep=True) + + if sources_is_astropy: + psf_cat = sources[psf_mask] + else: + psf_cat = sources[psf_mask].copy(deep=True) starXX = psf_cat[self.config.starShape + '_xx'] starYY = psf_cat[self.config.starShape + '_yy'] @@ -449,8 +444,8 @@ def maximum_nearest_psf_distance( Parameters ---------- image_mask : `lsst.afw.image.Mask` - The mask plane assosiated with the exposure. - psf_cat : `lsst.afw.table.SourceCatalog` + The mask plane associated with the exposure. + psf_cat : `lsst.afw.table.SourceCatalog` or `astropy.table.Table` Catalog containing only the stars used in the PSF modeling. sampling : `int` Sampling rate in each dimension to create the grid of points on which @@ -476,8 +471,8 @@ def maximum_nearest_psf_distance( dist_to_nearest_psf = np.full(good.shape, np.inf) for psf in psf_cat: - x_psf = psf.getX() - y_psf = psf.getY() + x_psf = psf["slot_Centroid_x"] + y_psf = psf["slot_Centroid_y"] dist_to_nearest_psf = np.minimum(dist_to_nearest_psf, np.hypot(xx - x_psf, yy - y_psf)) unmasked_dists = dist_to_nearest_psf * good max_dist_to_nearest_psf = np.max(unmasked_dists) @@ -498,9 +493,9 @@ def psf_trace_radius_delta( Parameters ---------- image_mask : `lsst.afw.image.Mask` - The mask plane assosiated with the exposure. + The mask plane associated with the exposure. image_psf : `lsst.afw.detection.Psf` - The PSF model assosiated with the exposure. + The PSF model associated with the exposure. sampling : `int` Sampling rate in each dimension to create the grid of points at which to evaluate ``image_psf``s trace radius value. The tradeoff is between diff --git a/python/lsst/pipe/tasks/healSparseMapping.py b/python/lsst/pipe/tasks/healSparseMapping.py index e9df380a9..6facc4a11 100644 --- a/python/lsst/pipe/tasks/healSparseMapping.py +++ b/python/lsst/pipe/tasks/healSparseMapping.py @@ -342,7 +342,7 @@ class HealSparsePropertyMapConnections(pipeBase.PipelineTaskConnections, ) visit_summaries = pipeBase.connectionTypes.Input( doc="Visit summary tables with aggregated statistics", - name="{calexpType}visitSummary", + name="finalVisitSummary", storageClass="ExposureCatalog", dimensions=("instrument", "visit"), multiple=True, diff --git a/python/lsst/pipe/tasks/imageDifference.py b/python/lsst/pipe/tasks/imageDifference.py index be4ab5a5e..690b2217a 100644 --- a/python/lsst/pipe/tasks/imageDifference.py +++ b/python/lsst/pipe/tasks/imageDifference.py @@ -104,7 +104,7 @@ class ImageDifferenceTaskConnections(pipeBase.PipelineTaskConnections, doc=("Per-visit finalized psf models and aperture correction maps. " "These catalogs use the detector id for the catalog id, " "sorted on id for fast lookup."), - name="finalized_psf_ap_corr_catalog", + name="finalVisitSummary", storageClass="ExposureCatalog", dimensions=("instrument", "visit"), ) diff --git a/python/lsst/pipe/tasks/makeWarp.py b/python/lsst/pipe/tasks/makeWarp.py index 8cd8e8bac..942ac35fa 100644 --- a/python/lsst/pipe/tasks/makeWarp.py +++ b/python/lsst/pipe/tasks/makeWarp.py @@ -88,7 +88,7 @@ class MakeWarpConnections(pipeBase.PipelineTaskConnections, doc=("Per-visit wcs calibrations computed globally (with no tract information). " "These catalogs use the detector id for the catalog id, sorted on id for " "fast lookup."), - name="{skyWcsName}SkyWcsCatalog", + name="finalVisitSummary", storageClass="ExposureCatalog", dimensions=("instrument", "visit"), ) @@ -103,7 +103,7 @@ class MakeWarpConnections(pipeBase.PipelineTaskConnections, doc=("Per-visit photometric calibrations computed globally (with no tract " "information). These catalogs use the detector id for the catalog id, " "sorted on id for fast lookup."), - name="{photoCalibName}PhotoCalibCatalog", + name="finalVisitSummary", storageClass="ExposureCatalog", dimensions=("instrument", "visit"), ) @@ -111,7 +111,7 @@ class MakeWarpConnections(pipeBase.PipelineTaskConnections, doc=("Per-visit finalized psf models and aperture correction maps. " "These catalogs use the detector id for the catalog id, " "sorted on id for fast lookup."), - name="finalized_psf_ap_corr_catalog", + name="finalVisitSummary", storageClass="ExposureCatalog", dimensions=("instrument", "visit"), ) @@ -146,8 +146,8 @@ class MakeWarpConnections(pipeBase.PipelineTaskConnections, multiple=True, ) visitSummary = connectionTypes.Input( - doc="Consolidated exposure metadata from ConsolidateVisitSummaryTask", - name="{calexpType}visitSummary", + doc="Consolidated exposure metadata", + name="finalVisitSummary", storageClass="ExposureCatalog", dimensions=("instrument", "visit",), ) diff --git a/python/lsst/pipe/tasks/postprocess.py b/python/lsst/pipe/tasks/postprocess.py index 9d8b45ed8..a81a88275 100644 --- a/python/lsst/pipe/tasks/postprocess.py +++ b/python/lsst/pipe/tasks/postprocess.py @@ -285,7 +285,7 @@ class WriteRecalibratedSourceTableConnections(WriteSourceTableConnections, doc=("Per-visit wcs calibrations computed globally (with no tract information). " "These catalogs use the detector id for the catalog id, sorted on id for " "fast lookup."), - name="{skyWcsName}SkyWcsCatalog", + name="finalVisitSummary", storageClass="ExposureCatalog", dimensions=["instrument", "visit"], ) @@ -301,7 +301,7 @@ class WriteRecalibratedSourceTableConnections(WriteSourceTableConnections, doc=("Per-visit photometric calibrations computed globally (with no tract " "information). These catalogs use the detector id for the catalog id, " "sorted on id for fast lookup."), - name="{photoCalibName}PhotoCalibCatalog", + name="finalVisitSummary", storageClass="ExposureCatalog", dimensions=["instrument", "visit"], ) @@ -362,7 +362,7 @@ class WriteRecalibratedSourceTableConfig(WriteSourceTableConfig, ) useGlobalExternalSkyWcs = pexConfig.Field( dtype=bool, - default=False, + default=True, doc=("When using doApplyExternalSkyWcs, use 'global' calibrations " "that are not run per-tract. When False, use per-tract wcs " "files.") @@ -1212,7 +1212,7 @@ class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections, defaultTemplates={"calexpType": ""}): calexp = connectionTypes.Input( doc="Processed exposures used for metadata", - name="{calexpType}calexp", + name="calexp", storageClass="ExposureF", dimensions=("instrument", "visit", "detector"), deferLoad=True, @@ -1222,10 +1222,15 @@ class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections, doc=("Per-visit consolidated exposure metadata. These catalogs use " "detector id for the id and are sorted for fast lookups of a " "detector."), - name="{calexpType}visitSummary", + name="visitSummary", storageClass="ExposureCatalog", dimensions=("instrument", "visit"), ) + visitSummarySchema = connectionTypes.InitOutput( + doc="Schema of the visitSummary catalog", + name="visitSummary_schema", + storageClass="ExposureCatalog", + ) class ConsolidateVisitSummaryConfig(pipeBase.PipelineTaskConfig, @@ -1255,6 +1260,15 @@ class ConsolidateVisitSummaryTask(pipeBase.PipelineTask): _DefaultName = "consolidateVisitSummary" ConfigClass = ConsolidateVisitSummaryConfig + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.schema = afwTable.ExposureTable.makeMinimalSchema() + self.schema.addField('visit', type='L', doc='Visit number') + self.schema.addField('physical_filter', type='String', size=32, doc='Physical filter') + self.schema.addField('band', type='String', size=32, doc='Name of band') + ExposureSummaryStats.update_schema(self.schema) + self.visitSummarySchema = afwTable.ExposureCatalog(self.schema) + def runQuantum(self, butlerQC, inputRefs, outputRefs): dataRefs = butlerQC.get(inputRefs.calexp) visit = dataRefs[0].dataId.byName()['visit'] @@ -1283,8 +1297,7 @@ def _combineExposureMetadata(self, visit, dataRefs): visitSummary : `lsst.afw.table.ExposureCatalog` Exposure catalog with per-detector summary information. """ - schema = self._makeVisitSummarySchema() - cat = afwTable.ExposureCatalog(schema) + cat = afwTable.ExposureCatalog(self.schema) cat.resize(len(dataRefs)) cat['visit'] = visit @@ -1321,15 +1334,6 @@ def _combineExposureMetadata(self, visit, dataRefs): cat.sort() return cat - def _makeVisitSummarySchema(self): - """Make the schema for the visitSummary catalog.""" - schema = afwTable.ExposureTable.makeMinimalSchema() - schema.addField('visit', type='L', doc='Visit number') - schema.addField('physical_filter', type='String', size=32, doc='Physical filter') - schema.addField('band', type='String', size=32, doc='Name of band') - ExposureSummaryStats.update_schema(schema) - return schema - class ConsolidateSourceTableConnections(pipeBase.PipelineTaskConnections, defaultTemplates={"catalogType": ""}, @@ -1380,8 +1384,8 @@ class MakeCcdVisitTableConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument",), defaultTemplates={"calexpType": ""}): visitSummaryRefs = connectionTypes.Input( - doc="Data references for per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask", - name="{calexpType}visitSummary", + doc="Data references for per-visit consolidated exposure metadata", + name="finalVisitSummary", storageClass="ExposureCatalog", dimensions=("instrument", "visit"), multiple=True, @@ -1389,7 +1393,7 @@ class MakeCcdVisitTableConnections(pipeBase.PipelineTaskConnections, ) outputCatalog = connectionTypes.Output( doc="CCD and Visit metadata table", - name="{calexpType}ccdVisitTable", + name="ccdVisitTable", storageClass="DataFrame", dimensions=("instrument",) ) @@ -1401,13 +1405,13 @@ class MakeCcdVisitTableConfig(pipeBase.PipelineTaskConfig, class MakeCcdVisitTableTask(pipeBase.PipelineTask): - """Produce a `ccdVisitTable` from the `visitSummary` exposure catalogs. + """Produce a `ccdVisitTable` from the visit summary exposure catalogs. """ _DefaultName = 'makeCcdVisitTable' ConfigClass = MakeCcdVisitTableConfig def run(self, visitSummaryRefs): - """Make a table of ccd information from the `visitSummary` catalogs. + """Make a table of ccd information from the visit summary catalogs. Parameters ---------- @@ -1487,8 +1491,8 @@ class MakeVisitTableConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument",), defaultTemplates={"calexpType": ""}): visitSummaries = connectionTypes.Input( - doc="Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask", - name="{calexpType}visitSummary", + doc="Per-visit consolidated exposure metadata", + name="finalVisitSummary", storageClass="ExposureCatalog", dimensions=("instrument", "visit",), multiple=True, @@ -1496,7 +1500,7 @@ class MakeVisitTableConnections(pipeBase.PipelineTaskConnections, ) outputCatalog = connectionTypes.Output( doc="Visit metadata table", - name="{calexpType}visitTable", + name="visitTable", storageClass="DataFrame", dimensions=("instrument",) ) @@ -1508,13 +1512,13 @@ class MakeVisitTableConfig(pipeBase.PipelineTaskConfig, class MakeVisitTableTask(pipeBase.PipelineTask): - """Produce a `visitTable` from the `visitSummary` exposure catalogs. + """Produce a `visitTable` from the visit summary exposure catalogs. """ _DefaultName = 'makeVisitTable' ConfigClass = MakeVisitTableConfig def run(self, visitSummaries): - """Make a table of visit information from the `visitSummary` catalogs. + """Make a table of visit information from the visit summary catalogs. Parameters ---------- diff --git a/python/lsst/pipe/tasks/processCcdWithFakes.py b/python/lsst/pipe/tasks/processCcdWithFakes.py index b38be56d4..5be22eb56 100644 --- a/python/lsst/pipe/tasks/processCcdWithFakes.py +++ b/python/lsst/pipe/tasks/processCcdWithFakes.py @@ -88,7 +88,7 @@ class ProcessCcdWithFakesConnections(PipelineTaskConnections, doc=("Per-visit wcs calibrations computed globally (with no tract information). " "These catalogs use the detector id for the catalog id, sorted on id for " "fast lookup."), - name="{wcsName}SkyWcsCatalog", + name="finalVisitSummary", storageClass="ExposureCatalog", dimensions=("instrument", "visit"), ) @@ -106,7 +106,7 @@ class ProcessCcdWithFakesConnections(PipelineTaskConnections, externalPhotoCalibGlobalCatalog = cT.Input( doc=("Per-visit photometric calibrations. These catalogs use the " "detector id for the catalog id, sorted on id for fast lookup."), - name="{photoCalibName}PhotoCalibCatalog", + name="finalVisitSummary", storageClass="ExposureCatalog", dimensions=("instrument", "visit"), ) @@ -254,6 +254,7 @@ def setDefaults(self): self.calibrate.doAstrometry = False self.calibrate.doWriteMatches = False self.calibrate.doPhotoCal = False + self.calibrate.doComputeSummaryStats = False self.calibrate.detection.reEstimateBackground = False diff --git a/python/lsst/pipe/tasks/selectImages.py b/python/lsst/pipe/tasks/selectImages.py index d9af41066..c2c0a6186 100644 --- a/python/lsst/pipe/tasks/selectImages.py +++ b/python/lsst/pipe/tasks/selectImages.py @@ -352,24 +352,21 @@ def isValid(self, visitSummary, detectorId): psfTraceRadiusDelta = row["psfTraceRadiusDelta"] valid = True - if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual: + if self.config.maxEllipResidual and not (medianE <= self.config.maxEllipResidual): self.log.info("Removing visit %d detector %d because median e residual too large: %f vs %f", row["visit"], detectorId, medianE, self.config.maxEllipResidual) valid = False - elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter: + elif self.config.maxSizeScatter and not (scatterSize <= self.config.maxSizeScatter): self.log.info("Removing visit %d detector %d because size scatter too large: %f vs %f", row["visit"], detectorId, scatterSize, self.config.maxSizeScatter) valid = False - elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter: + elif self.config.maxScaledSizeScatter and not (scaledScatterSize <= self.config.maxScaledSizeScatter): self.log.info("Removing visit %d detector %d because scaled size scatter too large: %f vs %f", row["visit"], detectorId, scaledScatterSize, self.config.maxScaledSizeScatter) valid = False elif ( - self.config.maxPsfTraceRadiusDelta - and ( - psfTraceRadiusDelta > self.config.maxPsfTraceRadiusDelta - or ~np.isfinite(psfTraceRadiusDelta) - ) + self.config.maxPsfTraceRadiusDelta is not None + and not (psfTraceRadiusDelta <= self.config.maxPsfTraceRadiusDelta) ): self.log.info( "Removing visit %d detector %d because max-min delta of model PSF trace radius values " @@ -391,8 +388,8 @@ class BestSeeingSelectVisitsConnections(pipeBase.PipelineTaskConnections, dimensions=("skymap",), ) visitSummaries = pipeBase.connectionTypes.Input( - doc="Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask", - name="visitSummary", + doc="Per-visit consolidated exposure metadata", + name="finalVisitSummary", storageClass="ExposureCatalog", dimensions=("instrument", "visit",), multiple=True, diff --git a/tests/surveyPropertyMapsTestUtils.py b/tests/surveyPropertyMapsTestUtils.py index cd6bc21d5..3e95c598a 100644 --- a/tests/surveyPropertyMapsTestUtils.py +++ b/tests/surveyPropertyMapsTestUtils.py @@ -99,7 +99,7 @@ def makeMockVisitSummary(visit, # We are making a 2 detector "camera" n_detector = 2 - schema = ConsolidateVisitSummaryTask()._makeVisitSummarySchema() + schema = ConsolidateVisitSummaryTask().schema visit_summary = afwTable.ExposureCatalog(schema) visit_summary.resize(n_detector) diff --git a/tests/test_isPrimaryFlag.py b/tests/test_isPrimaryFlag.py index de74a6605..0e5ec2bb0 100755 --- a/tests/test_isPrimaryFlag.py +++ b/tests/test_isPrimaryFlag.py @@ -178,6 +178,7 @@ def testIsSinglePrimaryFlag(self): calibConfig = CalibrateConfig() calibConfig.doAstrometry = False calibConfig.doPhotoCal = False + calibConfig.doComputeSummaryStats = False calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(self.charImResults.exposure) outputCat = calibResults.outputCat diff --git a/tests/test_skySources.py b/tests/test_skySources.py index b7656a395..88e49fa6b 100755 --- a/tests/test_skySources.py +++ b/tests/test_skySources.py @@ -65,6 +65,7 @@ def _checkSkySourceColumnExistence(self, doSkySources): calibConfig.doAstrometry = False calibConfig.doPhotoCal = False calibConfig.doSkySources = doSkySources + calibConfig.doComputeSummaryStats = False calibTask = CalibrateTask(config=calibConfig) calibResults = calibTask.run(charImResults.exposure) if doSkySources: