From fa77006ab024ad8400c085e8bd16577c8ab946c9 Mon Sep 17 00:00:00 2001 From: John Parejko Date: Tue, 27 Jun 2023 01:04:27 -0700 Subject: [PATCH 1/2] Remove now-defaulted overrides Appropriate config overrides have been set in the subtasks, so these pipetask overrides are now either unnecessary (default is now the value that was here), or incorrect (default is now the value that had been overridden in obs_lsst and obs_subaru). This commit is not intended to change pipeline behavior: the combined effect of this branch across all packages is to cleanup our configs so that the task defaults are what we believe to be most appropriate for most cases. --- python/lsst/pipe/tasks/calibrate.py | 2 -- python/lsst/pipe/tasks/calibrateImage.py | 10 ---------- python/lsst/pipe/tasks/characterizeImage.py | 1 - tests/test_calibrateImage.py | 3 +++ 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/python/lsst/pipe/tasks/calibrate.py b/python/lsst/pipe/tasks/calibrate.py index 0fa1f3472..622d9a4f6 100644 --- a/python/lsst/pipe/tasks/calibrate.py +++ b/python/lsst/pipe/tasks/calibrate.py @@ -303,8 +303,6 @@ class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Calibrate def setDefaults(self): super().setDefaults() - self.detection.doTempLocalBackground = False - self.deblend.maxFootprintSize = 2000 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"] self.postCalibrationMeasurement.doReplaceWithNoise = False for key in self.postCalibrationMeasurement.slots: diff --git a/python/lsst/pipe/tasks/calibrateImage.py b/python/lsst/pipe/tasks/calibrateImage.py index daf8cbf62..8a2f0f55d 100644 --- a/python/lsst/pipe/tasks/calibrateImage.py +++ b/python/lsst/pipe/tasks/calibrateImage.py @@ -240,7 +240,6 @@ def setDefaults(self): # Only use high S/N sources for PSF determination. self.psf_detection.thresholdValue = 50.0 - self.psf_detection.thresholdType = "pixel_stdev" # TODO investigation: Probably want False here, but that may require # tweaking the background spatial scale, to make it small enough to # prevent extra peaks in the wings of bright objects. @@ -268,9 +267,6 @@ def setDefaults(self): # https://github.com/lsst/meas_extensions_psfex/blob/main/config/default-lsst.psfex#L14 self.psf_source_measurement.plugins["base_CircularApertureFlux"].radii = [9.0, 12.0] - self.psf_measure_psf.starSelector["objectSize"].doFluxLimit = False - self.psf_measure_psf.starSelector["objectSize"].doSignalToNoiseLimit = True - # No extendeness information available: we need the aperture # corrections to determine that. self.measure_aperture_correction.sourceSelector["science"].doUnresolved = False @@ -283,7 +279,6 @@ def setDefaults(self): # Detection for good S/N for astrometry/photometry and other # downstream tasks. self.star_detection.thresholdValue = 10.0 - self.star_detection.thresholdType = "pixel_stdev" self.star_measurement.plugins = ["base_PixelFlags", "base_SdssCentroid", "ext_shapeHSM_HsmSourceMoments", @@ -296,8 +291,6 @@ def setDefaults(self): self.star_measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments" # Only measure the apertures we need for star selection. self.star_measurement.plugins["base_CircularApertureFlux"].radii = [12.0] - # Restrict footprint area to prevent memory blowup on huge footprints. - self.star_deblend.maxFootprintArea = 10000 # Select isolated stars with reliable measurements and no bad flags. self.star_selector["science"].doFlags = True @@ -311,9 +304,6 @@ def setDefaults(self): # phot_g_mean is the primary Gaia band for all input bands. self.astrometry_ref_loader.anyFilterMapsToThis = "phot_g_mean" - # Reject magnitude outliers (TODO DM-39796: should be task default) - self.astrometry.doMagnitudeOutlierRejection = True - # Do not subselect during fitting; we already selected good stars. self.astrometry.sourceSelector = "null" self.photometry.match.sourceSelection.retarget(sourceSelector.NullSourceSelectorTask) diff --git a/python/lsst/pipe/tasks/characterizeImage.py b/python/lsst/pipe/tasks/characterizeImage.py index cbcdf4a27..f9e0da29c 100644 --- a/python/lsst/pipe/tasks/characterizeImage.py +++ b/python/lsst/pipe/tasks/characterizeImage.py @@ -232,7 +232,6 @@ def setDefaults(self): # but these are the values we have been using self.detection.thresholdValue = 5.0 self.detection.includeThresholdMultiplier = 10.0 - self.detection.doTempLocalBackground = False # do not deblend, as it makes a mess self.doDeblend = False # measure and apply aperture correction; note: measuring and applying aperture diff --git a/tests/test_calibrateImage.py b/tests/test_calibrateImage.py index 97f20443d..fe43af757 100644 --- a/tests/test_calibrateImage.py +++ b/tests/test_calibrateImage.py @@ -107,6 +107,9 @@ def setUp(self): # We don't have many test points, so can't match on complicated shapes. self.config.astrometry.matcher.numPointsForShape = 3 + # Something about this test dataset prefers the older fluxRatio here. + self.config.star_catalog_calculation.plugins['base_ClassificationExtendedness'].fluxRatio = 0.925 + def test_run(self): """Test that run() returns reasonable values to be butler put. """ From 72701232645b2989e2dac580f87f5231feb2913c Mon Sep 17 00:00:00 2001 From: John Parejko Date: Tue, 19 Sep 2023 15:41:47 -0700 Subject: [PATCH 2/2] Default to doWriteMatchesDenormalized=True LSST, DECam, and HSC were all overriding this to True; it is not that much more space (compared to e.g. images) and makes debugging much eaiser. --- python/lsst/pipe/tasks/calibrate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lsst/pipe/tasks/calibrate.py b/python/lsst/pipe/tasks/calibrate.py index 622d9a4f6..58201066e 100644 --- a/python/lsst/pipe/tasks/calibrate.py +++ b/python/lsst/pipe/tasks/calibrate.py @@ -171,10 +171,10 @@ class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Calibrate ) doWriteMatchesDenormalized = pexConfig.Field( dtype=bool, - default=False, + default=True, doc=("Write reference matches in denormalized format? " "This format uses more disk space, but is more convenient to " - "read. Ignored if doWriteMatches=False or doWrite=False."), + "read for debugging. Ignored if doWriteMatches=False or doWrite=False."), ) doAstrometry = pexConfig.Field( dtype=bool,