From 454fca062ea351d6efbb4e85aa40fe6d8fa205a8 Mon Sep 17 00:00:00 2001 From: jrmullaney Date: Sun, 17 Mar 2024 03:44:56 -0700 Subject: [PATCH 1/3] Added creation of metrics to calibrate.py Added the ability to create and write metrics from the calexp summary stats. The task to calculate the metrics must be retargeted at runtime to avoid circular imports in calibrate. This is the reason for the private _EmptyTargetTask. --- python/lsst/pipe/tasks/calibrate.py | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/python/lsst/pipe/tasks/calibrate.py b/python/lsst/pipe/tasks/calibrate.py index dd9cc663c..4ae89bfaa 100644 --- a/python/lsst/pipe/tasks/calibrate.py +++ b/python/lsst/pipe/tasks/calibrate.py @@ -46,6 +46,13 @@ from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask +class _EmptyTargetTask(pipeBase.PipelineTask): + ConfigClass = pipeBase.PipelineTaskConfig + + def __init__(self, **kwargs) -> None: + raise NotImplementedError("CreateSummaryMetrics must be retargeted") + + class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), defaultTemplates={}): @@ -121,6 +128,13 @@ class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instru dimensions=("instrument", "visit", "detector"), ) + outputSummaryMetrics = cT.Output( + doc="Summary metrics created by the calibration task", + name="calexpSummary_metrics", + storageClass="MetricMeasurementBundle", + dimensions=("instrument", "visit", "detector"), + ) + matches = cT.Output( doc="Source/refObj matches from the astrometry solver", name="srcMatch", @@ -148,6 +162,9 @@ def __init__(self, *, config=None): if config.doWriteMatchesDenormalized is False or config.doAstrometry is False: self.outputs.remove("matchesDenormalized") + if config.doCreateSummaryMetrics is False: + self.outputs.remove("outputSummaryMetrics") + class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections): """Config for CalibrateTask.""" @@ -292,6 +309,16 @@ class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Calibrate target=ComputeExposureSummaryStatsTask, doc="Subtask to run computeSummaryStats on exposure" ) + doCreateSummaryMetrics = pexConfig.Field( + dtype=bool, + default=False, + doc="Run the subtask to create summary metrics, and then write those metrics?" + ) + createSummaryMetrics = pexConfig.ConfigurableField( + target=_EmptyTargetTask, + doc="Subtask to create metrics from the summary stats. This must be retargeted, likely to an" + "analysis_tools task such as CalexpSummaryMetrics." + ) doWriteExposure = pexConfig.Field( dtype=bool, default=True, @@ -445,6 +472,8 @@ def __init__(self, astromRefObjLoader=None, schema=self.schema) if self.config.doComputeSummaryStats: self.makeSubtask('computeSummaryStats') + if self.config.doCreateSummaryMetrics: + self.makeSubtask('createSummaryMetrics') if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None): raise RuntimeError("PipelineTask form of this task should not be initialized with " @@ -643,11 +672,14 @@ def run(self, exposure, background=None, exposureId=idGenerator.catalog_id, ) + summaryMetrics = None if self.config.doComputeSummaryStats: summary = self.computeSummaryStats.run(exposure=exposure, sources=sourceCat, background=background) exposure.getInfo().setSummaryStats(summary) + if self.config.doCreateSummaryMetrics: + summaryMetrics = self.createSummaryMetrics.run(data=summary.__dict__).metrics frame = getDebugFrame(self._display, "calibrate") if frame: @@ -666,6 +698,7 @@ def run(self, exposure, background=None, outputExposure=exposure, outputCat=sourceCat, outputBackground=background, + outputSummaryMetrics=summaryMetrics ) def setMetadata(self, exposure, photoRes=None): From 846ea3f70dba29b0d9735a16a4dc5af32b4af176 Mon Sep 17 00:00:00 2001 From: jrmullaney Date: Tue, 19 Mar 2024 11:01:29 -0700 Subject: [PATCH 2/3] Changed exception text if CreateSummaryMetrics isn't retargeted. --- python/lsst/pipe/tasks/calibrate.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/lsst/pipe/tasks/calibrate.py b/python/lsst/pipe/tasks/calibrate.py index 4ae89bfaa..4b0a545a8 100644 --- a/python/lsst/pipe/tasks/calibrate.py +++ b/python/lsst/pipe/tasks/calibrate.py @@ -50,7 +50,10 @@ class _EmptyTargetTask(pipeBase.PipelineTask): ConfigClass = pipeBase.PipelineTaskConfig def __init__(self, **kwargs) -> None: - raise NotImplementedError("CreateSummaryMetrics must be retargeted") + raise NotImplementedError( + "doCreateSummaryMetrics is set to True, in which case" + "CreateSummaryMetrics must be retargeted." + ) class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"), @@ -312,7 +315,7 @@ class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Calibrate doCreateSummaryMetrics = pexConfig.Field( dtype=bool, default=False, - doc="Run the subtask to create summary metrics, and then write those metrics?" + doc="Run the subtask to create summary metrics, and then write those metrics." ) createSummaryMetrics = pexConfig.ConfigurableField( target=_EmptyTargetTask, From 6a7085f05ed05b7627e9142d1cbd7058ee5b5318 Mon Sep 17 00:00:00 2001 From: jrmullaney Date: Tue, 19 Mar 2024 15:48:03 -0700 Subject: [PATCH 3/3] Added docstring to _EmptyTargetTask --- python/lsst/pipe/tasks/calibrate.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/python/lsst/pipe/tasks/calibrate.py b/python/lsst/pipe/tasks/calibrate.py index 4b0a545a8..55ed853e6 100644 --- a/python/lsst/pipe/tasks/calibrate.py +++ b/python/lsst/pipe/tasks/calibrate.py @@ -47,12 +47,19 @@ class _EmptyTargetTask(pipeBase.PipelineTask): + """ + This is a placeholder target for CreateSummaryMetrics and must be retargeted at runtime. + CreateSummaryMetrics should target an analysis tool task, but that would, at the time + of writing, result in a circular import. + + As a result, this class should not be used for anything else. + """ ConfigClass = pipeBase.PipelineTaskConfig def __init__(self, **kwargs) -> None: raise NotImplementedError( - "doCreateSummaryMetrics is set to True, in which case" - "CreateSummaryMetrics must be retargeted." + "doCreateSummaryMetrics is set to True, in which case " + "createSummaryMetrics must be retargeted." )