diff --git a/doc/changes/DM-46525.misc.md b/doc/changes/DM-46525.misc.md new file mode 100644 index 00000000..545a6ade --- /dev/null +++ b/doc/changes/DM-46525.misc.md @@ -0,0 +1,3 @@ +Set the default for raising on partial output error to `True`. + +Allowing processing to proceed when we encounter an error that may not be fatal is functionality we'll still want eventually, but enabling it by default was premature, since our processing-status reporting tools are yet able to distinguish these cases from unqualified successes. diff --git a/python/lsst/ctrl/mpexec/cli/opt/options.py b/python/lsst/ctrl/mpexec/cli/opt/options.py index 98b8fafc..43035521 100644 --- a/python/lsst/ctrl/mpexec/cli/opt/options.py +++ b/python/lsst/ctrl/mpexec/cli/opt/options.py @@ -450,9 +450,10 @@ ) raise_on_partial_outputs_option = MWOptionDecorator( - "--raise-on-partial-outputs", + "--raise-on-partial-outputs/--no-raise-on-partial-outputs", help="Consider partial outputs from a task an error instead of a qualified success.", is_flag=True, + default=True, ) save_execution_butler_option = MWOptionDecorator( diff --git a/python/lsst/ctrl/mpexec/separablePipelineExecutor.py b/python/lsst/ctrl/mpexec/separablePipelineExecutor.py index 4fc8ae9f..6e2950e4 100644 --- a/python/lsst/ctrl/mpexec/separablePipelineExecutor.py +++ b/python/lsst/ctrl/mpexec/separablePipelineExecutor.py @@ -100,7 +100,7 @@ def __init__( skip_existing_in: Iterable[str] | None = None, task_factory: lsst.pipe.base.TaskFactory | None = None, resources: lsst.pipe.base.ExecutionResources | None = None, - raise_on_partial_outputs: bool = False, + raise_on_partial_outputs: bool = True, ): self._butler = Butler.from_config(butler=butler, collections=butler.collections, run=butler.run) if not self._butler.collections: diff --git a/python/lsst/ctrl/mpexec/simple_pipeline_executor.py b/python/lsst/ctrl/mpexec/simple_pipeline_executor.py index b22f1e76..d95c7cd2 100644 --- a/python/lsst/ctrl/mpexec/simple_pipeline_executor.py +++ b/python/lsst/ctrl/mpexec/simple_pipeline_executor.py @@ -84,7 +84,7 @@ def __init__( quantum_graph: QuantumGraph, butler: Butler, resources: ExecutionResources | None = None, - raise_on_partial_outputs: bool = False, + raise_on_partial_outputs: bool = True, ): self.quantum_graph = quantum_graph self.butler = butler @@ -148,7 +148,7 @@ def from_pipeline_filename( bind: Mapping[str, Any] | None = None, butler: Butler, resources: ExecutionResources | None = None, - raise_on_partial_outputs: bool = False, + raise_on_partial_outputs: bool = True, ) -> SimplePipelineExecutor: """Create an executor by building a QuantumGraph from an on-disk pipeline YAML file. @@ -201,7 +201,7 @@ def from_task_class( bind: Mapping[str, Any] | None = None, butler: Butler, resources: ExecutionResources | None = None, - raise_on_partial_outputs: bool = False, + raise_on_partial_outputs: bool = True, ) -> SimplePipelineExecutor: """Create an executor by building a QuantumGraph from a pipeline containing a single task. @@ -268,7 +268,7 @@ def from_pipeline( bind: Mapping[str, Any] | None = None, butler: Butler, resources: ExecutionResources | None = None, - raise_on_partial_outputs: bool = False, + raise_on_partial_outputs: bool = True, ) -> SimplePipelineExecutor: """Create an executor by building a QuantumGraph from an in-memory pipeline. @@ -321,7 +321,7 @@ def from_pipeline_graph( bind: Mapping[str, Any] | None = None, butler: Butler, resources: ExecutionResources | None = None, - raise_on_partial_outputs: bool = False, + raise_on_partial_outputs: bool = True, ) -> SimplePipelineExecutor: """Create an executor by building a QuantumGraph from an in-memory pipeline graph. diff --git a/python/lsst/ctrl/mpexec/singleQuantumExecutor.py b/python/lsst/ctrl/mpexec/singleQuantumExecutor.py index 54c8ef87..1a86f411 100644 --- a/python/lsst/ctrl/mpexec/singleQuantumExecutor.py +++ b/python/lsst/ctrl/mpexec/singleQuantumExecutor.py @@ -138,7 +138,7 @@ def __init__( resources: ExecutionResources | None = None, skipExisting: bool = False, assumeNoExistingOutputs: bool = False, - raise_on_partial_outputs: bool = False, + raise_on_partial_outputs: bool = True, ): self.butler = butler self.taskFactory = taskFactory diff --git a/tests/test_simple_pipeline_executor.py b/tests/test_simple_pipeline_executor.py index 6cfe449e..b32c87db 100644 --- a/tests/test_simple_pipeline_executor.py +++ b/tests/test_simple_pipeline_executor.py @@ -249,8 +249,10 @@ def test_partial_outputs_success(self): pipeline_graph = PipelineGraph() pipeline_graph.add_task("a", DynamicTestPipelineTask, config_a) pipeline_graph.add_task("b", DynamicTestPipelineTask, config_b) - # Default behavior is to consider the partial a success and proceed. - executor = SimplePipelineExecutor.from_pipeline_graph(pipeline_graph, butler=self.butler) + # Consider the partial a success and proceed. + executor = SimplePipelineExecutor.from_pipeline_graph( + pipeline_graph, butler=self.butler, raise_on_partial_outputs=False + ) (_, _) = executor.as_generator(register_dataset_types=True) self.assertFalse(self.butler.exists("intermediate")) self.assertEqual(self.butler.get("output").storage_class, get_mock_name("StructuredDataDict"))