Merge pull request #309 from lsst/tickets/DM-46525

DM-46525: switch back to raising on partial output errors by default

doc/changes/DM-46525.misc.md

@@ -0,0 +1,3 @@
+Set the default for raising on partial output error to `True`.
+
+Allowing processing to proceed when we encounter an error that may not be fatal is functionality we'll still want eventually, but enabling it by default was premature, since our processing-status reporting tools are yet able to distinguish these cases from unqualified successes.

python/lsst/ctrl/mpexec/cli/opt/options.py

@@ -450,9 +450,10 @@
 )
 
 raise_on_partial_outputs_option = MWOptionDecorator(
-    "--raise-on-partial-outputs",
+    "--raise-on-partial-outputs/--no-raise-on-partial-outputs",
     help="Consider partial outputs from a task an error instead of a qualified success.",
     is_flag=True,
+    default=True,
 )
 
 save_execution_butler_option = MWOptionDecorator(

python/lsst/ctrl/mpexec/separablePipelineExecutor.py

@@ -100,7 +100,7 @@ def __init__(
         skip_existing_in: Iterable[str] | None = None,
         task_factory: lsst.pipe.base.TaskFactory | None = None,
         resources: lsst.pipe.base.ExecutionResources | None = None,
-        raise_on_partial_outputs: bool = False,
+        raise_on_partial_outputs: bool = True,
     ):
         self._butler = Butler.from_config(butler=butler, collections=butler.collections, run=butler.run)
         if not self._butler.collections:

python/lsst/ctrl/mpexec/simple_pipeline_executor.py

@@ -84,7 +84,7 @@ def __init__(
         quantum_graph: QuantumGraph,
         butler: Butler,
         resources: ExecutionResources | None = None,
-        raise_on_partial_outputs: bool = False,
+        raise_on_partial_outputs: bool = True,
     ):
         self.quantum_graph = quantum_graph
         self.butler = butler
@@ -148,7 +148,7 @@ def from_pipeline_filename(
         bind: Mapping[str, Any] | None = None,
         butler: Butler,
         resources: ExecutionResources | None = None,
-        raise_on_partial_outputs: bool = False,
+        raise_on_partial_outputs: bool = True,
     ) -> SimplePipelineExecutor:
         """Create an executor by building a QuantumGraph from an on-disk
         pipeline YAML file.
@@ -201,7 +201,7 @@ def from_task_class(
         bind: Mapping[str, Any] | None = None,
         butler: Butler,
         resources: ExecutionResources | None = None,
-        raise_on_partial_outputs: bool = False,
+        raise_on_partial_outputs: bool = True,
     ) -> SimplePipelineExecutor:
         """Create an executor by building a QuantumGraph from a pipeline
         containing a single task.
@@ -268,7 +268,7 @@ def from_pipeline(
         bind: Mapping[str, Any] | None = None,
         butler: Butler,
         resources: ExecutionResources | None = None,
-        raise_on_partial_outputs: bool = False,
+        raise_on_partial_outputs: bool = True,
     ) -> SimplePipelineExecutor:
         """Create an executor by building a QuantumGraph from an in-memory
         pipeline.
@@ -321,7 +321,7 @@ def from_pipeline_graph(
         bind: Mapping[str, Any] | None = None,
         butler: Butler,
         resources: ExecutionResources | None = None,
-        raise_on_partial_outputs: bool = False,
+        raise_on_partial_outputs: bool = True,
     ) -> SimplePipelineExecutor:
         """Create an executor by building a QuantumGraph from an in-memory
         pipeline graph.

python/lsst/ctrl/mpexec/singleQuantumExecutor.py

@@ -138,7 +138,7 @@ def __init__(
         resources: ExecutionResources | None = None,
         skipExisting: bool = False,
         assumeNoExistingOutputs: bool = False,
-        raise_on_partial_outputs: bool = False,
+        raise_on_partial_outputs: bool = True,
     ):
         self.butler = butler
         self.taskFactory = taskFactory

tests/test_simple_pipeline_executor.py

@@ -249,8 +249,10 @@ def test_partial_outputs_success(self):
         pipeline_graph = PipelineGraph()
         pipeline_graph.add_task("a", DynamicTestPipelineTask, config_a)
         pipeline_graph.add_task("b", DynamicTestPipelineTask, config_b)
-        # Default behavior is to consider the partial a success and proceed.
-        executor = SimplePipelineExecutor.from_pipeline_graph(pipeline_graph, butler=self.butler)
+        # Consider the partial a success and proceed.
+        executor = SimplePipelineExecutor.from_pipeline_graph(
+            pipeline_graph, butler=self.butler, raise_on_partial_outputs=False
+        )
         (_, _) = executor.as_generator(register_dataset_types=True)
         self.assertFalse(self.butler.exists("intermediate"))
         self.assertEqual(self.butler.get("output").storage_class, get_mock_name("StructuredDataDict"))