handle launcher monitoring failure condition (#2240)

NVIDIA · Dec 21, 2023 · 2ad8da2 · 2ad8da2
1 parent d67baf4
commit 2ad8da2
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/nvflare/app_common/executors/launcher_executor.py b/nvflare/app_common/executors/launcher_executor.py
@@ -396,6 +396,11 @@ def _monitor_launcher(self, fl_ctx: FLContext):
                             f"launcher completed {task_name} with status {run_status} at time {self._launcher_finish_time}",
                         )
 
+                    if run_status == LauncherRunStatus.COMPLETE_FAILED:
+                        msg = f"Launcher failed with at time {self._launcher_finish_time} "
+                        self._abort_signal.trigger(msg)
+                        break
+
                 if not self._launcher_finish:
                     continue