launch new run only on main process

mosaicml · Nov 9, 2023 · 21e9880 · 21e9880
1 parent ba871d7
commit 21e9880
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/llmfoundry/callbacks/async_eval_callback.py b/llmfoundry/callbacks/async_eval_callback.py
@@ -109,8 +109,12 @@ def __init__(
 
     def run_event(self, event: Event, state: State, logger: Logger) -> None:
         del logger
-        if state.get_elapsed_duration() is not None and self.check_interval(
-                state, event) and self.last_launch != state.timestamp.batch:
+        if all([
+                state.get_elapsed_duration() is not None,
+                self.check_interval(state, event), self.last_launch
+                != state.timestamp.batch,
+                dist.get_global_rank() == 0
+        ]):
             self.launch_run()
 
             self.last_launch = state.timestamp.batch