feat: synchronize more

sehoffmann · Apr 2, 2024 · 0f1b720 · 0f1b720
1 parent 04a8a4f
commit 0f1b720
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/dmlcloud/stage.py b/dmlcloud/stage.py
@@ -136,6 +136,7 @@ def run(self):
             if self._stop_requested:
                 break
         self._post_stage()
+        dist.barrier()  # this will time out if not all processes reach this point
 
     def _pre_stage(self):
         self.start_time = datetime.now()