diff --git a/dmlcloud/stage.py b/dmlcloud/stage.py index 6ace04d..d4b5a25 100644 --- a/dmlcloud/stage.py +++ b/dmlcloud/stage.py @@ -136,6 +136,7 @@ def run(self): if self._stop_requested: break self._post_stage() + dist.barrier() # this will time out if not all processes reach this point def _pre_stage(self): self.start_time = datetime.now()