Use hardlinks rather than copies for checkpoint creation.

chorowski-lab · Dec 30, 2020 · 904d119 · 904d119
1 parent 3858c03
commit 904d119
Showing 1 changed file with 8 additions and 3 deletions.
diff --git a/fairseq/checkpoint_utils.py b/fairseq/checkpoint_utils.py
@@ -93,9 +93,14 @@ def is_better(a, b):
     if len(checkpoints) > 0:
         trainer.save_checkpoint(checkpoints[0], extra_state)
         for cp in checkpoints[1:]:
-            assert PathManager.copy(
-                checkpoints[0], cp, overwrite=True
-            ), f"Failed to copy {checkpoints[0]} to {cp}"
+            # assert PathManager.copy(
+            #     checkpoints[0], cp, overwrite=True
+            # ), f"Failed to copy {checkpoints[0]} to {cp}"
+            try:
+                os.remove(cp)
+            except:
+                pass
+            os.link(checkpoints[0], cp)
 
         write_timer.stop()
         logger.info(