From db997f2c16fc971ebd7e776780ac3d3dac932c2e Mon Sep 17 00:00:00 2001 From: Sean Lee Date: Sat, 29 Jun 2024 10:44:12 +0800 Subject: [PATCH] save tokenizer to best checkpoint folder --- angle_emb/angle.py | 1 + 1 file changed, 1 insertion(+) diff --git a/angle_emb/angle.py b/angle_emb/angle.py index 6be0462..895e689 100644 --- a/angle_emb/angle.py +++ b/angle_emb/angle.py @@ -1493,6 +1493,7 @@ def fit(self, best_ckpt_dir = None if output_dir is not None: best_ckpt_dir = os.path.join(output_dir, 'best-checkpoint') + self.tokenizer.save_pretrained(best_ckpt_dir) evaluate_callback = EvaluateCallback(self.backbone, valid_ds, partial(self.evaluate, batch_size=batch_size, device=self.device), save_dir=best_ckpt_dir)