warn about not pre-processing

axolotl-ai-cloud · Jan 20, 2024 · 334f02c · 334f02c
1 parent a787d57
commit 334f02c
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/src/axolotl/utils/trainer.py b/src/axolotl/utils/trainer.py
@@ -107,6 +107,10 @@ def drop_long_seq(sample, sequence_len=2048):
 
 
 def process_datasets_for_packing(cfg, train_dataset, eval_dataset, tokenizer):
+    if cfg.is_preprocess:
+        LOG.warning(
+            "Processing datasets during training can lead to VRAM instability. Please pre-process your dataset"
+        )
     drop_long = partial(drop_long_seq, sequence_len=cfg.sequence_len)
     with zero_first(is_main_process()):
         if cfg.group_by_length: