drop empty tokenized rows too (#509)

axolotl-ai-cloud · Aug 30, 2023 · c56b450 · c56b450
1 parent 1e07c16
commit c56b450
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/axolotl/utils/trainer.py b/src/axolotl/utils/trainer.py
@@ -361,7 +361,7 @@ def add_position_ids(sample):
 
 
 def drop_long_seq(sample, sequence_len=2048):
-    return len(sample["input_ids"]) <= sequence_len
+    return len(sample["input_ids"]) <= sequence_len and len(sample["input_ids"]) > 0
 
 
 @contextmanager