fix: Set tokenizer.model_max_length if it is None

alexandrainst · Nov 28, 2022 · c57f874 · c57f874
1 parent 150724f
commit c57f874
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/scandinli/train.py b/src/scandinli/train.py
@@ -54,7 +54,7 @@ def train(config: DictConfig) -> None:
     )
 
     # Ensure that `model_max_length` is set
-    if tokenizer.model_max_length > 100_000:
+    if tokenizer.model_max_length > 100_000 or tokenizer.model_max_length is None:
         tokenizer.model_max_length = 512
 
     # Tokenize the dataset