algo

mosaicml · Nov 30, 2024 · e2f9cc5 · e2f9cc5
1 parent b20e7fd
commit e2f9cc5
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/tests/test_full_nlp.py b/tests/test_full_nlp.py
@@ -259,7 +259,6 @@ def test_full_nlp_pipeline(
 
     device = get_device(device)
     config = None
-    tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased', model_max_length=128)
     if model_type == 'tinybert_hf':
         # Updated minimal BERT configuration
         config = BertConfig(
@@ -270,7 +269,7 @@ def test_full_nlp_pipeline(
             intermediate_size=256,
         )
         tiny_bert_model = BertForMaskedLM(config)
-
+        tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased', model_max_length=128)
         pretraining_metrics = [LanguageCrossEntropy(ignore_index=-100), MaskedAccuracy(ignore_index=-100)]
         pretraining_model = HuggingFaceModel(
             tiny_bert_model,