tokenizer is never built when converting finetuning dataset (mosaicml…

…#1496)
ShashankMosaicML · Aug 29, 2024 · 100f40f · 100f40f
1 parent 8516181
commit 100f40f
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/llmfoundry/command_utils/data_prep/convert_finetuning_dataset.py b/llmfoundry/command_utils/data_prep/convert_finetuning_dataset.py
@@ -165,7 +165,6 @@ def convert_finetuning_dataset(
         decoder_only_format=not encoder_decoder,
     )
 
-    tokenizer = None
     tokenizer_kwargs = tokenizer_kwargs
     tokenizer_kwargs.update({'model_max_length': max_seq_len})
     if tokenizer: