update

mosaicml · Oct 11, 2024 · 3e50fbf · 3e50fbf
1 parent 0478d49
commit 3e50fbf
Showing 1 changed file with 0 additions and 4 deletions.
diff --git a/llmfoundry/data/finetuning/dataloader.py b/llmfoundry/data/finetuning/dataloader.py
@@ -204,10 +204,6 @@ def build_finetuning_dataloader(
         allowed_dataset_keys=allowed_dataset_config_keys,
     )
 
-    allowed_dataset_config_keys = allowed_dataset_config_keys.union(
-        extraneous_keys,
-    )
-
     # Use EOS as the pad token if none exists
     if tokenizer.pad_token is None:  # type: ignore (sometimes it's none and that's ok)
         tokenizer.pad_token = tokenizer.eos_token