pc

mosaicml · Jul 23, 2024 · 07f54dc · 07f54dc
1 parent 3c5498f
commit 07f54dc
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/llmfoundry/command_utils/data_prep/convert_text_to_mds.py b/llmfoundry/command_utils/data_prep/convert_text_to_mds.py
@@ -398,7 +398,8 @@ def convert_text_to_mds(
     # Load the tokenizer once on the main process so that the files are cached to avoid race conditions
     # in the Hugging Face load code
     AutoTokenizer.from_pretrained(
-        tokenizer_name, trust_remote_code=trust_remote_code
+        tokenizer_name,
+        trust_remote_code=trust_remote_code,
     )
 
     is_remote_output = is_remote_path(output_folder)