Merge branch 'main' into chuck/aws-docker

llmfoundry/utils/builders.py

@@ -190,9 +190,11 @@ def build_tokenizer(
 
     signal_file_path = f'.node_{dist.get_node_rank()}_local_rank0_completed_tokenizer_setup'
 
-    # Make sure the tokenizer files are downloaded and cached first by local rank 0
-    with dist.local_rank_zero_download_and_wait(signal_file_path):
-        pass
+    if dist.is_available() and dist.is_initialized(
+    ) and dist.get_world_size() > 1:
+        # Make sure the tokenizer files are downloaded and cached first by local rank 0
+        with dist.local_rank_zero_download_and_wait(signal_file_path):
+            pass
 
     if tokenizer_name.startswith('tiktoken'):
         tokenizer = TiktokenTokenizerWrapper(**tokenizer_kwargs)
@@ -208,14 +210,16 @@ def build_tokenizer(
             int(1e30),
         )
 
-    if dist.get_local_rank() == 0:
-        with open(signal_file_path, 'wb') as f:
-            f.write(b'local_rank0_completed_tokenizer_setup')
+    if dist.is_available() and dist.is_initialized(
+    ) and dist.get_world_size() > 1:
+        if dist.get_local_rank() == 0:
+            with open(signal_file_path, 'wb') as f:
+                f.write(b'local_rank0_completed_tokenizer_setup')
 
-    dist.barrier()
+        dist.barrier()
 
-    if dist.get_local_rank() == 0:
-        os.remove(signal_file_path)
+        if dist.get_local_rank() == 0:
+            os.remove(signal_file_path)
 
     return tokenizer