diff --git a/llmfoundry/callbacks/hf_checkpointer.py b/llmfoundry/callbacks/hf_checkpointer.py
index 303208ed6b..4eabd8a060 100644
--- a/llmfoundry/callbacks/hf_checkpointer.py
+++ b/llmfoundry/callbacks/hf_checkpointer.py
@@ -217,10 +217,13 @@ def __init__(
             )
 
         self.mlflow_logging_config = mlflow_logging_config
-        self.pretrained_model_name = self.mlflow_logging_config['metadata'].get(
-            'pretrained_model_name',
-            None,
-        )
+        if 'metadata' in self.mlflow_logging_config:
+            self.pretrained_model_name = self.mlflow_logging_config['metadata'].get(
+                'pretrained_model_name',
+                None,
+            )
+        else:
+            self.pretrained_model_name = None
 
         self.huggingface_folder_name_fstr = os.path.join(
             'huggingface',