diff --git a/llmfoundry/callbacks/hf_checkpointer.py b/llmfoundry/callbacks/hf_checkpointer.py
index fe3028ab19..492816ea07 100644
--- a/llmfoundry/callbacks/hf_checkpointer.py
+++ b/llmfoundry/callbacks/hf_checkpointer.py
@@ -46,7 +46,7 @@ def __init__(
         save_folder: str,
         save_interval: Union[str, int, Time],
         huggingface_folder_name: str = 'ba{batch}',
-        precision: str = 'fp32',
+        precision: str = 'float32',
         overwrite: bool = False,
     ):
         self.backend, self.bucket_name, self.save_dir_format_str = parse_uri(
diff --git a/tests/test_hf_conversion_script.py b/tests/test_hf_conversion_script.py
index 2a175a04e9..c944dcfc97 100644
--- a/tests/test_hf_conversion_script.py
+++ b/tests/test_hf_conversion_script.py
@@ -176,6 +176,10 @@ def get_config(
     return cast(DictConfig, test_cfg)
 
 
+def test_callback_inits_with_defaults():
+    _ = HuggingFaceCheckpointer(save_folder='test', save_interval='1ba')
+
+
 @pytest.mark.world_size(2)
 @pytest.mark.gpu
 @pytest.mark.parametrize('model', ['mpt', 'neo', 'llama2'])