zero3 can'y use 8bit optimizer

axolotl-ai-cloud · Nov 18, 2024 · 543cec0 · 543cec0
1 parent b7b74e9
commit 543cec0
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/tests/e2e/multigpu/test_llama.py b/tests/e2e/multigpu/test_llama.py
@@ -637,7 +637,7 @@ def test_8bit_lora_ds_zero3(self, temp_dir):
                 "gradient_accumulation_steps": 4,
                 "output_dir": temp_dir,
                 "learning_rate": 0.00001,
-                "optimizer": "adamw_8bit",
+                "optimizer": "adamw_torch",
                 "lr_scheduler": "cosine",
                 "flash_attention": True,
                 "deepspeed": "deepspeed_configs/zero3_bf16_cpuoffload_all.json",