improve vram use w gradient checkpointing

axolotl-ai-cloud · Jan 22, 2024 · 6029ced · 6029ced
1 parent fccb542
commit 6029ced
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/src/axolotl/utils/config.py b/src/axolotl/utils/config.py
@@ -160,6 +160,13 @@ def normalize_config(cfg):
     if isinstance(cfg.pretraining_dataset, dict):
         cfg.pretraining_dataset = [cfg.pretraining_dataset]
 
+    if (
+        cfg.gradient_checkpointing
+        and cfg.unfrozen_parameters is None
+        and cfg.gradient_checkpointing_kwargs is None
+    ):
+        cfg.gradient_checkpointing_kwargs = {"use_reentrant": True}
+
     log_gpu_memory_usage(LOG, "baseline", cfg.device)