Fix: eval table conflict with eval_sample_packing (#769)

axolotl-ai-cloud · Oct 22, 2023 · 9923b72 · 9923b72
1 parent 21cf09b
commit 9923b72
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 0 deletions.
diff --git a/src/axolotl/utils/config.py b/src/axolotl/utils/config.py
@@ -348,6 +348,15 @@ def validate_config(cfg):
             "eval_steps and evaluation_strategy are not supported with val_set_size == 0"
         )
 
+    if (
+        cfg.sample_packing
+        and cfg.eval_table_size
+        and cfg.eval_sample_packing is not False
+    ):
+        raise ValueError(
+            "eval_table_size and eval_sample_packing are not supported together with sample_packing. Please set 'eval_sample_packing' to false."
+        )
+
     # TODO
     # MPT 7b
     # https://github.com/facebookresearch/bitsandbytes/issues/25

diff --git a/tests/test_validation.py b/tests/test_validation.py
@@ -565,3 +565,44 @@ def test_no_conflict_eval_strategy(self):
         )
 
         validate_config(cfg)
+
+    def test_eval_table_size_conflict_eval_packing(self):
+        cfg = DictDefault(
+            {
+                "sample_packing": True,
+                "eval_table_size": 100,
+            }
+        )
+
+        with pytest.raises(
+            ValueError, match=r".*Please set 'eval_sample_packing' to false.*"
+        ):
+            validate_config(cfg)
+
+        cfg = DictDefault(
+            {
+                "sample_packing": True,
+                "eval_sample_packing": False,
+            }
+        )
+
+        validate_config(cfg)
+
+        cfg = DictDefault(
+            {
+                "sample_packing": False,
+                "eval_table_size": 100,
+            }
+        )
+
+        validate_config(cfg)
+
+        cfg = DictDefault(
+            {
+                "sample_packing": True,
+                "eval_table_size": 100,
+                "eval_sample_packing": False,
+            }
+        )
+
+        validate_config(cfg)