Gauntlet v0.3: Fix chain-of-thought tasks

llmfoundry/utils/builders.py

@@ -501,7 +501,11 @@ def _validate_cfg(icl_cfg: DictConfig):
             if dist.get_local_rank() == 0 and os.path.exists(destination_path):
                 os.remove(destination_path)
             dist.barrier()
-
+            early_stopping_criteria = icl_cfg.get('early_stopping_criteria',
+                                                  None)
+            early_stopping_criteria = list(
+                early_stopping_criteria
+            ) if early_stopping_criteria is not None else None
             dataloaders = get_icl_task_dataloader(
                 icl_cfg.icl_task_type,
                 icl_cfg.dataset_uri,
@@ -518,7 +522,9 @@ def _validate_cfg(icl_cfg: DictConfig):
                 pass_at_k=icl_cfg.pass_at_k,
                 generations_per_sample=icl_cfg.num_beams,
                 has_categories=icl_cfg.get('has_categories', False),
-                cot_delimiter=icl_cfg.get('cot_delimiter', ''))
+                cot_delimiter=icl_cfg.get('cot_delimiter', ''),
+                early_stopping_criteria=early_stopping_criteria,
+                do_normalization=icl_cfg.get('do_normalization', True))
             if hasattr(
                     icl_cfg,
                     'has_categories') and icl_cfg.has_categories and isinstance(

mcli/mcli-hf-eval.yaml

@@ -50,5 +50,5 @@ parameters:
     limit_all_gathers: True
 
 
-  icl_tasks: 'eval/yamls/tasks_v0.2.yaml'
-  eval_gauntlet: 'eval/yamls/eval_gauntlet_v0.2.yaml'
+  icl_tasks: 'eval/yamls/tasks_v0.3.yaml'
+  eval_gauntlet: 'eval/yamls/eval_gauntlet_v0.3.yaml'

scripts/eval/yamls/eval_gauntlet_v0.3.yaml

@@ -0,0 +1,144 @@
+eval_gauntlet:
+  weighting: EQUAL
+  subtract_random_baseline: true
+  rescale_accuracy: true
+  averages:
+    core_average:
+    - world_knowledge
+    - commonsense_reasoning
+    - language_understanding
+    - symbolic_problem_solving
+    - reading_comprehension
+  categories:
+  - name: world_knowledge
+    benchmarks:
+    - name: jeopardy
+      num_fewshot: 3
+      random_baseline: 0
+    - name: bigbench_qa_wikidata
+      num_fewshot: 3
+      random_baseline: 0
+    - name: arc_easy
+      num_fewshot: 3
+      random_baseline: 0.25
+    - name: arc_challenge
+      num_fewshot: 3
+      random_baseline: 0.25
+    - name: mmlu
+      num_fewshot: 5
+      random_baseline: 0.25
+    - name: triviaqa_sm_sub
+      num_fewshot: 3
+      random_baseline: 0.0
+  - name: commonsense_reasoning
+    benchmarks:
+    - name: copa
+      num_fewshot: 0
+      random_baseline: 0.5
+    - name: siqa
+      num_fewshot: 3
+      random_baseline: 0.5
+    - name: commonsense_qa
+      num_fewshot: 0
+      random_baseline: 0.25
+    - name: piqa
+      num_fewshot: 0
+      random_baseline: 0.5
+    - name: openbook_qa
+      num_fewshot: 10
+      random_baseline: 0.25
+    - name: bigbench_strange_stories
+      num_fewshot: 0
+      random_baseline: 0.5
+    - name: bigbench_strategy_qa
+      num_fewshot: 0
+      random_baseline: 0.5
+  - name: language_understanding
+    benchmarks:
+    - name: lambada_openai
+      num_fewshot: 0
+      random_baseline: 0.0
+    - name: hellaswag
+      num_fewshot: 0
+      random_baseline: 0.25
+    - name: winograd
+      num_fewshot: 3
+      random_baseline: 0.5
+    - name: winogrande
+      num_fewshot: 5
+      random_baseline: 0.5
+  - name: symbolic_problem_solving
+    benchmarks:
+    - name: bigbench_elementary_math_qa
+      num_fewshot: 1
+      random_baseline: 0.25
+    - name: bigbench_dyck_languages
+      num_fewshot: 5
+      random_baseline: 0
+    - name: bigbench_operators
+      num_fewshot: 3
+      random_baseline: 0.0
+    - name: simple_arithmetic_withspaces
+      num_fewshot: 5
+      random_baseline: 0.0
+    - name: simple_arithmetic_nospaces
+      num_fewshot: 5
+      random_baseline: 0.0
+    - name: aqua
+      num_fewshot: 3
+      random_baseline: 0.0
+    - name: gsm8k
+      num_fewshot: 0
+      random_baseline: 0.0
+    - name: svamp
+      num_fewshot: 5
+      random_baseline: 0
+    - name: agi_eval_sat_math
+      num_fewshot: 3
+      random_baseline: 0.0
+    - name: agi_eval_lsat_ar
+      num_fewshot: 5
+      random_baseline: 0.25
+    - name: math
+      num_fewshot: 4
+      random_baseline: 0.0
+    - name: math_simple
+      num_fewshot: 4
+      random_baseline: 0.0
+  - name: reading_comprehension
+    benchmarks:
+    - name: squad
+      num_fewshot: 3
+      random_baseline: 0
+    - name: boolq
+      num_fewshot: 0
+      random_baseline: 0.5
+    - name: coqa
+      num_fewshot: 0
+      random_baseline: 0.0
+    - name: agi_eval_lsat_rc
+      num_fewshot: 5
+      random_baseline: 0.25
+    - name: agi_eval_lsat_lr
+      num_fewshot: 5
+      random_baseline: 0.25
+    - name: agi_eval_sat_en
+      num_fewshot: 5
+      random_baseline: 0.25
+  - name: programming
+    benchmarks:
+    - name: human_eval
+      num_fewshot: 0
+      random_baseline: 0
+    - name: human_eval_cpp
+      num_fewshot: 0
+      random_baseline: 0
+    - name: human_eval_js
+      num_fewshot: 0
+      random_baseline: 0
+    - name: human_eval_return_simple
+      num_fewshot: 0
+      random_baseline: 0
+    - name: human_eval_25
+      num_fewshot: 0
+      random_baseline: 0

scripts/eval/yamls/hf_eval.yaml

@@ -43,5 +43,5 @@ device_eval_batch_size: 4
 #   forward_prefetch: True
 #   limit_all_gathers: True
 
-icl_tasks: 'eval/yamls/tasks_v0.1.yaml'
-eval_gauntlet: 'eval/yamls/eval_gauntlet_v0.1.yaml'
+icl_tasks: 'eval/yamls/tasks_v0.3.yaml'
+eval_gauntlet: 'eval/yamls/eval_gauntlet_v0.3.yaml'