maybe done; test out on mcli now

mosaicml · Feb 2, 2024 · 3696f8d · 3696f8d
1 parent 289ca55
commit 3696f8d
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/mcli/mcli-hf-eval.yaml b/mcli/mcli-hf-eval.yaml
@@ -15,8 +15,8 @@ command: |
 # Mosaic Cloud will use run_name (with a unique suffix) to populate the env var $RUN_NAME
 run_name: mpt-eval
 gpu_num: 8
-# gpu_type:
-# cluster:  # replace with your cluster here!
+gpu_type: a100_80gb
+cluster: r1z1 # replace with your cluster here!
 
 image: mosaicml/llm-foundry:2.1.0_cu121_flash2-latest
 

diff --git a/tests/eval/test_nlp_metrics.py b/tests/eval/test_nlp_metrics.py
@@ -41,7 +41,7 @@ def test_in_context_learning_qa_accuracy():
     labels = [['Correct'], ['blah', 'blah2'], ['blah', 'correct']]
     batch = {'cot_delimiter': '', 'labels': labels}
     metric = InContextLearningQAAccuracy()
-    metric.update(outputs, labels, batch)
+    metric.update(batch, outputs, labels)
 
     assert metric.compute() == (2 / 3)
 
@@ -55,7 +55,7 @@ def test_in_context_learning_qa_cot_accuracy():
     labels = [['Correct'], ['blah', 'blah2'], ['blah', 'correct'], ['correct']]
     batch = {'cot_delimiter': ' ### ', 'labels': labels, 'do_normalization': True, 'stopping_criteria': '\n\n'}
     metric = InContextLearningQAAccuracy()
-    metric.update(outputs, labels, batch)
+    metric.update(batch, outputs, labels)
 
     assert metric.compute() == (2 / 4)