add simple human_eval

mosaicml · Oct 3, 2023 · a84dda0 · a84dda0
1 parent 0f8b160
commit a84dda0
Show file tree

Hide file tree

Showing 7 changed files with 83 additions and 35 deletions.
diff --git a/...l/local_data/programming/split-0.25.jsonl → ...al_data/programming/human_eval-0.25.jsonl b/...l/local_data/programming/split-0.25.jsonl → ...al_data/programming/human_eval-0.25.jsonl
diff --git a/...al/local_data/programming/split-0.5.jsonl → ...cal_data/programming/human_eval-0.5.jsonl b/...al/local_data/programming/split-0.5.jsonl → ...cal_data/programming/human_eval-0.5.jsonl
diff --git a/...l/local_data/programming/split-0.75.jsonl → ...al_data/programming/human_eval-0.75.jsonl b/...l/local_data/programming/split-0.75.jsonl → ...al_data/programming/human_eval-0.75.jsonl
diff --git a/scripts/eval/yamls/coding_tasks.yaml b/scripts/eval/yamls/coding_tasks.yaml
@@ -20,3 +20,37 @@ icl_tasks:
   pass_at_k: 1
   num_beams: 20
   icl_task_type: code_evaluation
+-
+  label: human_eval_return_simple
+  dataset_uri: eval/local_data/programming/human_eval_return_simple.jsonl # ADD YOUR OWN DATASET URI
+  num_fewshot: [0]
+  pass_at_k : 1
+  num_beams: 20
+-
+  label: human_eval_return_complex
+  dataset_uri: eval/local_data/programming/human_eval_return_complex.jsonl # ADD YOUR OWN DATASET URI
+  num_fewshot: [0]
+  pass_at_k : 1
+  num_beams: 20
+  icl_task_type: code_evaluation
+-
+  label: human_eval_25
+  dataset_uri: eval/local_data/programming/split-0.25.jsonl # ADD YOUR OWN DATASET URI
+  num_fewshot: [0]
+  pass_at_k : 1
+  num_beams: 20
+  icl_task_type: code_evaluation
+-
+  label: human_eval_50
+  dataset_uri: eval/local_data/programming/split-0.5.jsonl # ADD YOUR OWN DATASET URI
+  num_fewshot: [0]
+  pass_at_k : 1
+  num_beams: 20
+  icl_task_type: code_evaluation
+-
+  label: human_eval_75
+  dataset_uri: eval/local_data/programming/split-0.75.jsonl # ADD YOUR OWN DATASET URI
+  num_fewshot: [0]
+  pass_at_k : 1
+  num_beams: 20
+  icl_task_type: code_evaluation
diff --git a/scripts/eval/yamls/coding_tasks_simple.yaml b/scripts/eval/yamls/coding_tasks_simple.yaml
diff --git a/scripts/eval/yamls/eval_gauntlet.yaml b/scripts/eval/yamls/eval_gauntlet.yaml
@@ -123,6 +123,21 @@ eval_gauntlet:
     - name: human_eval_js
       num_fewshot: 0
       random_baseline: 0.0
+    - name: human_eval_return_simple
+      num_fewshot: 0
+      random_baseline: 0.0
+    - name: human_eval_return_complex
+      num_fewshot: 0
+      random_baseline: 0.0
+    - name: human_eval_25
+      num_fewshot: 0
+      random_baseline: 0.0
+    - name: human_eval_50
+      num_fewshot: 0
+      random_baseline: 0.0
+    - name: human_eval_75
+      num_fewshot: 0
+      random_baseline: 0.0
   - name: world_knowledge_lm_task_subscore
     benchmarks:
     - name: jeopardy

diff --git a/scripts/eval/yamls/tasks.yaml b/scripts/eval/yamls/tasks.yaml
@@ -194,3 +194,37 @@ icl_tasks:
   pass_at_k: 1
   num_beams: 20
   icl_task_type: code_evaluation
+-
+  label: human_eval_return_simple
+  dataset_uri: eval/local_data/programming/human_eval_return_simple.jsonl # ADD YOUR OWN DATASET URI
+  num_fewshot: [0]
+  pass_at_k : 1
+  num_beams: 20
+-
+  label: human_eval_return_complex
+  dataset_uri: eval/local_data/programming/human_eval_return_complex.jsonl # ADD YOUR OWN DATASET URI
+  num_fewshot: [0]
+  pass_at_k : 1
+  num_beams: 20
+  icl_task_type: code_evaluation
+-
+  label: human_eval_25
+  dataset_uri: eval/local_data/programming/human_eval-0.25.jsonl # ADD YOUR OWN DATASET URI
+  num_fewshot: [0]
+  pass_at_k : 1
+  num_beams: 20
+  icl_task_type: code_evaluation
+-
+  label: human_eval_50
+  dataset_uri: eval/local_data/programming/human_eval-0.5.jsonl # ADD YOUR OWN DATASET URI
+  num_fewshot: [0]
+  pass_at_k : 1
+  num_beams: 20
+  icl_task_type: code_evaluation
+-
+  label: human_eval_75
+  dataset_uri: eval/local_data/programming/human_eval-0.75.jsonl # ADD YOUR OWN DATASET URI
+  num_fewshot: [0]
+  pass_at_k : 1
+  num_beams: 20
+  icl_task_type: code_evaluation