upload dev jsonl

scicode-bench · Jul 18, 2024 · 1aa33c0 · 1aa33c0
1 parent 230af84
commit 1aa33c0
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 7 deletions.
diff --git a/eval/data/problems_dev.jsonl b/eval/data/problems_dev.jsonl
diff --git a/eval/scripts/test_generated_code.py b/eval/scripts/test_generated_code.py
@@ -10,10 +10,12 @@
 
 
 PROB_NUM = 80
+DEV_PROB_NUM = 15
 STEP_NUM = 288
+DEV_STEP_NUM = 50
 
 
-def test_code(model_name, code_dir, log_dir, output_dir, jsonl_path):
+def test_code(model_name, code_dir, log_dir, output_dir, jsonl_path, dev_set=False):
 
     jsonl_data = read_from_jsonl(jsonl_path)
     json_dct = {}
@@ -111,14 +113,14 @@ def run_script(script_path):
                            correct_prob[i] == tot_prob[i]
                            and tot_prob[i] != 0)
 
-    print(f'correct problems: {correct_prob_num}/{PROB_NUM - 15}')
-    print(f'correct steps: {len(correct_step)}/{STEP_NUM}')
+    print(f'correct problems: {correct_prob_num}/{DEV_PROB_NUM if dev_set else PROB_NUM - DEV_PROB_NUM}')
+    print(f'correct steps: {len(correct_step)}/{DEV_STEP_NUM if dev_set else STEP_NUM}')
 
     output_dir.mkdir(parents=True, exist_ok=True)
 
     with open(f'{output_dir}/{model_name}.txt', 'w') as f:
-        f.write(f'correct problems: {correct_prob_num}/{PROB_NUM - 15}\n')
-        f.write(f'correct steps: {len(correct_step)}/{STEP_NUM}\n\n')
+        f.write(f'correct problems: {correct_prob_num}/{DEV_PROB_NUM if dev_set else PROB_NUM - DEV_PROB_NUM}\n')
+        f.write(f'correct steps: {len(correct_step)}/{DEV_STEP_NUM if dev_set else STEP_NUM}\n\n')
         f.write(f'duration: {test_time} seconds\n')
         f.write('\ncorrect problems: ')
         f.write(f'\n\n{[i + 1 for i in range(PROB_NUM) if correct_prob[i] == tot_prob[i] and tot_prob[i] != 0]}\n')
@@ -160,16 +162,22 @@ def get_cli() -> argparse.ArgumentParser:
         default=Path("eval", "data", "problems_all.jsonl"),
         help="Path to jsonl file",
     )
+    parser.add_argument(
+        "--dev-set",
+        action='store_true',
+        help="Test dev set if enabled",
+    )
     return parser
 
 
 def main(model: str,
          code_dir: Path,
          log_dir: Path,
          output_dir: Path,
-         jsonl_path: Path
+         jsonl_path: Path,
+         dev_set: bool
 ) -> None:
-    test_code(model, code_dir, log_dir, output_dir, jsonl_path)
+    test_code(model, code_dir, log_dir, output_dir, jsonl_path, dev_set)
 
 
 if __name__ == "__main__":