[eval] fix evaluation empty answers issue

imoneoi · Mar 8, 2024 · bbc6e51 · bbc6e51
1 parent 859c868
commit bbc6e51
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 4 deletions.
diff --git a/ochat/evaluation/match_answer.py b/ochat/evaluation/match_answer.py
@@ -109,7 +109,7 @@ def fs_cothub_bbh_match_answer(task_data, response):
         return False, ans
     else:
         # Free form, direct return
-        if ans[-1] == '.':
+        if len(ans) and ans[-1] == '.':
             ans = ans[:-1]
 
         return True, ans

diff --git a/ochat/evaluation/run_eval.py b/ochat/evaluation/run_eval.py
@@ -146,7 +146,7 @@ def get_model_answers(
                  max_model_len=model_config.model_max_context,
                  tensor_parallel_size=tensor_parallel_size)
     sampling_params = SamplingParams(temperature=0,
-                                     max_tokens=model_config.model_max_context,
+                                     max_tokens=None,
                                      stop_token_ids=conv_template.eot_tokens_,  # Override stop tokens
                                      ignore_eos=True)
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -26,14 +26,14 @@ dependencies = [
     "torch", 
     "ray", 
     "sentencepiece", 
-    "transformers>=4.35.0", 
+    "transformers>=4.38.2", 
     "accelerate", 
     "protobuf",
     "fastapi", 
     "pydantic", 
     "shortuuid", 
     "uvicorn", 
-    "vllm",
+    "vllm>=0.3.3",
     "pytest"
 ]