Fix the bug

huggingface · Jul 7, 2024 · 663a7ae · 663a7ae
1 parent 843a0f8
commit 663a7ae
Showing 1 changed file with 12 additions and 0 deletions.
diff --git a/src/lighteval/tasks/lighteval_task.py b/src/lighteval/tasks/lighteval_task.py
@@ -538,6 +538,18 @@ def construct_requests(
                     generation_size=self.generation_size,
                 )
             ]
+        if self.has_metric_category[MetricCategory.LLM_AS_JUDGE]:
+            requests[RequestType.GREEDY_UNTIL] += [
+                GreedyUntilRequest(
+                    task_name=current_task_name,
+                    example_index=document_id_seed,
+                    request_index=0,
+                    context=context,
+                    stop_sequence=self.stop_sequence,
+                    generation_size=self.generation_size,
+                    num_samples=1,
+                )
+            ]
 
         return requests