diff --git a/src/lighteval/tasks/extended/ifeval/main.py b/src/lighteval/tasks/extended/ifeval/main.py index 119a0403..69fe53c5 100644 --- a/src/lighteval/tasks/extended/ifeval/main.py +++ b/src/lighteval/tasks/extended/ifeval/main.py @@ -46,23 +46,6 @@ def ifeval_prompt(line, task_name: str = None): ) -# We create the task config -ifeval = LightevalTaskConfig( - name="ifeval", - prompt_function=ifeval_prompt, - suite=["extended"], - hf_repo="wis-k/instruction-following-eval", - hf_subset="default", - metric=["ifeval_metric"], - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split="train", - few_shots_select="random_sampling", - generation_size=1280, - stop_sequence=[], # no stop sequence, will use eot token -) - - submetric_names = [ "prompt_level_strict_acc", "inst_level_strict_acc", @@ -156,6 +139,22 @@ def agg_inst_level_acc(items): }, ) +# We create the task config +ifeval = LightevalTaskConfig( + name="ifeval", + prompt_function=ifeval_prompt, + suite=["extended"], + hf_repo="wis-k/instruction-following-eval", + hf_subset="default", + metric=[ifeval_metrics], + hf_avail_splits=["train"], + evaluation_splits=["train"], + few_shots_split="train", + few_shots_select="random_sampling", + generation_size=1280, + stop_sequence=[], # no stop sequence, will use eot token +) + TASKS_TABLE = [ifeval]