Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Expose details of samples #227

Closed
wants to merge 26 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add files
NathanHB committed Apr 22, 2024
commit a128176dbf37eab3204163f1827b6770a409ba5d
135 changes: 135 additions & 0 deletions src/lighteval/commands/parsers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
# MIT License

# Copyright (c) 2024 Taratra D. RAHARISON and The HuggingFace Team

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import argparse
import os


TOKEN = os.getenv("HF_TOKEN")
CACHE_DIR = os.getenv("HF_HOME")


def parser_accelerate(parser=None):
if parser is None:
parser = argparse.ArgumentParser(
description="CLI tool for lighteval, a lightweight framework for LLM evaluation"
)

group = parser.add_mutually_exclusive_group(required=True)
task_type_group = parser.add_mutually_exclusive_group(required=True)

# Model type: either use a config file or simply the model name
task_type_group.add_argument(
"--model_config_path",
type=str,
help="Path to the model config file, e.g. 'examples/model_configs/base_model.yaml'",
)
task_type_group.add_argument(
"--model_args",
type=str,
help="Model arguments to pass to the model class, e.g. 'pretrained=gpt2,dtype=float16'",
)

# Accelerate parameters
parser.add_argument(
"--num_processes",
type=int,
default=1,
help="Number of processes for accelerate to use, this directly translates to the data parallelism level",
)

# Debug
parser.add_argument("--max_samples", type=int, default=None, help="Maximum number of samples to evaluate on")
parser.add_argument("--override_batch_size", type=int, default=-1)
parser.add_argument("--job_id", type=str, help="Optional Job ID for future reference", default="")

# Saving
parser.add_argument("--output_dir", required=True, type=str, help="Directory to save the results")
parser.add_argument(
"--push_results_to_hub", default=False, action="store_true", help="Set to push the results to the hub"
)
parser.add_argument("--save_details", action="store_true", help="Save the details of the run in the output_dir")
parser.add_argument(
"--push_details_to_hub", default=False, action="store_true", help="Set to push the details to the hub"
)
parser.add_argument(
"--public_run", default=False, action="store_true", help="Push results and details to a public repo"
)
parser.add_argument(
"--cache_dir", type=str, default=CACHE_DIR, help="Cache directory used to store datasets and models"
)
parser.add_argument(
"--results_org",
type=str,
help="Hub organisation where you want to store the results. Your current token must have write access to it",
)
# Common parameters
parser.add_argument(
"--use_chat_template",
default=False,
action="store_true",
help="Use the chat template (from the model's tokenizer) for the prompt",
)
parser.add_argument(
"--system_prompt", type=str, default=None, help="System prompt to use, e.g. 'You are a helpful assistant.'"
)
parser.add_argument(
"--dataset_loading_processes", type=int, default=1, help="Number of processes to use for loading the datasets"
)
parser.add_argument(
"--custom_tasks",
type=str,
default=None,
help="Path to a file with custom tasks (a TASK list of dict and potentially prompt formating functions)",
)
group.add_argument(
"--tasks",
type=str,
default=None,
help="Id of a task, e.g. 'original|mmlu:abstract_algebra|5' or path to a texte file with a list of tasks",
)
parser.add_argument("--num_fewshot_seeds", type=int, default=1, help="Number of trials the few shots")
return parser


def parser_nanotron(parser=None):
if parser is None:
parser = argparse.ArgumentParser(
description="CLI tool for lighteval, a lightweight framework for LLM evaluation"
)

parser.add_argument(
"--checkpoint-config-path",
type=str,
required=True,
help="Path to the brr checkpoint YAML or python config file, potentially on S3",
)
parser.add_argument(
"--lighteval-override",
type=str,
help="Path to an optional YAML or python Lighteval config to override part of the checkpoint Lighteval config",
)
parser.add_argument(
"--cache-dir",
type=str,
default=None,
help="Cache directory",
)
134 changes: 134 additions & 0 deletions src/lighteval/commands/utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
#!/usr/bin/env python

# MIT License

# Copyright (c) 2024 Taratra D. RAHARISON and The HuggingFace Team

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import importlib
import json
import os

import pkg_resources


def load_tasks_table_extended(module_name: any) -> list:
"""
load the module module_name

Args:
- module_name the name of the module we want to load
Returns:
- TASKS_TABLE: a list of the task in the module
"""
module_path = f"lighteval.tasks.extended.{module_name}.main"
module_loaded = importlib.import_module(module_path)
tasks_list = None
try:
tasks_list = module_loaded.TASKS_TABLE
except Exception as e:
print(e)
return tasks_list if tasks_list is not None else []


def get_tasks_table_json() -> list:
"""
Fetch tasks/tasks_table.jsonl
Returns
- a list of all the tasks in tasks/tasks_table.jsonl
"""
tasks = []
# Handling tasks_table.jsonl
# Get the path to the resource file
tasks_table_path = pkg_resources.resource_filename("lighteval", "tasks/tasks_table.jsonl")
with open(tasks_table_path) as jsonl_tasks_table:
jsonl_tasks_table_content = jsonl_tasks_table.read()
for jline in jsonl_tasks_table_content.splitlines():
tasks.append(json.loads(jline))
return tasks


def get_extended_tasks() -> list:
"""
Fetch all the tasks in the extended suite
Returns
- a list of all the extended tasks
"""
tasks_extended = []
extended_tasks_dir = pkg_resources.resource_filename("lighteval", "tasks/extended")
for root, dirs, files in os.walk(extended_tasks_dir):
for file in files:
if file == "main.py":
module_name = os.path.basename(root)
tasks_table = load_tasks_table_extended(module_name)
tasks_extended += tasks_table
return tasks_extended


def group_by_suite(tasks: list, tasks_extended: list) -> dict:
"""
Group tasks by suite and sort them alphabetically
Args:
- tasks: list of tasks in tasks/tasks_table.jsonl
- tasks_extended: list of extended tasks
Returns:
- a dict of tasks grouped by suite
"""
grouped_by_suite = {}
for task in tasks:
for suite in task["suite"]:
if suite not in grouped_by_suite.keys():
grouped_by_suite[suite] = [task["name"]]
else:
grouped_by_suite[suite].append(task["name"])
grouped_by_suite[suite].sort()

grouped_by_suite["extended"] = []
# Adding extended suite
for task in tasks_extended:
grouped_by_suite["extended"].append(task["name"])
grouped_by_suite["extended"].sort()
return grouped_by_suite


def list_tasks_command():
"""
List all the available tasks in tasks_table.jsonl and the extended directory
Assumes the existence of TASKS_TABLE in the main.py file for each extended
tasks in tasks/extended
"""
try:
# Handling tasks_table.jsonl
tasks = get_tasks_table_json()

# Handling extended tasks
tasks_extended = get_extended_tasks()

# Grouping by suite the tasks
grouped_by_suite = group_by_suite(tasks, tasks_extended)

# Print tasks
print("Available tasks: (Grouped by suite)\n")
for suite, task_list in grouped_by_suite.items():
print("- " + suite)
for task in task_list:
print("\t - " + task)
except Exception as e:
print("Error: ", e)