From 2a6da98263eabc7d49db636c1a6cab7fe1a7b2eb Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Wed, 17 Jul 2024 09:14:06 +0000 Subject: [PATCH] add examples of samples (does not include few shot), and add robustness to logger --- src/lighteval/__main__.py | 41 ++++++++++++++++---- src/lighteval/logging/hierarchical_logger.py | 19 ++++++--- src/lighteval/main_accelerate.py | 1 - src/lighteval/parsers.py | 9 ----- 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/src/lighteval/__main__.py b/src/lighteval/__main__.py index 9deb09251..3d4318d34 100644 --- a/src/lighteval/__main__.py +++ b/src/lighteval/__main__.py @@ -23,9 +23,13 @@ # SOFTWARE. import argparse +import os from lighteval.parsers import parser_accelerate, parser_nanotron -from lighteval.tasks.registry import Registry +from lighteval.tasks.registry import Registry, taskinfo_selector + + +CACHE_DIR = os.getenv("HF_HOME") def cli_evaluate(): @@ -40,25 +44,48 @@ def cli_evaluate(): parser_b = subparsers.add_parser("nanotron", help="use nanotron as backend for evaluation.") parser_nanotron(parser_b) - parser.add_argument("--list-tasks", action="store_true", help="List available tasks") + # values which should always be set + parser.add_argument( + "--cache_dir", type=str, default=CACHE_DIR, help="Cache directory used to store datasets and models" + ) + # utils functions + parser.add_argument("--list-tasks", action="store_true", help="List available tasks") + parser.add_argument( + "--tasks-examples", + type=str, + default=None, + help="Id of tasks or path to a text file with a list of tasks (e.g. 'original|mmlu:abstract_algebra|5') for which you want to manually inspect samples.", + ) args = parser.parse_args() if args.subcommand == "accelerate": from lighteval.main_accelerate import main as main_accelerate main_accelerate(args) - return - if args.subcommand == "nanotron": + elif args.subcommand == "nanotron": from lighteval.main_nanotron import main as main_nanotron main_nanotron(args.checkpoint_config_path, args.lighteval_override, args.cache_dir) - return - if args.list_tasks: + elif args.list_tasks: Registry(cache_dir="").print_all_tasks() - return + + elif args.tasks_examples: + print(f"Loading the tasks dataset to cache folder: {args.cache_dir}") + print( + "All examples will be displayed without few shot, as few shot sample construction requires loading a model and using its tokenizer." + ) + task_names_list, _ = taskinfo_selector(args.tasks_examples) + task_dict = Registry(cache_dir=args.cache_dir).get_task_dict(task_names_list) + for name, task in task_dict.items(): + print("-" * 10, name, "-" * 10) + for sample in task.eval_docs()[:10]: + print(sample) + + else: + print("You did not provide any argument. Exiting") if __name__ == "__main__": diff --git a/src/lighteval/logging/hierarchical_logger.py b/src/lighteval/logging/hierarchical_logger.py index 5efb44153..9cf718f18 100644 --- a/src/lighteval/logging/hierarchical_logger.py +++ b/src/lighteval/logging/hierarchical_logger.py @@ -23,6 +23,7 @@ import sys import time from datetime import timedelta +from logging import Logger from typing import Any, Callable from lighteval.utils import is_accelerate_available, is_nanotron_available @@ -37,8 +38,6 @@ logger = get_logger(__name__, log_level="INFO") else: - from logging import Logger - logger = Logger(__name__, level="INFO") from colorama import Fore, Style @@ -76,6 +75,7 @@ def log(self, x: Any) -> None: HIERARCHICAL_LOGGER = HierarchicalLogger() +BACKUP_LOGGER = Logger(__name__, level="INFO") # Exposed public methods @@ -84,7 +84,10 @@ def hlog(x: Any) -> None: Logs a string version of x through the singleton [`HierarchicalLogger`]. """ - HIERARCHICAL_LOGGER.log(x) + try: + HIERARCHICAL_LOGGER.log(x) + except RuntimeError: + BACKUP_LOGGER.warning(x) def hlog_warn(x: Any) -> None: @@ -92,7 +95,10 @@ def hlog_warn(x: Any) -> None: Logs a string version of x, which will appear in a yellow color, through the singleton [`HierarchicalLogger`]. """ - HIERARCHICAL_LOGGER.log(Fore.YELLOW + str(x) + Style.RESET_ALL) + try: + HIERARCHICAL_LOGGER.log(Fore.YELLOW + str(x) + Style.RESET_ALL) + except RuntimeError: + BACKUP_LOGGER.warning(Fore.YELLOW + str(x) + Style.RESET_ALL) def hlog_err(x: Any) -> None: @@ -100,7 +106,10 @@ def hlog_err(x: Any) -> None: Logs a string version of x, which will appear in a red color, through the singleton [`HierarchicalLogger`]. """ - HIERARCHICAL_LOGGER.log(Fore.RED + str(x) + Style.RESET_ALL) + try: + HIERARCHICAL_LOGGER.log(Fore.RED + str(x) + Style.RESET_ALL) + except RuntimeError: + BACKUP_LOGGER.warning(Fore.RED + str(x) + Style.RESET_ALL) class htrack_block: diff --git a/src/lighteval/main_accelerate.py b/src/lighteval/main_accelerate.py index 12122c527..904a68322 100644 --- a/src/lighteval/main_accelerate.py +++ b/src/lighteval/main_accelerate.py @@ -43,7 +43,6 @@ hlog_warn("Using either accelerate or text-generation to run this script is advised.") TOKEN = os.getenv("HF_TOKEN") -CACHE_DIR = os.getenv("HF_HOME") if is_accelerate_available(): from accelerate import Accelerator, InitProcessGroupKwargs diff --git a/src/lighteval/parsers.py b/src/lighteval/parsers.py index d05ba312f..95191c6f2 100644 --- a/src/lighteval/parsers.py +++ b/src/lighteval/parsers.py @@ -66,9 +66,6 @@ def parser_accelerate(parser=None): parser.add_argument( "--public_run", default=False, action="store_true", help="Push results and details to a public repo" ) - parser.add_argument( - "--cache_dir", type=str, default=CACHE_DIR, help="Cache directory used to store datasets and models" - ) parser.add_argument( "--results_org", type=str, @@ -120,9 +117,3 @@ def parser_nanotron(parser=None): type=str, help="Path to an optional YAML or python Lighteval config to override part of the checkpoint Lighteval config", ) - parser.add_argument( - "--cache-dir", - type=str, - default=None, - help="Cache directory", - )