restore

mosaicml · Dec 28, 2023 · fd9daf0 · fd9daf0
1 parent 2a83bdd
commit fd9daf0
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 79 deletions.
diff --git a/llmfoundry/utils/builders.py b/llmfoundry/utils/builders.py
@@ -11,9 +11,9 @@
 
 import torch
 from composer import algorithms
-from composer.callbacks import (EarlyStopper, Generate, LRMonitor, MemoryMonitor,
-                                OptimizerMonitor, RuntimeEstimator,
-                                SpeedMonitor)
+from composer.callbacks import (EarlyStopper, Generate, LRMonitor,
+                                MemoryMonitor, OptimizerMonitor,
+                                RuntimeEstimator, SpeedMonitor)
 from composer.core import Algorithm, Callback, Evaluator
 from composer.datasets.in_context_learning_evaluation import \
     get_icl_task_dataloader
@@ -494,7 +494,6 @@ def _validate_cfg(icl_cfg: DictConfig):
             if dist.get_local_rank() == 0 and os.path.exists(destination_path):
                 os.remove(destination_path)
             dist.barrier()
-            print(f'Stopping criteria: {icl_cfg.get("early_stopping_criteria", None)}')
             early_stopping_criteria = icl_cfg.get('early_stopping_criteria', None)
             early_stopping_criteria = list(early_stopping_criteria) if early_stopping_criteria is not None else None
             dataloaders = get_icl_task_dataloader(

diff --git a/scripts/eval/eval.py b/scripts/eval/eval.py
@@ -6,7 +6,6 @@
 import sys
 import time
 import warnings
-from composer.core.callback import Callback
 from typing import Any, Dict, List, Optional, Tuple, Union
 
 import pandas as pd
@@ -22,7 +21,7 @@
 
 from llmfoundry.models import MPTForCausalLM
 from llmfoundry.models.model_registry import COMPOSER_MODEL_REGISTRY
-from llmfoundry.utils.builders import (add_metrics_to_eval_loaders, build_callback,
+from llmfoundry.utils.builders import (add_metrics_to_eval_loaders,
                                        build_evaluators, build_logger,
                                        build_tokenizer)
 from llmfoundry.utils.config_utils import pop_config, process_init_device
@@ -115,7 +114,6 @@ def evaluate_model(
     precision: str,
     eval_gauntlet_df: Optional[pd.DataFrame],
     icl_subset_num_batches: Optional[int],
-    callback_configs: Optional[Dict]
 ):
 
     print(f'Evaluating model: {model_cfg.model_name}', flush=True)
@@ -137,12 +135,7 @@ def evaluate_model(
         icl_subset_num_batches=icl_subset_num_batches,
     )
 
-    # Callbacks
-    callbacks: List[Callback] = [
-        build_callback(str(name), callback_cfg)
-        for name, callback_cfg in callback_configs.items()
-    ] if callback_configs else []
-
+    callbacks = []
     if eval_gauntlet_callback is not None:
         callbacks.append(eval_gauntlet_callback)
 
@@ -199,7 +192,6 @@ def evaluate_model(
         dist_timeout=dist_timeout,
         python_log_level=python_log_level,
     )
-
 
     if torch.cuda.is_available():
         torch.cuda.synchronize()
@@ -280,11 +272,7 @@ def main(cfg: DictConfig) -> Tuple[List[Trainer], pd.DataFrame]:
                                              default_value=None)
     # Pop out interpolation variables.
     pop_config(cfg, 'model_name_or_path', must_exist=False, default_value=None)
-    callback_configs: Optional[DictConfig] = pop_config(cfg,
-                                                        'callbacks',
-                                                        must_exist=False,
-                                                        default_value=None)
-
+
     # Warn for unused parameters
     for key in cfg:
         warnings.warn(
@@ -325,7 +313,6 @@ def main(cfg: DictConfig) -> Tuple[List[Trainer], pd.DataFrame]:
              python_log_level=python_log_level,
              precision=precision,
              eval_gauntlet_df=eval_gauntlet_df,
-             callback_configs=callback_configs,
              icl_subset_num_batches=icl_subset_num_batches)
         trainers.append(trainer)
 
@@ -369,9 +356,6 @@ def main(cfg: DictConfig) -> Tuple[List[Trainer], pd.DataFrame]:
     return trainers, eval_gauntlet_df
 
 
-
-
-
 def calculate_markdown_results(logger_keys: List[str], trainer: Trainer,
                                benchmark_to_taxonomy: Dict[str, str],
                                model_name: str):

diff --git a/scripts/eval/yamls/eval_gauntlet_v0.1.yaml b/scripts/eval/yamls/eval_gauntlet_v0.1.yaml
@@ -121,12 +121,6 @@ eval_gauntlet:
     - name: gsm8k
       num_fewshot: 3
       random_baseline: 0.0
-    - name: math_simple
-      num_fewshot: 3
-      random_baseline: 0.0
-    - name: math_complex
-      num_fewshot: 3
-      random_baseline: 0.0
     - name: svamp
       num_fewshot: 3
       random_baseline: 0

diff --git a/scripts/eval/yamls/hf_eval.yaml b/scripts/eval/yamls/hf_eval.yaml
@@ -8,19 +8,16 @@ model_name_or_path: EleutherAI/gpt-neo-125m
 
 models:
 -
-    model_name: mosaicml/mpt-7b
-    # Tokenizer
-    tokenizer:
-      name: mosaicml/mpt-7b
-      kwargs:
-        model_max_length: ${max_seq_len}
-
-    model:
-      name: hf_causal_lm
-      pretrained_model_name_or_path: mosaicml/mpt-7b
-      init_device: cpu
-      pretrained: true
-      use_auth_token: false
+  model_name: ${model_name_or_path}
+  model:
+    name: hf_causal_lm
+    pretrained_model_name_or_path: ${model_name_or_path}
+    init_device: mixed
+    pretrained: true
+  tokenizer:
+    name: ${model_name_or_path}
+    kwargs:
+      model_max_length: ${max_seq_len}
 # # if you are evaluating more than one model, list them all as YAML blocks without variable interpolation
 # -
 #   model_name: mosaicml/mpt-7b
@@ -46,17 +43,5 @@ device_eval_batch_size: 4
 #   forward_prefetch: True
 #   limit_all_gathers: True
 
-
-icl_tasks:
--
-  label: gsm8k
-  dataset_uri: eval/local_data/symbolic_problem_solving/gsm8k_prepended_8shot.jsonl
-  num_fewshot: [0]
-  icl_task_type: question_answering
-  cot_delimiter: 'The answer is '
-  continuation_delimiter: "\n\nA:"
-  question_prelimiter: ""
-  do_normalization: false
-  early_stopping_criteria:
-  - "\n\n"
-  - "Question:"
+icl_tasks: 'eval/yamls/tasks_v0.1.yaml'
+eval_gauntlet: 'eval/yamls/eval_gauntlet_v0.1.yaml'
diff --git a/scripts/eval/yamls/tasks_v0.1.yaml b/scripts/eval/yamls/tasks_v0.1.yaml
@@ -19,24 +19,6 @@ icl_tasks:
   cot_delimiter: ' #### '
   continuation_delimiter: "\nA: Let's think step by step. "
   question_prelimiter: "Q: "
--
-  label: math_complex
-  dataset_uri: eval/local_data/symbolic_problem_solving/math_complex_soln.jsonl
-  num_fewshot: [3]
-  icl_task_type: question_answering
-  cot_delimiter: ' #### '
-  continuation_delimiter: "\nA: Let's think step by step. "
-  question_prelimiter: "Q: "
-  has_categories: true
--
-  label: math_simple
-  dataset_uri: eval/local_data/symbolic_problem_solving/math_simple_soln.jsonl
-  num_fewshot: [3]
-  icl_task_type: question_answering
-  cot_delimiter: ' #### '
-  continuation_delimiter: "\nA: Let's think step by step. "
-  question_prelimiter: "Q: "
-  has_categories: true
 -
   label: agi_eval_sat_math
   dataset_uri: eval/local_data/symbolic_problem_solving/agi_eval_sat_math.jsonl

diff --git a/scripts/eval/yamls/tasks_v0.2.yaml b/scripts/eval/yamls/tasks_v0.2.yaml
@@ -13,12 +13,12 @@ icl_tasks:
   icl_task_type: question_answering
 -
   label: gsm8k
-  dataset_uri: eval/local_data/symbolic_problem_solving/gsm8k_prepended_8shot.jsonl
-  num_fewshot: [0]
+  dataset_uri: eval/local_data/symbolic_problem_solving/gsm8k.jsonl
+  num_fewshot: [8, 5]
   icl_task_type: question_answering
-  cot_delimiter: 'The answer is '
-  continuation_delimiter: "\n\nA:"
-  question_prelimiter: ""
+  cot_delimiter: ' #### '
+  continuation_delimiter: "\nA: Let's think step by step. "
+  question_prelimiter: "Q: "
 -
   label: agi_eval_sat_math
   dataset_uri: eval/local_data/symbolic_problem_solving/agi_eval_sat_math.jsonl