Add new ICL kwargs in eval.py and long_context yamls

llmfoundry/utils/builders.py

@@ -52,6 +52,7 @@ def build_evaluators(
     tokenizer: PreTrainedTokenizerBase,
     device_eval_batch_size: int,
     icl_seq_len: int,
+    fewshot_random_seed: Optional[int],
     icl_subset_num_batches: Optional[int],
 ) -> Tuple[List[Evaluator], List[str], Optional[EvalGauntlet]]:
 
@@ -72,6 +73,7 @@ def build_evaluators(
             tokenizer,
             device_eval_batch_size,
             icl_seq_len,
+            fewshot_random_seed,
             icl_subset_num_batches,
         )
         evaluators.extend(icl_evaluators)
@@ -129,13 +131,15 @@ def build_icl_data_and_gauntlet(
     tokenizer: PreTrainedTokenizerBase,
     device_eval_batch_size: int,
     icl_seq_len: int,
+    fewshot_random_seed: Optional[int],
     icl_subset_num_batches: Optional[int] = None
 ) -> Tuple[List[Evaluator], List[str], Optional[EvalGauntlet]]:
     icl_evaluators, logger_keys = build_icl_evaluators(
         icl_tasks_config,
         tokenizer,
         icl_seq_len,
         device_eval_batch_size,
+        fewshot_random_seed=fewshot_random_seed,
         icl_subset_num_batches=icl_subset_num_batches)
     eval_gauntlet_cb = None
     if eval_gauntlet_config is not None:
@@ -427,6 +431,7 @@ def build_icl_evaluators(
     default_max_seq_len: int,
     default_batch_size: int,
     destination_dir: Optional[str] = None,
+    fewshot_random_seed: Optional[int] = None,
     icl_subset_num_batches: Optional[int] = None,
 ) -> Tuple[List[Evaluator], List[str]]:
     if destination_dir is None:
@@ -485,6 +490,7 @@ def _validate_cfg(icl_cfg: DictConfig):
         if 'num_beams' not in icl_cfg:
             icl_cfg.num_beams = 20
 
+
     for icl_cfg in icl_tasks_list:
         assert isinstance(icl_cfg, DictConfig)
         _validate_cfg(icl_cfg)
@@ -502,6 +508,9 @@ def _validate_cfg(icl_cfg: DictConfig):
                 os.remove(destination_path)
             dist.barrier()
 
+            hf_parsing_map = icl_cfg.get('hf_parsing_map', {})
+            hf_loading_vars = icl_cfg.get('hf_loading_vars', {}) 
+
             dataloaders = get_icl_task_dataloader(
                 icl_cfg.icl_task_type,
                 icl_cfg.dataset_uri,
@@ -512,13 +521,20 @@ def _validate_cfg(icl_cfg: DictConfig):
                 num_fewshot=num_fewshot,
                 prompt_string=icl_cfg.prompt_string,
                 example_delimiter=icl_cfg.example_delimiter,
+                hf_loading_vars=hf_loading_vars, 
+                hf_parsing_map=hf_parsing_map, 
                 continuation_delimiter=icl_cfg.continuation_delimiter,
                 question_prelimiter=icl_cfg.get('question_prelimiter', ''),
                 destination_path=destination_path,
+                fewshot_random_seed=fewshot_random_seed,
                 pass_at_k=icl_cfg.pass_at_k,
                 generations_per_sample=icl_cfg.num_beams,
                 has_categories=icl_cfg.get('has_categories', False),
-                cot_delimiter=icl_cfg.get('cot_delimiter', ''))
+                cot_delimiter=icl_cfg.get('cot_delimiter', ''),
+                generation_kwargs=icl_cfg.get('generation_kwargs', {}),
+                early_stopping_criteria=icl_cfg.get('early_stopping_criteria'),
+                do_normalization=icl_cfg.get('do_normalization', True),
+            )
             if hasattr(
                     icl_cfg,
                     'has_categories') and icl_cfg.has_categories and isinstance(

scripts/eval/eval.py

@@ -118,6 +118,7 @@ def evaluate_model(
     python_log_level: Optional[str],
     precision: str,
     eval_gauntlet_df: Optional[pd.DataFrame],
+    fewshot_random_seed: Optional[int],
     eval_subset_num_batches: int,
     icl_subset_num_batches: Optional[int],
     metadata: Optional[Dict[str, str]],
@@ -141,6 +142,7 @@ def evaluate_model(
         tokenizer=tokenizer,
         device_eval_batch_size=device_eval_batch_size,
         icl_seq_len=max_seq_len,
+        fewshot_random_seed=fewshot_random_seed,
         icl_subset_num_batches=icl_subset_num_batches,
     )
 
@@ -301,6 +303,10 @@ def main(cfg: DictConfig) -> Tuple[List[Trainer], pd.DataFrame]:
                                              'loggers',
                                              must_exist=False,
                                              default_value={})
+    fewshot_random_seed: int = pop_config(cfg,
+                                          'fewshot_random_seed',
+                                          must_exist=False,
+                                          default_value=1234)
     eval_subset_num_batches: int = pop_config(cfg,
                                               'eval_subset_num_batches',
                                               must_exist=False,
@@ -318,6 +324,7 @@ def main(cfg: DictConfig) -> Tuple[List[Trainer], pd.DataFrame]:
                                          'log_config',
                                          must_exist=False,
                                          default_value=True)
+
 
     # Pop out interpolation variables.
     pop_config(cfg, 'model_name_or_path', must_exist=False, default_value=None)
@@ -362,6 +369,7 @@ def main(cfg: DictConfig) -> Tuple[List[Trainer], pd.DataFrame]:
              python_log_level=python_log_level,
              precision=precision,
              eval_gauntlet_df=eval_gauntlet_df,
+             fewshot_random_seed=fewshot_random_seed,
              eval_subset_num_batches=eval_subset_num_batches,
              icl_subset_num_batches=icl_subset_num_batches,
              metadata=metadata,

scripts/eval/yamls/eval_gauntlet_8k_length.yaml

@@ -0,0 +1,74 @@
+eval_gauntlet:
+  weighting: EQUAL
+  subtract_random_baseline: true
+  rescale_accuracy: true
+  categories:
+  - name: 2k
+    benchmarks:
+    - name: hotpotqa_beginning_2k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: hotpotqa_middle_2k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: hotpotqa_end_2k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_beginning_2k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_middle_2k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_end_2k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: wikiqa_2k
+      num_fewshot: 0
+      random_baseline: 0
+  - name: 4k 
+    benchmarks:
+    - name: hotpotqa_beginning_4k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: hotpotqa_middle_4k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: hotpotqa_end_4k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_beginning_4k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_middle_4k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_end_4k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: wikiqa_4k
+      num_fewshot: 0
+      random_baseline: 0
+  - name: 8k 
+    benchmarks:
+    - name: hotpotqa_beginning_8k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: hotpotqa_middle_8k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: hotpotqa_end_8k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_beginning_8k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_middle_8k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_end_8k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: wikiqa_8k
+      num_fewshot: 0
+      random_baseline: 0

scripts/eval/yamls/eval_gauntlet_8k_section.yaml

@@ -0,0 +1,76 @@
+eval_gauntlet:
+  weighting: EQUAL
+  subtract_random_baseline: true
+  rescale_accuracy: true
+  categories:
+  - name: beginning 
+    benchmarks:
+    - name: hotpotqa_beginning_2k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_beginning_2k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: hotpotqa_beginning_4k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_beginning_4k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: hotpotqa_beginning_8k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_beginning_8k
+      num_fewshot: 0
+      random_baseline: 0
+  - name: middle
+    benchmarks:
+    - name: hotpotqa_middle_2k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_middle_2k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: hotpotqa_middle_4k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_middle_4k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: hotpotqa_middle_8k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_middle_8k
+      num_fewshot: 0
+      random_baseline: 0
+  - name: end
+    benchmarks:
+    - name: hotpotqa_end_2k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_end_2k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: hotpotqa_end_4k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_end_4k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: hotpotqa_end_8k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: kv_pairs_end_8k
+      num_fewshot: 0
+      random_baseline: 0
+  - name: full
+    benchmarks:
+    - name: wikiqa_2k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: wikiqa_4k
+      num_fewshot: 0
+      random_baseline: 0
+    - name: wikiqa_8k
+      num_fewshot: 0
+      random_baseline: 0

scripts/eval/yamls/hf_eval.yaml

@@ -12,7 +12,7 @@ models:
   model:
     name: hf_causal_lm
     pretrained_model_name_or_path: ${model_name_or_path}
-    init_device: mixed
+    init_device: cpu
     pretrained: true
   tokenizer:
     name: ${model_name_or_path}
@@ -37,11 +37,11 @@ models:
 device_eval_batch_size: 4
 
 # FSDP config for model sharding
-# fsdp_config:
-#   sharding_strategy: FULL_SHARD
-#   mixed_precision: FULL
-#   forward_prefetch: True
-#   limit_all_gathers: True
+fsdp_config:
+  sharding_strategy: FULL_SHARD
+  mixed_precision: FULL
+  forward_prefetch: True
+  limit_all_gathers: True
 
 icl_tasks: 'eval/yamls/tasks_v0.1.yaml'
 eval_gauntlet: 'eval/yamls/eval_gauntlet_v0.1.yaml'

scripts/eval/yamls/long_context_eval_8k.yaml

@@ -0,0 +1,27 @@
+max_seq_len: 8196 
+seed: 1
+precision: amp_bf16
+
+models:
+-
+  model_name: EleutherAI/gpt-neo-125m
+  model:
+    name: hf_causal_lm
+    pretrained_model_name_or_path: EleutherAI/gpt-neo-125m
+    init_device: cpu
+    pretrained: true
+  tokenizer:
+    name: EleutherAI/gpt-neo-125m
+    kwargs:
+      model_max_length: ${max_seq_len}
+
+device_eval_batch_size: 1 
+icl_subset_num_batches: 2
+
+# FSDP config for model sharding
+fsdp_config:
+  sharding_strategy: FULL_SHARD
+  mixed_precision: FULL
+
+icl_tasks: 'eval/yamls/long_context_tasks.yaml'
+eval_gauntlet: 'eval/yamls/eval_gauntlet_8k_section.yaml'