fixes

scripts/eval/yamls/test_eval.yaml

@@ -25,7 +25,7 @@ device_eval_batch_size: 4
 icl_subset_num_batches: 1
 icl_tasks:
 - label: lambada_openai
-  dataset_uri: scripts/eval/local_data/language_understanding/lambada_openai.jsonl
+  dataset_uri: eval/local_data/language_understanding/lambada_openai.jsonl
   num_fewshot: [0]
   icl_task_type: language_modeling
 eval_gauntlet:

tests/callbacks/test_eval_gauntlet_callback.py

@@ -1,7 +1,6 @@
 # Copyright 2022 MosaicML LLM Foundry authors
 # SPDX-License-Identifier: Apache-2.0
 
-import os
 from typing import Dict, List, Optional
 
 import omegaconf as om
@@ -15,17 +14,6 @@
 from llmfoundry.utils.builders import build_icl_data_and_gauntlet
 
 
-@pytest.fixture(autouse=True)
-def set_correct_cwd():
-    if not os.getcwd().endswith('llm-foundry/scripts'):
-        os.chdir('scripts')
-
-    yield
-
-    if os.getcwd().endswith('llm-foundry/scripts'):
-        os.chdir('..')
-
-
 class MockState(State):
 
     def __init__(self, logger_keys: List[str], accuracy: float = 0.25) -> None:

tests/data_utils.py

@@ -1,6 +1,7 @@
 # Copyright 2022 MosaicML LLM Foundry authors
 # SPDX-License-Identifier: Apache-2.0
 
+import json
 import os
 import pathlib
 import shutil
@@ -113,10 +114,14 @@ def create_arxiv_dataset(path: pathlib.Path) -> str:
     arxiv_dir = os.path.join(path, f'my-copy-arxiv')
     downloaded_split = 'train'
 
+    arxiv_path = 'data_prep/example_data/arxiv.jsonl'
+    if not os.getcwd().endswith('scripts'):
+        arxiv_path = os.path.join('scripts', arxiv_path)
+
     main_json(
         Namespace(
             **{
-                'path': 'data_prep/example_data/arxiv.jsonl',
+                'path': arxiv_path,
                 'out_root': arxiv_dir,
                 'compression': None,
                 'split': downloaded_split,

tests/scripts/eval/test_eval.py

@@ -18,6 +18,17 @@
                               gpt_tiny_cfg)
 
 
+@pytest.fixture(autouse=True)
+def set_correct_cwd():
+    if not os.getcwd().endswith('llm-foundry/scripts'):
+        os.chdir('scripts')
+
+    yield
+
+    if os.getcwd().endswith('llm-foundry/scripts'):
+        os.chdir('..')
+
+
 @pytest.fixture
 def eval_cfg(foundry_dir: str) -> Union[om.ListConfig, om.DictConfig]:
     yaml_path = os.path.join(foundry_dir, 'scripts/eval/yamls/test_eval.yaml')

tests/scripts/train/test_train.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 import copy
 import pathlib
-from typing import Any, Optional
+from typing import Optional
 
 import pytest
 from composer.loggers import InMemoryLogger
@@ -18,8 +18,7 @@
 @pytest.mark.parametrize('averages', [{
     'core_average': ['language_understanding_lite']
 }, None])
-def test_train_gauntlet(averages: Optional[dict], set_correct_cwd: Any,
-                        tmp_path: pathlib.Path):
+def test_train_gauntlet(averages: Optional[dict], tmp_path: pathlib.Path):
     """Test training run with a small dataset."""
     dataset_name = create_c4_dataset_xxsmall(tmp_path)
     test_cfg = gpt_tiny_cfg(dataset_name, 'cpu')
@@ -28,7 +27,7 @@ def test_train_gauntlet(averages: Optional[dict], set_correct_cwd: Any,
             'label':
                 'lambada_openai',
             'dataset_uri':
-                'eval/local_data/language_understanding/lambada_openai_small.jsonl',
+                'scripts/eval/local_data/language_understanding/lambada_openai_small.jsonl',
             'num_fewshot': [0],
             'icl_task_type':
                 'language_modeling'
@@ -98,7 +97,7 @@ def test_train_gauntlet(averages: Optional[dict], set_correct_cwd: Any,
         -1][-1] == 0
 
 
-def test_train_multi_eval(set_correct_cwd: Any, tmp_path: pathlib.Path):
+def test_train_multi_eval(tmp_path: pathlib.Path):
     """Test training run with multiple eval datasets."""
     c4_dataset_name = create_c4_dataset_xxsmall(tmp_path)
     test_cfg = gpt_tiny_cfg(c4_dataset_name, 'cpu')