revert

llmfoundry/models/inference_api_wrapper/__init__.py

@@ -1,7 +1,7 @@
 # Copyright 2022 MosaicML LLM Foundry authors
 # SPDX-License-Identifier: Apache-2.0
 
-from llmfoundry.models.inference_api_wrapper.fmapi_lm import (
+from llmfoundry.models.inference_api_wrapper.fmapi import (
     FMAPICasualLMEvalWrapper, FMAPIChatAPIEvalWrapper)
 from llmfoundry.models.inference_api_wrapper.interface import \
     InferenceAPIEvalWrapper

llmfoundry/models/inference_api_wrapper/fmapi.py

@@ -0,0 +1,70 @@
+# Copyright 2022 MosaicML LLM Foundry authors
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import logging
+from typing import Dict
+import time
+import requests
+from transformers import AutoTokenizer
+
+from llmfoundry.models.inference_api_wrapper.openai_causal_lm import (
+    OpenAICausalLMEvalWrapper, OpenAIChatAPIEvalWrapper)
+
+__all__ = [
+    'FMAPICasualLMEvalWrapper',
+    'FMAPIChatAPIEvalWrapper',
+]
+
+log = logging.getLogger(__name__)
+
+
+def block_until_ready(base_url: str):
+    """Block until the endpoint is ready."""
+
+    sleep_s = 5
+    remaining_s = 5 * 50 # At max, wait 5 minutes
+
+    ping_url = f'{base_url}/ping'
+
+    while True:
+        try:
+            requests.get(ping_url)
+            break
+        except requests.exceptions.ConnectionError:
+            log.debug(f'Endpoint {ping_url} not ready yet. Sleeping {sleep_s} seconds')
+            time.sleep(sleep_s)
+            remaining_s -= sleep_s
+        else:
+            log.info(f'Endpoint {ping_url} is ready')
+            break
+
+        if remaining_s <= 0:
+            raise TimeoutError(f'Endpoint {ping_url} never became ready, exiting')
+
+class FMAPIEvalInterface:
+
+    def __init__(self, model_cfg: Dict, tokenizer: AutoTokenizer):
+        is_local = model_cfg.get('local', False)
+        if is_local:
+            base_url = os.environ.get('MOSAICML_MODEL_ENDPOINT',
+                                'http://0.0.0.0:8080/v2')
+        elif 'base_url' in model_cfg:
+            base_url = model_cfg['base_url']
+        else:
+            raise ValueError('Must specify base_url in model_cfg for FMAPIsEvalWrapper')
+
+        model_cfg['base_url'] = base_url
+
+        if is_local:
+            block_until_ready(base_url)
+
+        super().__init__(model_cfg, tokenizer)
+
+
+class FMAPICasualLMEvalWrapper(FMAPIEvalInterface, OpenAICausalLMEvalWrapper):
+    """Databricks Foundational Model API wrapper for causal LM models."""
+
+
+class FMAPIChatAPIEvalWrapper(FMAPIEvalInterface, OpenAIChatAPIEvalWrapper):
+    """Databricks Foundational Model API wrapper for chat models."""

scripts/eval/yamls/openai_eval.yaml

@@ -3,19 +3,22 @@ max_seq_len: 1024
 device_eval_batch_size: 4
 models:
 -
-  model_name: mpt-example # required for some reason
+  model_name: openai/gpt-3.5-turbo
   model:
-    name: fmapi_chat
-    local: true
-
+    name: openai_chat
+    version: gpt-3.5-turbo
   tokenizer:
     name: tiktoken
     kwargs:
       model_name: gpt-3.5-turbo
-
-icl_tasks:
 -
-  label: bigbench_simple_arithmetic
-  dataset_uri: eval/local_data/symbolic_problem_solving/bigbench_simple_arithmetic.jsonl
-  num_fewshot: [2] # [10]
-  icl_task_type: language_modeling
+  model_name: openai/davinci
+  model:
+    name: openai_causal_lm
+    version: davinci
+  tokenizer:
+    name: tiktoken
+    kwargs:
+      model_name: davinci
+
+icl_tasks: 'eval/yamls/lm_tasks_v0.2.yaml'