chore(api, tests): refactor api tests to centralize file pathing (#1165)

* add utility for centralizing all data path access within tests
* adds make target for running conformance tests
* fix some reference issues with conformance tests
* eliminate text_file_path from client
* change names of conformance tests for consistency
* add conformance test documentation to testing readme

tests/Makefile

@@ -74,3 +74,6 @@ test-api-unit:
 
 test-load:
 	python -m locust -f $$(pwd)/tests/load/loadtest.py --web-port 8089
+
+test-conformance:
+	PYTHONPATH=$$(pwd) pytest -vv -s tests/conformance

tests/README.md

@@ -107,3 +107,23 @@ python -m pytest tests/e2e/test_llama.py -v
 # Cleanup after yourself
 k3d cluster delete uds
 ```
+
+## Conformance Testing
+
+We include a set of conformance tests to verify our spec against OpenAI to guarantee interoperability with tools that support OpenAI's API (MatterMost, Continue.dev, etc.) and SDKs (Vercel, Azure, etc.). To run these tests the environment variables need to be set:
+
+```bash
+LEAPFROGAI_API_KEY="<api key>" # this can be created via the LeapfrogAI UI or Supabase
+LEAPFROGAI_API_URL="https://leapfrogai-api.uds.dev/openai/v1" # This is the default when using a UDS-bundle locally
+LEAPFROGAI_MODEL="vllm" # or whatever model you have installed
+OPENAI_API_KEY="<api key>" # you need a funded OpenAI account for this
+OPENAI_MODEL="gpt-4o-mini" # or whatever model you prefer
+```
+
+To run the tests, from the root directory of the LeapfrogAI project:
+
+```bash
+make install # to ensure all python dependencies are installed
+
+make test-conformance # runs the entire suite
+```

tests/conformance/test_completions.py

@@ -1,7 +1,7 @@
 import pytest
 from openai.types.beta.threads import Run, Message, TextContentBlock, Text
 
-from .utils import client_config_factory
+from tests.utils.client import client_config_factory
 
 
 def make_mock_message_object(role, message_text):
@@ -37,12 +37,12 @@ def make_mock_message_simple(role, message_text):
 def test_run_completion(client_name, test_messages):
     # Setup
     config = client_config_factory(client_name)
-    client = config["client"]
+    client = config.client
 
     assistant = client.beta.assistants.create(
         name="Test Assistant",
         instructions="You must provide a response based on the attached files.",
-        model=config["model"],
+        model=config.model,
     )
     thread = client.beta.threads.create()
 

tests/conformance/test_files.py

@@ -6,16 +6,17 @@
 )
 from openai.types.beta.vector_stores.vector_store_file import VectorStoreFile
 
-from ..utils.client import client_config_factory, text_file_path
+from tests.utils.client import client_config_factory
+from tests.utils.data_path import data_path, TXT_DATA_FILE
 
 
 @pytest.mark.parametrize("client_name", ["openai", "leapfrogai"])
 def test_file_upload(client_name):
     config = client_config_factory(client_name)
-    client = config.client  # shorthand
+    client = config.client
 
     vector_store = client.beta.vector_stores.create(name="Test data")
-    with open(text_file_path(), "rb") as file:
+    with open(data_path(TXT_DATA_FILE), "rb") as file:
         vector_store_file = client.beta.vector_stores.files.upload(
             vector_store_id=vector_store.id, file=file
         )
@@ -24,13 +25,14 @@ def test_file_upload(client_name):
     assert isinstance(vector_store_file, VectorStoreFile)
 
 
+@pytest.mark.xfail(reason="File Batch Upload is not yet implemented in LeapfrogAI")
 @pytest.mark.parametrize("client_name", ["openai", "leapfrogai"])
 def test_file_delete(client_name):
     config = client_config_factory(client_name)
     client = config.client
 
     vector_store = client.beta.vector_stores.create(name="Test data")
-    with open(text_file_path(), "rb") as file:
+    with open(data_path(TXT_DATA_FILE), "rb") as file:
         vector_store_file = client.beta.vector_stores.files.upload(
             vector_store_id=vector_store.id, file=file
         )

tests/conformance/test_messages.py

@@ -2,7 +2,7 @@
 
 from openai.types.beta.threads.message import Message
 
-from ..utils.client import client_config_factory
+from tests.utils.client import client_config_factory
 
 
 @pytest.mark.parametrize("client_name", ["openai", "leapfrogai"])

tests/conformance/test_conformance_runs.py → tests/conformance/test_runs.py

@@ -1,7 +1,7 @@
 import pytest
 from openai.types.beta.threads import Run, Message, TextContentBlock, Text
 
-from .utils import client_config_factory
+from tests.utils.client import client_config_factory
 
 
 def make_mock_message_object(role, message_text):
@@ -37,12 +37,12 @@ def make_mock_message_simple(role, message_text):
 def test_run_create(client_name, test_messages):
     # Setup
     config = client_config_factory(client_name)
-    client = config["client"]
+    client = config.client
 
     assistant = client.beta.assistants.create(
         name="Test Assistant",
         instructions="You must provide a response based on the attached files.",
-        model=config["model"],
+        model=config.model,
     )
     thread = client.beta.threads.create()
 

tests/conformance/test_conformance_threads.py → tests/conformance/test_threads.py

@@ -2,7 +2,7 @@
 from openai.types.beta.thread import Thread
 from openai.types.beta.threads import Message, TextContentBlock, Text
 
-from ..utils.client import client_config_factory
+from tests.utils.client import client_config_factory
 
 
 def make_mock_message_object(role, message_text):

tests/conformance/test_conformance_tools.py → tests/conformance/test_tools.py

@@ -7,12 +7,13 @@
 from openai.types.beta.threads.message import Message
 import re
 
-from ..utils.client import client_config_factory, text_file_path
+from tests.utils.client import client_config_factory
+from tests.utils.data_path import data_path, TXT_DATA_FILE
 
 
 def make_vector_store_with_file(client):
     vector_store = client.beta.vector_stores.create(name="Test data")
-    with open(text_file_path(), "rb") as file:
+    with open(data_path(TXT_DATA_FILE), "rb") as file:
         client.beta.vector_stores.files.upload(
             vector_store_id=vector_store.id, file=file
         )
@@ -46,7 +47,7 @@ def validate_annotation_format(annotation):
 @pytest.mark.parametrize("client_name", ["openai", "leapfrogai"])
 def test_thread_file_annotations(client_name):
     config = client_config_factory(client_name)
-    client = config.client  # shorthand
+    client = config.client
 
     vector_store = make_vector_store_with_file(client)
     assistant = make_test_assistant(client, config.model, vector_store.id)

tests/conformance/test_conformance_vectorstore.py → tests/conformance/test_vectorstore.py

@@ -3,13 +3,13 @@
 from openai.types.beta.vector_store import VectorStore
 from openai.types.beta.vector_store_deleted import VectorStoreDeleted
 
-from ..utils.client import client_config_factory
+from tests.utils.client import client_config_factory
 
 
 @pytest.mark.parametrize("client_name", ["openai", "leapfrogai"])
 def test_vector_store_create(client_name):
     config = client_config_factory(client_name)
-    client = config.client  # shorthand
+    client = config.client
 
     vector_store = client.beta.vector_stores.create(name="Test data")
 
@@ -19,7 +19,7 @@ def test_vector_store_create(client_name):
 @pytest.mark.parametrize("client_name", ["openai", "leapfrogai"])
 def test_vector_store_list(client_name):
     config = client_config_factory(client_name)
-    client = config.client  # shorthand
+    client = config.client
 
     client.beta.vector_stores.create(name="Test data")
 

tests/e2e/test_llm_generation.py

@@ -1,11 +1,11 @@
 import os
-from pathlib import Path
 from typing import Iterable
 import warnings
 
 import pytest
 from openai import InternalServerError, OpenAI
 from openai.types.chat import ChatCompletionMessageParam
+from tests.utils.data_path import data_path, WAV_FILE
 
 DEFAULT_LEAPFROGAI_MODEL = "llama-cpp-python"
 
@@ -72,7 +72,8 @@ def test_embeddings(client: OpenAI, model_name: str):
 def test_transcriptions(client: OpenAI, model_name: str):
     with pytest.raises(InternalServerError) as excinfo:
         client.audio.transcriptions.create(
-            model=model_name, file=Path("tests/data/0min12sec.wav")
+            model=model_name,
+            file=data_path(WAV_FILE),
         )
 
     assert str(excinfo.value) == "Internal Server Error"

tests/e2e/test_text_embeddings.py

@@ -1,7 +1,6 @@
-from pathlib import Path
-
 import pytest
 from openai import InternalServerError, OpenAI
+from tests.utils.data_path import data_path, WAV_FILE
 
 model_name = "text-embeddings"
 
@@ -41,6 +40,7 @@ def test_embeddings(client: OpenAI):
 def test_transcriptions(client: OpenAI):
     with pytest.raises(InternalServerError) as excinfo:
         client.audio.transcriptions.create(
-            model=model_name, file=Path("tests/data/0min12sec.wav")
+            model=model_name,
+            file=data_path(WAV_FILE),
         )
     assert str(excinfo.value) == "Internal Server Error"

tests/e2e/test_whisper.py

@@ -4,6 +4,7 @@
 import pytest
 from openai import InternalServerError, OpenAI
 import unicodedata
+from tests.utils.data_path import data_path, WAV_FILE, WAV_FILE_ARABIC
 
 
 def test_completions(client: OpenAI):
@@ -38,7 +39,7 @@ def test_embeddings(client: OpenAI):
 def test_transcriptions(client: OpenAI):
     transcription = client.audio.transcriptions.create(
         model="whisper",
-        file=Path("tests/data/0min12sec.wav"),
+        file=data_path(WAV_FILE),
         language="en",
         prompt="This is a test transcription.",
         response_format="json",
@@ -53,7 +54,7 @@ def test_transcriptions(client: OpenAI):
 def test_translations(client: OpenAI):
     translation = client.audio.translations.create(
         model="whisper",
-        file=Path("tests/data/arabic-audio.wav"),
+        file=data_path(WAV_FILE_ARABIC),
         prompt="This is a test translation.",
         response_format="json",
         temperature=0.0,
@@ -79,7 +80,7 @@ def test_non_english_transcription(client: OpenAI):
     # Arabic transcription
     arabic_transcription = client.audio.transcriptions.create(
         model="whisper",
-        file=Path("tests/data/arabic-audio.wav"),
+        file=data_path(WAV_FILE_ARABIC),
         response_format="json",
         temperature=0.5,
         timestamp_granularities=["word", "segment"],

tests/integration/api/test_assistants.py

@@ -21,6 +21,7 @@
     CreateAssistantRequest,
     ModifyAssistantRequest,
 )
+from tests.utils.data_path import data_path, TXT_FILE
 
 INSTRUCTOR_XL_EMBEDDING_SIZE: int = 768
 
@@ -92,9 +93,7 @@ class MissingEnvironmentVariable(Exception):
 def read_testfile():
     """Read the test file content."""
 
-    with open(
-        os.path.dirname(__file__) + "/../../../tests/data/test.txt", "rb"
-    ) as testfile:
+    with open(data_path(TXT_FILE), "rb") as testfile:
         testfile_content = testfile.read()
 
     return testfile_content
@@ -109,7 +108,7 @@ def create_file(read_testfile):  # pylint: disable=redefined-outer-name, unused-
 
     file_response = files_client.post(
         "/openai/v1/files",
-        files={"file": ("test.txt", read_testfile, "text/plain")},
+        files={"file": (TXT_FILE, read_testfile, "text/plain")},
         data={"purpose": "assistants"},
     )
 

tests/integration/api/test_files.py

@@ -8,6 +8,7 @@
 
 from leapfrogai_api.backend.rag.document_loader import load_file, split
 from leapfrogai_api.routers.openai.files import router
+from tests.utils.data_path import data_path, WAV_FILE, TXT_FILE, PPTX_FILE, XLSX_FILE
 
 file_response: Response
 testfile_content: bytes
@@ -34,7 +35,7 @@ class MissingEnvironmentVariable(Exception):
 def read_testfile():
     """Read the test file content."""
     global testfile_content  # pylint: disable=global-statement
-    with open(os.path.dirname(__file__) + "/../../data/test.txt", "rb") as testfile:
+    with open(data_path(TXT_FILE), "rb") as testfile:
         testfile_content = testfile.read()
 
 
@@ -46,7 +47,7 @@ def create_file(read_testfile):  # pylint: disable=redefined-outer-name, unused-
 
     file_response = client.post(
         "/openai/v1/files",
-        files={"file": ("test.txt", testfile_content, "text/plain")},
+        files={"file": (TXT_FILE, testfile_content, "text/plain")},
         data={"purpose": "assistants"},
     )
 
@@ -132,15 +133,11 @@ def test_get_nonexistent():
 def test_invalid_file_type():
     """Test creating uploading an invalid file type."""
 
-    file_path = "../../../tests/data/0min12sec.wav"
-    dir_path = os.path.dirname(os.path.realpath(__file__))
-    relative_file_path = os.path.join(dir_path, file_path)
-
     with pytest.raises(HTTPException) as exception:
-        with open(relative_file_path, "rb") as testfile:
+        with open(data_path(WAV_FILE), "rb") as testfile:
             _ = client.post(
                 "/openai/v1/files",
-                files={"file": ("0min12sec.wav", testfile, "audio/wav")},
+                files={"file": (WAV_FILE, testfile, "audio/wav")},
                 data={"purpose": "assistants"},
             )
             assert exception.status_code == status.HTTP_415_UNSUPPORTED_MEDIA_TYPE
@@ -149,16 +146,8 @@ def test_invalid_file_type():
 @pytest.mark.asyncio
 async def test_excel_file_handling():
     """Test handling of an Excel file including upload, retrieval, and deletion."""
-    # Path to the test Excel file
-    excel_file_path = os.path.join(os.path.dirname(__file__), "../../data/test.xlsx")
-
-    # Ensure the file exists
-    assert os.path.exists(
-        excel_file_path
-    ), f"Test Excel file not found at {excel_file_path}"
-
     # Test file loading and splitting
-    documents = await load_file(excel_file_path)
+    documents = await load_file(data_path(XLSX_FILE))
     assert len(documents) > 0, "No documents were loaded from the Excel file"
     assert documents[0].page_content, "The first document has no content"
 
@@ -167,12 +156,12 @@ async def test_excel_file_handling():
     assert split_documents[0].page_content, "The first split document has no content"
 
     # Test file upload via API
-    with open(excel_file_path, "rb") as excel_file:
+    with open(data_path(XLSX_FILE), "rb") as excel_file:
         response = client.post(
             "/openai/v1/files",
             files={
                 "file": (
-                    "test.xlsx",
+                    XLSX_FILE,
                     excel_file,
                     "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
                 )
@@ -228,16 +217,9 @@ async def test_excel_file_handling():
 @pytest.mark.asyncio
 async def test_powerpoint_file_handling():
     """Test handling of a PowerPoint file including upload, retrieval, and deletion."""
-    # Path to the test PowerPoint file
-    pptx_file_path = os.path.join(os.path.dirname(__file__), "../../data/test.pptx")
-
-    # Ensure the file exists
-    assert os.path.exists(
-        pptx_file_path
-    ), f"Test PowerPoint file not found at {pptx_file_path}"
 
     # Test file loading and splitting
-    documents = await load_file(pptx_file_path)
+    documents = await load_file(data_path(PPTX_FILE).__str__())
     assert len(documents) > 0, "No documents were loaded from the PowerPoint file"
     assert documents[0].page_content, "The first document has no content"
 
@@ -246,13 +228,13 @@ async def test_powerpoint_file_handling():
     assert split_documents[0].page_content, "The first split document has no content"
 
     # Test file upload via API
-    with open(pptx_file_path, "rb") as pptx_file:
+    with open(data_path(PPTX_FILE), "rb") as file:
         response = client.post(
             "/openai/v1/files",
             files={
                 "file": (
-                    "test.pptx",
-                    pptx_file,
+                    PPTX_FILE,
+                    file,
                     "application/vnd.openxmlformats-officedocument.presentationml.presentation",
                 )
             },