diff --git a/docs/community/integrations/uptrain.md b/docs/community/integrations/uptrain.md index c91ed8ba48d5b..da808ae3db271 100644 --- a/docs/community/integrations/uptrain.md +++ b/docs/community/integrations/uptrain.md @@ -461,12 +461,13 @@ pip install uptrain llama_index ## Import required libraries ```python +import httpx import os import openai import pandas as pd -from llama_index.core import VectorStoreIndex, SimpleDirectoryReader -from uptrain import Evals, EvalLlamaIndex, Settings +from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, Settings +from uptrain import Evals, EvalLlamaIndex, Settings as UpTrainSettings ``` ## Create the dataset folder for the query engine @@ -480,8 +481,6 @@ if not os.path.exists("nyc_wikipedia"): dataset_path = os.path.join("./nyc_wikipedia", "nyc_text.txt") if not os.path.exists(dataset_path): - import httpx - r = httpx.get(url) with open(dataset_path, "wb") as f: f.write(r.content) @@ -517,8 +516,6 @@ openai.api_key = "sk-************************" # your OpenAI API key Let's create a vector store index using LLamaIndex and then use that as a query engine to retrieve relevant sections from the documentation. ```python -from llama_index.core import Settings - Settings.chunk_size = 512 documents = SimpleDirectoryReader("./nyc_wikipedia/").load_data() @@ -533,7 +530,7 @@ query_engine = vector_index.as_query_engine() # Alternative 1: Evaluate using UpTrain's Open-Source Software (OSS) ```python -settings = Settings( +settings = UpTrainSettings( openai_api_key=openai.api_key, ) ``` @@ -583,7 +580,7 @@ You can create a free UpTrain account [here](https://uptrain.ai/) and get free t UPTRAIN_API_KEY = "up-**********************" # your UpTrain API key # We use `uptrain_access_token` parameter instead of 'openai_api_key' in settings in this case -settings = Settings( +settings = UpTrainSettings( uptrain_access_token=UPTRAIN_API_KEY, ) ``` diff --git a/docs/examples/evaluation/UpTrain.ipynb b/docs/examples/evaluation/UpTrain.ipynb index eacb48b43d2f8..ba718a41234dc 100644 --- a/docs/examples/evaluation/UpTrain.ipynb +++ b/docs/examples/evaluation/UpTrain.ipynb @@ -49,12 +49,25 @@ "id": "0b101745", "metadata": {}, "source": [ - "## Install UpTrain and LlamaIndex\n", - "\n", - "\n", - "```bash\n", - "pip install uptrain llama_index\n", - "```" + "## Install UpTrain and LlamaIndex" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a6734276", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -q uptrain llama-index" ] }, { @@ -70,14 +83,24 @@ "execution_count": null, "id": "6c6e7a1d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/dhruvchawla/Work/llama_index/venv/lib/python3.11/site-packages/lazy_loader/__init__.py:185: RuntimeWarning: subpackages can technically be lazily loaded, but it causes the package to be eagerly loaded even if it is already lazily loaded.So, you probably shouldn't use subpackages with this lazy feature.\n", + " warnings.warn(msg, RuntimeWarning)\n" + ] + } + ], "source": [ + "import httpx\n", "import os\n", "import openai\n", "import pandas as pd\n", "\n", - "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n", - "from uptrain import Evals, EvalLlamaIndex, Settings" + "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, Settings\n", + "from uptrain import Evals, EvalLlamaIndex, Settings as UpTrainSettings" ] }, { @@ -103,8 +126,6 @@ "dataset_path = os.path.join(\"./nyc_wikipedia\", \"nyc_text.txt\")\n", "\n", "if not os.path.exists(dataset_path):\n", - " import httpx\n", - "\n", " r = httpx.get(url)\n", " with open(dataset_path, \"wb\") as f:\n", " f.write(r.content)" @@ -176,8 +197,6 @@ "metadata": {}, "outputs": [], "source": [ - "from llama_index.core import Settings\n", - "\n", "Settings.chunk_size = 512\n", "\n", "documents = SimpleDirectoryReader(\"./nyc_wikipedia/\").load_data()\n", @@ -204,7 +223,7 @@ "metadata": {}, "outputs": [], "source": [ - "settings = Settings(\n", + "settings = UpTrainSettings(\n", " openai_api_key=openai.api_key,\n", ")" ] @@ -306,101 +325,101 @@ " 0\n", " What is the population of New York City?\n", " The population of New York City is 8,804,190 a...\n", - " New York, often called New York City or NYC, i...\n", - " 1.0\n", - " The question is asking for the population of N...\n", - " 1.0\n", - " The question asks for the population of New Yo...\n", + " === Population density ===\\n\\nIn 2020, the cit...\n", + " None\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 1\n", " What is the area of New York City?\n", " New York City has a total area of 468.484 squa...\n", - " New York, often called New York City or NYC, i...\n", - " 1.0\n", - " Step 1: The question asks for the area of New ...\n", - " 1.0\n", - " The question asks for the area of New York Cit...\n", + " Some of the natural relief in topography has b...\n", + " None\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 2\n", " What is the largest borough in New York City?\n", " Queens is the largest borough in New York City.\n", " ==== Brooklyn ====\\nBrooklyn (Kings County), o...\n", - " 1.0\n", - " The question is asking for the largest borough...\n", - " 1.0\n", - " The question asks for the largest borough in N...\n", + " None\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 3\n", " What is the average temperature in New York City?\n", - " The average temperature in New York City is 57...\n", + " The average temperature in New York City is 33...\n", " Similarly, readings of 0 °F (−18 °C) are also ...\n", - " 0.5\n", - " The question is asking for the average tempera...\n", - " 1.0\n", - " The question asks for the average temperature ...\n", + " None\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 4\n", " What is the main airport in New York City?\n", - " The main airport in New York City is John F. K...\n", + " John F. Kennedy International Airport\n", " along the Northeast Corridor, and long-distanc...\n", - " 1.0\n", - " The question is asking for the main airport in...\n", - " 1.0\n", - " The question asks for the main airport in New ...\n", + " None\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 5\n", " What is the famous landmark in New York City?\n", - " The famous landmark in New York City is the Em...\n", - " A record 66.6 million tourists visited New Yor...\n", - " 1.0\n", - " The question is asking for the famous landmark...\n", - " 1.0\n", - " The question asks for the famous landmark in N...\n", + " The famous landmark in New York City is the St...\n", + " The settlement was named New Amsterdam (Dutch:...\n", + " None\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 6\n", " What is the official language of New York City?\n", - " The official language of New York City is not ...\n", + " As many as 800 languages are spoken in New Yor...\n", " === Accent and dialect ===\\n\\nThe New York are...\n", - " 0.0\n", - " The question is asking for the official langua...\n", - " 1.0\n", - " The question asks for the official language of...\n", + " None\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 7\n", " What is the currency used in New York City?\n", - " The currency used in New York City is the Unit...\n", + " The currency used in New York City is the US D...\n", " === Real estate ===\\n\\nReal estate is a major ...\n", - " 0.0\n", - " The question is asking for the currency used i...\n", - " 1.0\n", - " The question asks specifically for the currenc...\n", + " None\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 8\n", " What is the time zone of New York City?\n", " Eastern Standard Time (EST)\n", " According to the New York City Comptroller, wo...\n", - " 0.0\n", - " The question is \"What is the time zone of New ...\n", - " 1.0\n", - " The question asks for the time zone of New Yor...\n", + " None\n", + " None\n", + " None\n", + " None\n", " \n", " \n", " 9\n", " What is the famous sports team in New York City?\n", " The famous sports team in New York City is the...\n", - " ==== Baseball ====\\nNew York has been describe...\n", - " 0.5\n", - " The question is asking for the famous sports t...\n", - " 1.0\n", - " The question asks for the famous sports team i...\n", + " ==== Soccer ====\\nIn soccer, New York City is ...\n", + " None\n", + " None\n", + " None\n", + " None\n", " \n", " \n", "\n", @@ -423,61 +442,49 @@ "0 The population of New York City is 8,804,190 a... \n", "1 New York City has a total area of 468.484 squa... \n", "2 Queens is the largest borough in New York City. \n", - "3 The average temperature in New York City is 57... \n", - "4 The main airport in New York City is John F. K... \n", - "5 The famous landmark in New York City is the Em... \n", - "6 The official language of New York City is not ... \n", - "7 The currency used in New York City is the Unit... \n", + "3 The average temperature in New York City is 33... \n", + "4 John F. Kennedy International Airport \n", + "5 The famous landmark in New York City is the St... \n", + "6 As many as 800 languages are spoken in New Yor... \n", + "7 The currency used in New York City is the US D... \n", "8 Eastern Standard Time (EST) \n", "9 The famous sports team in New York City is the... \n", "\n", - " context score_context_relevance \\\n", - "0 New York, often called New York City or NYC, i... 1.0 \n", - "1 New York, often called New York City or NYC, i... 1.0 \n", - "2 ==== Brooklyn ====\\nBrooklyn (Kings County), o... 1.0 \n", - "3 Similarly, readings of 0 °F (−18 °C) are also ... 0.5 \n", - "4 along the Northeast Corridor, and long-distanc... 1.0 \n", - "5 A record 66.6 million tourists visited New Yor... 1.0 \n", - "6 === Accent and dialect ===\\n\\nThe New York are... 0.0 \n", - "7 === Real estate ===\\n\\nReal estate is a major ... 0.0 \n", - "8 According to the New York City Comptroller, wo... 0.0 \n", - "9 ==== Baseball ====\\nNew York has been describe... 0.5 \n", - "\n", - " explanation_context_relevance \\\n", - "0 The question is asking for the population of N... \n", - "1 Step 1: The question asks for the area of New ... \n", - "2 The question is asking for the largest borough... \n", - "3 The question is asking for the average tempera... \n", - "4 The question is asking for the main airport in... \n", - "5 The question is asking for the famous landmark... \n", - "6 The question is asking for the official langua... \n", - "7 The question is asking for the currency used i... \n", - "8 The question is \"What is the time zone of New ... \n", - "9 The question is asking for the famous sports t... \n", + " context score_context_relevance \\\n", + "0 === Population density ===\\n\\nIn 2020, the cit... None \n", + "1 Some of the natural relief in topography has b... None \n", + "2 ==== Brooklyn ====\\nBrooklyn (Kings County), o... None \n", + "3 Similarly, readings of 0 °F (−18 °C) are also ... None \n", + "4 along the Northeast Corridor, and long-distanc... None \n", + "5 The settlement was named New Amsterdam (Dutch:... None \n", + "6 === Accent and dialect ===\\n\\nThe New York are... None \n", + "7 === Real estate ===\\n\\nReal estate is a major ... None \n", + "8 According to the New York City Comptroller, wo... None \n", + "9 ==== Soccer ====\\nIn soccer, New York City is ... None \n", "\n", - " score_response_conciseness \\\n", - "0 1.0 \n", - "1 1.0 \n", - "2 1.0 \n", - "3 1.0 \n", - "4 1.0 \n", - "5 1.0 \n", - "6 1.0 \n", - "7 1.0 \n", - "8 1.0 \n", - "9 1.0 \n", + " explanation_context_relevance score_response_conciseness \\\n", + "0 None None \n", + "1 None None \n", + "2 None None \n", + "3 None None \n", + "4 None None \n", + "5 None None \n", + "6 None None \n", + "7 None None \n", + "8 None None \n", + "9 None None \n", "\n", - " explanation_response_conciseness \n", - "0 The question asks for the population of New Yo... \n", - "1 The question asks for the area of New York Cit... \n", - "2 The question asks for the largest borough in N... \n", - "3 The question asks for the average temperature ... \n", - "4 The question asks for the main airport in New ... \n", - "5 The question asks for the famous landmark in N... \n", - "6 The question asks for the official language of... \n", - "7 The question asks specifically for the currenc... \n", - "8 The question asks for the time zone of New Yor... \n", - "9 The question asks for the famous sports team i... " + " explanation_response_conciseness \n", + "0 None \n", + "1 None \n", + "2 None \n", + "3 None \n", + "4 None \n", + "5 None \n", + "6 None \n", + "7 None \n", + "8 None \n", + "9 None " ] }, "execution_count": null, @@ -530,7 +537,7 @@ "UPTRAIN_API_KEY = \"up-**********************\" # your UpTrain API key\n", "\n", "# We use `uptrain_access_token` parameter instead of 'openai_api_key' in settings in this case\n", - "settings = Settings(\n", + "settings = UpTrainSettings(\n", " uptrain_access_token=UPTRAIN_API_KEY,\n", ")" ] diff --git a/llama-index-integrations/callbacks/llama-index-callbacks-uptrain/pyproject.toml b/llama-index-integrations/callbacks/llama-index-callbacks-uptrain/pyproject.toml index 70b4ebe717f0a..d8b310ef645c0 100644 --- a/llama-index-integrations/callbacks/llama-index-callbacks-uptrain/pyproject.toml +++ b/llama-index-integrations/callbacks/llama-index-callbacks-uptrain/pyproject.toml @@ -21,18 +21,18 @@ ignore_missing_imports = true python_version = "3.8" [tool.poetry] -authors = ["Your Name "] +authors = ["Dhruv Chawla "] description = "llama-index callbacks uptrain integration" exclude = ["**/BUILD"] license = "MIT" name = "llama-index-callbacks-uptrain" readme = "README.md" -version = "0.1.1" +version = "0.1.2" [tool.poetry.dependencies] python = ">=3.8.1,<4.0" llama-index-core = ">=0.10.0" -uptrain = ">=0.6.5" +uptrain = ">=0.6.6" [tool.poetry.group.dev.dependencies] ipython = "8.10.0" diff --git a/llama-index-legacy/llama_index/legacy/callbacks/__init__.py b/llama-index-legacy/llama_index/legacy/callbacks/__init__.py index d419baa947969..097353e3bd341 100644 --- a/llama-index-legacy/llama_index/legacy/callbacks/__init__.py +++ b/llama-index-legacy/llama_index/legacy/callbacks/__init__.py @@ -5,7 +5,6 @@ from .open_inference_callback import OpenInferenceCallbackHandler from .schema import CBEvent, CBEventType, EventPayload from .token_counting import TokenCountingHandler -from .uptrain_callback import UpTrainCallbackHandler from .utils import trace_method from .wandb_callback import WandbCallbackHandler @@ -22,5 +21,4 @@ "OpenAIFineTuningHandler", "GradientAIFineTuningHandler", "trace_method", - "UpTrainCallbackHandler", ] diff --git a/llama-index-legacy/llama_index/legacy/callbacks/global_handlers.py b/llama-index-legacy/llama_index/legacy/callbacks/global_handlers.py index c52ed3ed354f1..f191de2181bda 100644 --- a/llama-index-legacy/llama_index/legacy/callbacks/global_handlers.py +++ b/llama-index-legacy/llama_index/legacy/callbacks/global_handlers.py @@ -14,7 +14,6 @@ ) from llama_index.legacy.callbacks.promptlayer_handler import PromptLayerHandler from llama_index.legacy.callbacks.simple_llm_handler import SimpleLLMHandler -from llama_index.legacy.callbacks.uptrain_callback import UpTrainCallbackHandler from llama_index.legacy.callbacks.wandb_callback import WandbCallbackHandler @@ -41,8 +40,6 @@ def create_global_handler(eval_mode: str, **eval_params: Any) -> BaseCallbackHan handler = deepeval_callback_handler(**eval_params) elif eval_mode == "simple": handler = SimpleLLMHandler(**eval_params) - elif eval_mode == "uptrain": - handler = UpTrainCallbackHandler(**eval_params) elif eval_mode == "argilla": handler = argilla_callback_handler(**eval_params) else: