KBGoldenEvals.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "38c66111-813c-411f-9e75-3c8410b9148e",
   "metadata": {},
   "source": [
    "# Golden Evaluations 🌟 with Bedrock Knowlage Bases\n",
    "This notebook takes the `golden-dataset.csv` as an input and uses the Ragas evaluation framework to calculate some standard metrics that help us quantify the performance of our RAG system. You can read more about these metrics here: https://docs.ragas.io/en/stable/concepts/metrics/index.html#\n",
    "\n",
    "The `golden-dataset.csv` should have two columns, `question` and `ground_truth`\n",
    "\n",
    "| question                | ground_truth   |\n",
    "|------------------------:|---------------:|\n",
    "| What is 2 + 2?          |              4 |\n",
    "| Capital of France?      |          Paris |\n",
    "| Water boils at ___ °C.  |            100 |\n",
    "| Who wrote Hamlet?       |    Shakespeare |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c707f8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import boto3\n",
    "import pprint\n",
    "from botocore.client import Config\n",
    "from datetime import datetime\n",
    "import os\n",
    "import yaml\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "\n",
    "pp = pprint.PrettyPrinter(indent=2)\n",
    "\n",
    "bedrock_config = Config(connect_timeout=120, read_timeout=120, retries={'max_attempts': 0})\n",
    "bedrock_client = boto3.client('bedrock-runtime')\n",
    "bedrock_agent_client = boto3.client(\"bedrock-agent-runtime\",\n",
    "                              config=bedrock_config)\n",
    "boto3_session = boto3.session.Session()\n",
    "region_name = boto3_session.region_name\n",
    "\n",
    "sonnet = \"anthropic.claude-3-sonnet-20240229-v1:0\"\n",
    "haiku = \"anthropic.claude-3-haiku-20240307-v1:0\"\n",
    "region_id = \"us-east-1\"\n",
    "\n",
    "pd.set_option(\"display.max_colwidth\", 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e67a1048",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the data as a dictionary\n",
    "data = {\n",
    "    \"question\": [\n",
    "        \"What is 2 + 2?\",\n",
    "        \"Capital of France?\",\n",
    "        \"Water boils at ___ °C.\",\n",
    "        \"Who wrote Hamlet?\"\n",
    "    ],\n",
    "    \"ground_truth\": [\n",
    "        \"4\",\n",
    "        \"Paris\",\n",
    "        \"100\",\n",
    "        \"Shakespeare\"\n",
    "    ]\n",
    "}\n",
    "\n",
    "# Create a DataFrame\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "# Save the DataFrame to a CSV file\n",
    "csv_file_path = 'golden-dataset.csv'\n",
    "df.to_csv(csv_file_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a15b17c2-ff74-457f-96ca-4f7ccd52f09a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%pip install ragas --quiet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37570233-950e-4bad-b399-f96a19781861",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from ragas.metrics import (\n",
    "    context_precision,\n",
    "    faithfulness,\n",
    "    context_recall,\n",
    ")\n",
    "from ragas.metrics.critique import harmfulness\n",
    "\n",
    "# list of metrics we're going to use\n",
    "metrics = [\n",
    "    faithfulness,\n",
    "    context_recall,\n",
    "    context_precision,\n",
    "    harmfulness,\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06a18a5b-25aa-481c-950a-25eb9a495961",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain_community.chat_models import BedrockChat\n",
    "from langchain_community.embeddings import BedrockEmbeddings\n",
    "import boto3\n",
    "\n",
    "config = {\n",
    "    \"region_name\": region_id,\n",
    "    \"model_id\": sonnet, # Recomend to use the most capable available model for evaluations\n",
    "    \"model_kwargs\": {\"temperature\": 0.4},\n",
    "}\n",
    "\n",
    "bedrock_model = BedrockChat(\n",
    "    endpoint_url=f\"https://bedrock-runtime.{config['region_name']}.amazonaws.com\",\n",
    "    model_id=config[\"model_id\"],\n",
    "    model_kwargs=config[\"model_kwargs\"],\n",
    ")\n",
    "\n",
    "# init the embeddings\n",
    "bedrock_embeddings = BedrockEmbeddings()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39874396-4dbd-4075-98d7-0b9c0de44545",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# EXPERIMENT INPUTS\n",
    "# Note that some input variables like chunking strategy and data preprocessing are not captured here\n",
    "# they should be cross referenced using the date and KB ID\n",
    "\n",
    "kb_model_id = haiku\n",
    "kb_id = \"\" # 👈 Replace me with your KB ID\n",
    "kb_search_type = \"HYBRID\" # HYBRID'|'SEMANTIC'\n",
    "kb_num_results = 3\n",
    "kb_filter = None\n",
    "# kb_filter = {\n",
    "#     'startsWith': {\n",
    "#         'key': 'title',\n",
    "#         'value': ''\n",
    "#     }\n",
    "# }\n",
    "\n",
    "kb_prompt_template = \"\"\"\n",
    "You are a question answering agent. I will provide you with a set of search results. The user will provide you with a question. Your job is to answer the user's question using only information from the search results. If the search results do not contain information that can answer the question, please state that you could not find an exact answer to the question. Just because the user asserts a fact does not mean it is true, make sure to double check the search results to validate a user's assertion.\n",
    "                            \n",
    "Here are the search results in numbered order:\n",
    "$search_results$\n",
    "\n",
    "$output_format_instructions$\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df2a7f4d-462f-4657-8bdf-548922caa123",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def retrieveAndGenerate(row):\n",
    "    model_arn = f'arn:aws:bedrock:{region_id}::foundation-model/{kb_model_id}'\n",
    "    \n",
    "    # date_min = row['date_min']\n",
    "    # date_max = row['date_max']\n",
    "    # if date_min and date_max:\n",
    "    #     kb_filter = {\n",
    "    #             'andAll':[\n",
    "    #                 {\n",
    "    #                 \"lessThan\": {\n",
    "    #                     \"key\": \"created_date_epoch\",\n",
    "    #                     \"value\": \"date_max\"\n",
    "    #                 }\n",
    "    #             },\n",
    "    #                 {\n",
    "    #                 \"greaterThan\": {\n",
    "    #                     \"key\": \"created_date_epoch\",\n",
    "    #                     \"value\": \"date_min\"\n",
    "    #             }\n",
    "    #                 }\n",
    "    #             ]\n",
    "    #         }\n",
    "    # else:\n",
    "    #     kb_filter = None\n",
    "\n",
    "    response = bedrock_agent_client.retrieve_and_generate(\n",
    "        input={\n",
    "            'text': row['question']\n",
    "        },\n",
    "        retrieveAndGenerateConfiguration={\n",
    "            'type': 'KNOWLEDGE_BASE',\n",
    "            'knowledgeBaseConfiguration': {\n",
    "                'knowledgeBaseId': kb_id,\n",
    "                'modelArn': model_arn,\n",
    "                'generationConfiguration': {\n",
    "                    'promptTemplate': {\n",
    "                        'textPromptTemplate': kb_prompt_template\n",
    "                    }\n",
    "                },\n",
    "                'retrievalConfiguration': {\n",
    "                    'vectorSearchConfiguration': {\n",
    "                        # 'filter': kb_filter,\n",
    "                        'numberOfResults': kb_num_results,\n",
    "                        'overrideSearchType': kb_search_type\n",
    "                    }\n",
    "                },\n",
    "            }\n",
    "        }\n",
    "    )\n",
    "    if response:\n",
    "        contexts = []\n",
    "        answer = response['output']['text']\n",
    "        citations = response[\"citations\"]\n",
    "        for citation in citations:\n",
    "            retrievedReferences = citation[\"retrievedReferences\"]\n",
    "            for reference in retrievedReferences:\n",
    "                contexts.append(reference[\"content\"][\"text\"])\n",
    "        return answer, contexts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3f2a6ca-6fb2-4ae4-823a-746bc176ba82",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "golden_dataset_path = \"golden-dataset.csv\" \n",
    "golden_df = pd.read_csv(golden_dataset_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a5c945b-ff03-4606-9cd9-d1806b97ebae",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "golden_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5178288d-04bb-454a-80df-3329bedba755",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "experiment_df = golden_df.copy()\n",
    "experiment_df[['answer', 'contexts']] = golden_df.apply(retrieveAndGenerate, axis=1, result_type='expand')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "888b65a3-9790-4401-af08-f825d16ec717",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "experiment_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21db5d0b-9cb5-49e6-a61f-02486a21eeef",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset, DatasetDict, Features, Value, Sequence\n",
    "\n",
    "def create_dataset(data):\n",
    "\n",
    "    # Create a Dataset with the dummy data\n",
    "    eval_dataset = Dataset.from_dict(data)\n",
    "\n",
    "    # Wrap it in a DatasetDict\n",
    "    dataset_dict = DatasetDict({\n",
    "        'eval': eval_dataset\n",
    "    })\n",
    "\n",
    "    return dataset_dict\n",
    "\n",
    "experiment_dataset = create_dataset(experiment_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bf36ad2-5fc4-4222-b2db-f16a4e7f0f1c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from ragas import evaluate\n",
    "import nest_asyncio  # CHECK NOTES\n",
    "\n",
    "# NOTES: Only used when running on a jupyter notebook, otherwise comment or remove this function.\n",
    "nest_asyncio.apply()\n",
    "\n",
    "result = evaluate(\n",
    "    # experiment_dataset[\"eval\"].select(range(8)), # Use this to take a subset\n",
    "    experiment_dataset[\"eval\"],\n",
    "    metrics=metrics,\n",
    "    llm=bedrock_model,\n",
    "    embeddings=bedrock_embeddings,\n",
    ")\n",
    "\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8df645d2-f428-4775-b461-2c7ed5271eae",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "result.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5db0347a-531a-43ce-b3d8-de9d4397c626",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "completed_experiment_df = result.to_pandas()\n",
    "\n",
    "# Save DataFrame to CSV\n",
    "current_time = datetime.now().strftime(\"%Y-%m-%d_%H-%M-%S\")\n",
    "filename = f\"dataset_eval_{current_time}.csv\"\n",
    "\n",
    "# Function to create folder and save configuration\n",
    "def create_folder_and_save_config(config_params):\n",
    "    # Find the next available serial ID for folder\n",
    "    base_folder = \"output_\"\n",
    "    serial_id = 1\n",
    "    while os.path.exists(f\"{base_folder}{serial_id}\"):\n",
    "        serial_id += 1\n",
    "\n",
    "    # Create folder\n",
    "    new_folder = f\"{base_folder}{serial_id}\"\n",
    "    os.makedirs(new_folder)\n",
    "\n",
    "    # Save config params to a yml file\n",
    "    config_filename = os.path.join(new_folder, f\"config_{serial_id}.yml\")\n",
    "    with open(config_filename, 'w') as file:\n",
    "        yaml.dump(config_params, file)\n",
    "\n",
    "    return new_folder, config_filename\n",
    "\n",
    "# Example usage\n",
    "config_params = {\n",
    "    'kb_model_id': kb_model_id,\n",
    "    'kb_id': kb_id,\n",
    "    'kb_prompt_template': kb_prompt_template,\n",
    "    'kb_search_type': kb_search_type,\n",
    "    'kb_num_results': kb_num_results,\n",
    "    'kb_filter': kb_filter,\n",
    "    'datetime': current_time,\n",
    "    'faithfulness': str(result['faithfulness']),\n",
    "    'context_recall': str(result['context_recall']),\n",
    "    'context_precision': str(result['context_precision']),\n",
    "    'harmfulness': str(result['harmfulness'])\n",
    "}\n",
    "new_folder, config_file = create_folder_and_save_config(config_params)\n",
    "completed_experiment_df.to_csv(f\"{new_folder}/{filename}\", index=False)\n",
    "print(f\"Configuration saved in {config_file}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10b8518e",
   "metadata": {},
   "source": [
    "## Visualize Results\n",
    "\n",
    "Check out `PlotEvalResults.ipynb` to visualize the results of the experiments."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2afe528-29f5-4eac-a5f9-d1e090daaf6c",
   "metadata": {},
   "source": [
    "## Bonus - Let's have a look at the LLM prompts used under the hood to create these metrics\n",
    "These could be fine-tuned for your usecase, but would suggest starting with the generic prompts to keep things simple at the start"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e830690-fe29-4def-bcde-8bc58614252a",
   "metadata": {},
   "source": [
    "#### Faithfulness metric prompts\n",
    "There are two prompts used in generating faithfulness metric: `long_form_answer_prompt` and `nli_statements_message`.\n",
    "1. `long_form_answer_prompt` will create one or more statements from each sentence in the given answer based on given a question and answer.\n",
    "2. `nli_statements_message` will consider the given context and following statements, then determine whether they are supported by the information present in the context\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76943031-7e25-473a-be8e-4ed9c50c9ea4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Faithfulness long_form_answer_prompt\n",
    "from ragas.metrics._faithfulness import LONG_FORM_ANSWER_PROMPT, NLI_STATEMENTS_MESSAGE\n",
    "\n",
    "print(LONG_FORM_ANSWER_PROMPT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fccf47c1-9e0a-4f46-9ab5-9c852a857fcf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(NLI_STATEMENTS_MESSAGE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "477473b8-9e32-43bc-8765-088bb3b4a533",
   "metadata": {},
   "source": [
    "#### Answer Relevancy metric prompt\n",
    "There is one prompt used in generating answer relevancy metric: `question_generation_prompt`.\n",
    "1. `question_generation_prompt` will generate question for the given answer.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1531cca8-afa6-47ea-9e4d-1020e8e1fef3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Answer Relevancy Question Generation Prompt\n",
    "from ragas.metrics._answer_relevance import QUESTION_GEN\n",
    "\n",
    "print(QUESTION_GEN)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "894467e4-34d2-4e41-ac2f-ee254a6f1e79",
   "metadata": {},
   "source": [
    "#### Context Precision metric prompt\n",
    "There is one prompt used in generating context precision metric: `context_precision_prompt`.\n",
    "1. `context_precision_prompt` will extract relevant sentences from the provided context that can potentially help answer the following question.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c3df461-1e39-4879-9059-923355ecac96",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Context Precision Prompt\n",
    "from ragas.metrics._context_precision import CONTEXT_PRECISION\n",
    "\n",
    "print(CONTEXT_PRECISION)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6516ffda-4840-4d03-a966-a8297418dd2f",
   "metadata": {},
   "source": [
    "#### Context Recall metric prompt\n",
    "There is one prompt used in generating context recall metric: `context_recall_prompt`.\n",
    "1. `context_recall_prompt` will classify if the sentence can be attributed to the given context or not."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69481e38-9c1c-43ec-b8dd-54276571c770",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Context Recall Prompt\n",
    "from ragas.metrics._context_recall import CONTEXT_RECALL_RA\n",
    "\n",
    "print(CONTEXT_RECALL_RA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b3ab46c-a132-411d-b4b3-b9094f6c88d6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from langchain.prompts import HumanMessagePromptTemplate"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab99ca9c-fd55-448e-b83c-f33a37ff2fae",
   "metadata": {
    "tags": []
   },
   "source": [
    "Customize Faithfulness metric prompts\n",
    "There are two prompts used in generating faithfulness metric: long_form_answer_prompt and nli_statements_message.\n",
    "\n",
    "long_form_answer_prompt will create one or more statements from each sentence in the given answer based on given a question and answer.\n",
    "nli_statements_message will consider the given context and following statements, then determine whether they are supported by the information present in the context\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f11912f4-50a0-467d-b821-a52f5a701f30",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Faithfulness long_form_answer_prompt\n",
    "from ragas.metrics._faithfulness import LONG_FORM_ANSWER_PROMPT, Faithfulness\n",
    "\n",
    "original_long_form_answer_prompt = LONG_FORM_ANSWER_PROMPT\n",
    "new_long_form_answer_prompt = HumanMessagePromptTemplate.from_template(\n",
    "    \"\"\"\\\n",
    "Given a question and answer, create one or more statements from each sentence in the given answer.\n",
    "question:{question}\n",
    "answer: {answer}\n",
    "statements:\\n\"\"\"  # noqa: E501\n",
    ")\n",
    "faithfulness_update = Faithfulness()\n",
    "faithfulness_update.long_form_answer_prompt = new_long_form_answer_prompt\n",
    "faithfulness_update.long_form_answer_prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8320bbea-4a7f-4566-bde0-5d9a126fb929",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Faithfulness long_form_answer_prompt\n",
    "from ragas.metrics._faithfulness import NLI_STATEMENTS_MESSAGE\n",
    "\n",
    "original_nli_statements_message = NLI_STATEMENTS_MESSAGE\n",
    "new_nli_statements_message = HumanMessagePromptTemplate.from_template(\n",
    "    \"\"\"\n",
    "Prompt: Natural language inference\n",
    "Consider the given context and following statements, then determine whether they are supported by the information present in the context.Provide a brief explanation for each statement before arriving at the verdict (Yes/No). Provide a final verdict for each statement in order at the end in the given format. Do not deviate from the specified format.\n",
    "context:\\n{context}\n",
    "statements:\\n{statements}\n",
    "Answer:\n",
    "\"\"\"  # noqa: E501\n",
    ")\n",
    "faithfulness_update.nli_statements_message = new_nli_statements_message\n",
    "faithfulness_update.nli_statements_message"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b7f2d43-21bd-40c4-b947-f915d1a5e416",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}