GAI_Example.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "763a36da"
      },
      "source": [
        "# Language Model Pipeline for Transcript Processing\n",
        "\n",
        "This pipeline loads and processes a collection of transcripts, splits them into chunks, and embeds them using OpenAI embeddings. The embeddings are then stored in a Pinecone vector store for efficient querying and retrieval.\n",
        "\n",
        "# 1. Install Dependencies\n",
        "Install the necessary libraries and packages for the pipeline."
      ],
      "id": "763a36da"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "434bbcc0",
        "outputId": "0df31047-2a66-472a-aacb-fb68066b54c3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.5/76.5 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m358.9/358.9 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.5/46.5 kB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m471.5/471.5 kB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m48.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m69.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m101.8/101.8 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.5/115.5 kB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m112.2/112.2 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m153.0/153.0 kB\u001b[0m \u001b[31m13.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.2/19.2 MB\u001b[0m \u001b[31m45.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m60.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.5/7.5 MB\u001b[0m \u001b[31m97.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m79.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.2/42.2 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.5/47.5 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m112.5/112.5 kB\u001b[0m \u001b[31m12.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m70.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m66.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for ebooklib (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for python-docx (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for olefile (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for iopath (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.2/50.2 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m154.5/154.5 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m131.0/131.0 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for detectron2 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for fvcore (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m179.1/179.1 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.0/60.0 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m300.4/300.4 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting google-cloud-secret-manager\n",
            "  Downloading google_cloud_secret_manager-2.16.3-py2.py3-none-any.whl (116 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.6/116.6 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-secret-manager) (2.11.1)\n",
            "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-secret-manager) (1.22.3)\n",
            "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.19.5 in /usr/local/lib/python3.10/dist-packages (from google-cloud-secret-manager) (3.20.3)\n",
            "Requirement already satisfied: grpc-google-iam-v1<1.0.0dev,>=0.12.4 in /usr/local/lib/python3.10/dist-packages (from google-cloud-secret-manager) (0.12.6)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (1.60.0)\n",
            "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (2.17.3)\n",
            "Requirement already satisfied: requests<3.0.0.dev0,>=2.18.0 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (2.31.0)\n",
            "Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (1.57.0)\n",
            "Requirement already satisfied: grpcio-status<2.0.dev0,>=1.33.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (1.48.2)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (5.3.1)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (0.3.0)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (1.16.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (4.9)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (2023.7.22)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-cloud-secret-manager) (0.5.0)\n",
            "Installing collected packages: google-cloud-secret-manager\n",
            "Successfully installed google-cloud-secret-manager-2.16.3\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "google"
                ]
              }
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "!pip install langchain -q\n",
        "!pip install openai -q\n",
        "!pip install unstructured -q\n",
        "!pip install unstructured[local-inference] -q\n",
        "!pip install detectron2@git+https://github.com/facebookresearch/detectron2.git@v0.6#egg=detectron2 -q\n",
        "!pip install poppler-utils -q\n",
        "!pip install tiktoken -q\n",
        "!pip install pinecone-client -q\n",
        "!pip install google-cloud-secret-manager\n"
      ],
      "id": "434bbcc0"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9a846f7a"
      },
      "source": [
        "# 2. Import Modules\n",
        "Import the required modules and classes for the pipeline."
      ],
      "id": "9a846f7a"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4ab84f7c",
        "outputId": "c07646d5-e351-46c8-f1ee-2ca446cce73a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/pinecone/index.py:4: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
            "  from tqdm.autonotebook import tqdm\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "import openai\n",
        "import pinecone\n",
        "from langchain.document_loaders import DirectoryLoader\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "from langchain.vectorstores import Pinecone\n",
        "from langchain.chains.question_answering import load_qa_chain\n",
        "from langchain.chains import RetrievalQA\n",
        "from google.cloud import secretmanager\n",
        "from google.colab import auth\n",
        "from google.colab import drive\n"
      ],
      "id": "4ab84f7c"
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "Mount your google drive is the root of your google drive with the directories (MyDrive and Shareddrives)"
      ],
      "metadata": {
        "id": "PE0AneI5h2Wq"
      },
      "id": "PE0AneI5h2Wq"
    },
    {
      "cell_type": "code",
      "source": [
        "drive.mount('/content/gdrive')"
      ],
      "metadata": {
        "id": "Yfb7v1G2hvEB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "84ee7015-04f3-482c-c4ff-ca89673cbc39"
      },
      "id": "Yfb7v1G2hvEB",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "This is a function that loads secrets from Google's secrets manager"
      ],
      "metadata": {
        "id": "UmYX0LWwi71t"
      },
      "id": "UmYX0LWwi71t"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UZk8ZRDBioAu"
      },
      "outputs": [],
      "source": [
        "def load_secrets(secrets_name, project_id):\n",
        "  # Build a client\n",
        "  auth.authenticate_user()\n",
        "  client = secretmanager.SecretManagerServiceClient()\n",
        "  secret_name = secrets_name\n",
        "  # Create path to latest secret\n",
        "  resource_name = f\"projects/{project_id}/secrets/{secret_name}/versions/latest\"\n",
        "  # Get your secret :\n",
        "  response = client.access_secret_version(request={\"name\": resource_name})\n",
        "  secret_string = response.payload.data.decode('UTF-8')\n",
        "  return secret_string"
      ],
      "id": "UZk8ZRDBioAu"
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "Load each secret individually"
      ],
      "metadata": {
        "id": "3-j68CFijINd"
      },
      "id": "3-j68CFijINd"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QRLJm3ebjYiV"
      },
      "outputs": [],
      "source": [
        "project_id = 'botchagalupep1'\n",
        "openai_api_key = load_secrets(\"openai_api_key\",project_id)\n",
        "os.environ['OPENAI_API_KEY'] = openai_api_key\n",
        "pinecone_api_key = load_secrets(\"pinecone_api_key\",project_id)\n",
        "#pinecone_environment = load_secrets(\"pinecone_environment\",project_id)\n",
        "#pinecone_index_name = load_secrets(\"pinecone_index\",project_id)\n",
        "langsmith_api_key = load_secrets(\"langsmith_api_key\",project_id)"
      ],
      "id": "QRLJm3ebjYiV"
    },
    {
      "cell_type": "markdown",
      "source": [
        "This is the code to enable Langsmith (Langchain tracing)"
      ],
      "metadata": {
        "id": "KtqdH7VqjREt"
      },
      "id": "KtqdH7VqjREt"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Vb-tQQrRLoK1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ec460e54-08a4-4c7e-a290-441cc3f99204"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "true\n",
            "https://api.smith.langchain.com\n",
            "GAI-Demo-Agents_Combined_with_vector_stores_as_tools\n"
          ]
        }
      ],
      "source": [
        "# connect notebook to langsmith\n",
        "os.environ['LANGCHAIN_TRACING_V2'] = 'true'\n",
        "os.environ['LANGCHAIN_ENDPOINT'] = 'https://api.smith.langchain.com'\n",
        "\n",
        "# This key is sourced from vars.env\n",
        "os.environ['LANGCHAIN_API_KEY'] = langsmith_api_key  # Uncomment and replace  with your actual API key\n",
        "\n",
        "os.environ['LANGCHAIN_PROJECT'] = 'GAI-Demo-Agents_Combined_with_vector_stores_as_tools'\n",
        "\n",
        "# To verify, you can print the variables\n",
        "print(os.environ.get('LANGCHAIN_TRACING_V2'))\n",
        "print(os.environ.get('LANGCHAIN_ENDPOINT'))\n",
        "#print(os.environ.get('LANGCHAIN_API_KEY'))  # Uncomment if you want to print your API key (be careful with sharing your notebook)\n",
        "print(os.environ.get('LANGCHAIN_PROJECT'))"
      ],
      "id": "Vb-tQQrRLoK1"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "998eff98"
      },
      "source": [
        "# 3. Set API Keys and Initialize Embeddings\n",
        "Set your OpenAI API key and initialize the OpenAI embeddings."
      ],
      "id": "998eff98"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5129cdef"
      },
      "outputs": [],
      "source": [
        "embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key)"
      ],
      "id": "5129cdef"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "479467c8"
      },
      "source": [
        "# 4. Define Functions for Loading and Processing\n",
        "\n",
        "Define a helper function to load and process documents from a given directory using the DirectoryLoader class."
      ],
      "id": "479467c8"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2adb05f7"
      },
      "outputs": [],
      "source": [
        "def load_and_process_docs(directory):\n",
        "    loader = DirectoryLoader(directory)\n",
        "    documents = loader.load()\n",
        "    processed_documents = []\n",
        "    for index, document in enumerate(documents):\n",
        "        print(f\"Document {index}: Successfully loaded and processed {document}\")\n",
        "        processed_documents.append(document)\n",
        "    return processed_documents"
      ],
      "id": "2adb05f7"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5beb1986"
      },
      "source": [
        "# 5. Define Function for Splitting Documents\n",
        "Define a function to split documents into chunks using the RecursiveCharacterTextSplitter."
      ],
      "id": "5beb1986"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f3ea26d4"
      },
      "outputs": [],
      "source": [
        "def split_docs(documents, chunk_size=1000, chunk_overlap=200):\n",
        "    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)\n",
        "    docs = text_splitter.split_documents(documents)\n",
        "    return docs"
      ],
      "id": "f3ea26d4"
    },
    {
      "cell_type": "markdown",
      "source": [
        "This function is for debbugging chunck sizes"
      ],
      "metadata": {
        "id": "gxBlhYrekH3U"
      },
      "id": "gxBlhYrekH3U"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BDMYLO2Nql5v"
      },
      "outputs": [],
      "source": [
        "def show_chunks(documents, chunk_size=1000, chunk_overlap=200):\n",
        "    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)\n",
        "    docs = text_splitter.split_documents(documents)\n",
        "\n",
        "    # Get the first 5 chunks\n",
        "    first_five_chunks = docs[:5]\n",
        "\n",
        "    # Print each chunk and its length\n",
        "    for idx, chunk in enumerate(first_five_chunks, start=1):\n",
        "        print(f\"Chunk {idx}:\")\n",
        "        print(chunk)\n",
        "    return"
      ],
      "id": "BDMYLO2Nql5v"
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "Load the transcript"
      ],
      "metadata": {
        "id": "4kyf20Q5kPet"
      },
      "id": "4kyf20Q5kPet"
    },
    {
      "cell_type": "code",
      "source": [
        "directory = \"/content/gdrive/MyDrive/GAI/transcripts\"\n",
        "documents = load_and_process_docs(directory)\n",
        "docs = split_docs(documents)\n",
        "print(f\"{len(docs)} chunks successfully loaded and processed\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4qsi01hGxi0K",
        "outputId": "7f261058-8a61-4c25-b256-b4b7a1d2a2c0"
      },
      "id": "4qsi01hGxi0K",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " delete ",
            "788 chunks successfully loaded and processed\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Slit and dump chunks with the function defaults"
      ],
      "metadata": {
        "id": "wV6Y7CjFkctR"
      },
      "id": "wV6Y7CjFkctR"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XEvLKRseqqQW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "775b526b-c770-4aa5-85e0-98a22a17e6ca"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Chunk 1:\n",
            "page_content='Advanced Praise\\n\\n“A thoroughly entertaining and educational look at Deming, a man whose insights are fundamental to modern software development. This book includes delightful stories of those around Deming who influenced his work and helped create the foundation for agile and DevOps.\"\\n\\n—Jim Whitehurst, Senior Advisor at IBM\\n\\n“John Willis has looked deep into Dr. Deming’s history and teachings and profoundly explains his profound knowledge. Read this book and you will understand like never before the wisdom and sound advice of Dr. Deming.”\\n\\n—Jeffrey K. Liker, author of The Toyota Way' metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n",
            "Chunk 2:\n",
            "page_content=\"—Jeffrey K. Liker, author of The Toyota Way\\n\\n“John Willis has crafted a delightful and insightful book that brings to life the personal story of W. Edwards Deming and how he created a tremendous impact in the world through his theory of profound knowledge. In this book, you'll go beyond the theory and principles to discover the human experience that has led to some of the greatest organizational successes in the past century. Deming’s Journey to Profound Knowledge is a must-read for any student of Lean, Agile, quality, the Toyota Production System, and continuous improvement— or any leader who seeks to create a culture of excellence in their organization.” —Katie Anderson, author of Learning to Lead, Leading to Learn\" metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n",
            "Chunk 3:\n",
            "page_content='“John has read everything available about Deming and has woven the many threads of Dr. Deming’s life into a work of love. The result is a very pleasant fable that will prompt the novice reader to want to know more. While experts will have differing views on Deming’s work and contribution to management science, there is no doubt that the IT community can only benefit from learn- ing about Deming’s work. John’s book is a step in this direction.”\\n\\n—Domenico Lepore, author of Deming and Goldratt: The Theory of Constraints and the System of Profound Knowledge\\n\\n“Willis manages to achieve a rare feat: provide fresh and unique insights on a famous subject. Whether you care about improving businesses or are simply fascinated by the human condition, this book is worthy of your time. “\\n\\n—Damon Edwards, Entrepreneur and Technology Executive\\n\\n“John beautifully weaves together business and biography to tell a story as intriguing as it is insightful.”\\n\\n—William Willis, Techstrong Group' metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n",
            "Chunk 4:\n",
            "page_content='—Damon Edwards, Entrepreneur and Technology Executive\\n\\n“John beautifully weaves together business and biography to tell a story as intriguing as it is insightful.”\\n\\n—William Willis, Techstrong Group\\n\\n25 NW 23rd Pl, Suite 6314 Portland, OR 97210\\n\\nCopyright © 2023 by John Willis\\n\\nAll rights reserved. For information about permission to reproduce selections from this book, write to Permissions, IT Revolution Press, LLC, 25 NW 23rd Pl, Suite 6314, Portland, OR 97210\\n\\n28 27 26 25 24 23\\n\\nFirst Edition Printed in the United States of America 1 2 3 4 5 6 7 8 9 10\\n\\nCover design by Devon Smith Creative Book design by Devon Smith Creative\\n\\nLibrary of Congress Cataloging-in-Publication Data' metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n",
            "Chunk 5:\n",
            "page_content='Cover design by Devon Smith Creative Book design by Devon Smith Creative\\n\\nLibrary of Congress Cataloging-in-Publication Data\\n\\nNames: Willis, John (Writer on information technology), author. | Lewis, Derek, 1983- author. | Hinkle, Mark R., writer of foreword. Title: Deming\\'s journey to profound knowledge : how Deming helped win a war, altered the face of industry, and holds the key to our future / by John \"Botchagalupe\" Willis ; with Derek Lewis ; foreword by Mark Hinkle.' metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n"
          ]
        }
      ],
      "source": [
        "show_chunks(docs)"
      ],
      "id": "XEvLKRseqqQW"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KgVZMV7aqz9C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e4ff0598-2f02-40c5-c53f-b1af96b6c066"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Chunk 1:\n",
            "page_content='Advanced Praise\\n\\n“A thoroughly entertaining and educational look at Deming, a man whose insights are fundamental to modern software development. This book includes delightful stories of those around Deming who influenced his work and helped create the foundation for agile and DevOps.\"\\n\\n—Jim Whitehurst, Senior Advisor at IBM' metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n",
            "Chunk 2:\n",
            "page_content='“John Willis has looked deep into Dr. Deming’s history and teachings and profoundly explains his profound knowledge. Read this book and you will understand like never before the wisdom and sound advice of Dr. Deming.”\\n\\n—Jeffrey K. Liker, author of The Toyota Way' metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n",
            "Chunk 3:\n",
            "page_content='—Jeffrey K. Liker, author of The Toyota Way' metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n",
            "Chunk 4:\n",
            "page_content=\"“John Willis has crafted a delightful and insightful book that brings to life the personal story of W. Edwards Deming and how he created a tremendous impact in the world through his theory of profound knowledge. In this book, you'll go beyond the theory and principles to discover the human experience that has led to some of the greatest organizational successes in the past century. Deming’s Journey to Profound Knowledge is a must-read for any student of Lean, Agile, quality, the Toyota\" metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n",
            "Chunk 5:\n",
            "page_content='quality, the Toyota Production System, and continuous improvement— or any leader who seeks to create a culture of excellence in their organization.” —Katie Anderson, author of Learning to Lead, Leading to Learn' metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n"
          ]
        }
      ],
      "source": [
        "show_chunks(docs, chunk_size=500, chunk_overlap=20)"
      ],
      "id": "KgVZMV7aqz9C"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2CbVEwxMrVSq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c0656e59-a570-4599-e208-2017560b4755"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Chunk 1:\n",
            "page_content='Advanced Praise' metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n",
            "Chunk 2:\n",
            "page_content='“A thoroughly entertaining and educational look at Deming, a man whose insights are fundamental to modern software development. This book includes delightful stories of those around Deming who influenced his work and helped create the foundation for' metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n",
            "Chunk 3:\n",
            "page_content='the foundation for agile and DevOps.\"' metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n",
            "Chunk 4:\n",
            "page_content='—Jim Whitehurst, Senior Advisor at IBM' metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n",
            "Chunk 5:\n",
            "page_content='“John Willis has looked deep into Dr. Deming’s history and teachings and profoundly explains his profound knowledge. Read this book and you will understand like never before the wisdom and sound advice of Dr. Deming.”' metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}\n"
          ]
        }
      ],
      "source": [
        "show_chunks(docs, chunk_size=250, chunk_overlap=20)"
      ],
      "id": "2CbVEwxMrVSq"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "210b87bd"
      },
      "source": [
        "# 6. Load and Process Documents\n",
        "\n",
        "Load and process the documents from the ./Transcripts directory, then split them into chunks."
      ],
      "id": "210b87bd"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "29cdb97d"
      },
      "source": [
        "# 7. Initialize Pinecone and Create Index\n",
        "Initialize Pinecone with your API key and environment, then create an index using the Pinecone.from_documents() method."
      ],
      "id": "29cdb97d"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "68deb106",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c94fb7fb-c02a-4aa4-f29e-84e99d8d5331"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "gai-test1\n"
          ]
        }
      ],
      "source": [
        "pinecone.init(\n",
        "    api_key=pinecone_api_key,  # find at app.pinecone.io\n",
        "    environment=\"asia-southeast1-gcp-free\"  # next to api key in console\n",
        ")\n",
        "\n",
        "index = pinecone.Index('gai-test1')\n",
        "index_name = \"gai-test1\"\n",
        "\n",
        "print(index_name)\n",
        "index = Pinecone.from_documents(docs, embeddings, index_name=index_name)\n",
        "pinecone_vectore_store = Pinecone.from_existing_index(index_name, embeddings)"
      ],
      "id": "68deb106"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ac864a1"
      },
      "source": [
        "# 8. Define the Question-Answering Function\n",
        "Define a function to perform question-answering tasks using the Pinecone index created in the previous steps. The function takes a query, a chain type, and a value for k (the number of similar chunks to consider) as input and returns the best answer along with the relevant source documents."
      ],
      "id": "5ac864a1"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "06bd4a26"
      },
      "outputs": [],
      "source": [
        "from langchain.chains import RetrievalQA\n",
        "from langchain.llms import OpenAI\n",
        "\n",
        "def qa(query, chain_type, k, index=index, openai_api_key=openai.api_key):\n",
        "\n",
        "    # Expose the Pinecone index in a retriever interface\n",
        "    retriever = index.as_retriever(search_type=\"similarity\", search_kwargs={\"k\": k})\n",
        "\n",
        "    # Create a chain to answer questions\n",
        "    qa_chain = RetrievalQA.from_chain_type(\n",
        "        llm=OpenAI(openai_api_key=openai_api_key),\n",
        "        chain_type=chain_type,\n",
        "        retriever=retriever,\n",
        "        return_source_documents=True,\n",
        "    )\n",
        "\n",
        "    result = qa_chain({\"query\": query})\n",
        "    print(result['result'])\n",
        "    return result\n"
      ],
      "id": "06bd4a26"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "10374bb9"
      },
      "source": [
        "# 9. Example Usage of the Question-Answering Function\n",
        "\n",
        "Below is an example of how to use the qa() function to answer a question about HRMC (Deming's Journey to Profound Knowledge) using the diferent chain types."
      ],
      "id": "10374bb9"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0095476b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9d2d733f-acda-459a-f2ac-9ecf40c8f380"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " The book is a modern retelling of the bestselling business book The Goal, a novel about supply-chain management in a manufacturing setting. The Phoenix Project provides a modern lens on the topic, focusing on software development and delivery. It includes DevOps in the subtitle. After the book is published, the authors wrote The DevOps Handbook to provide a prescriptive solution to readers. The two books have a combined sales of almost a million copies.\n"
          ]
        }
      ],
      "source": [
        "result = qa(\"Summarize the book\", \"stuff\", 5)"
      ],
      "id": "0095476b"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fEeLewARsyFB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "18df87e5-44ee-4caa-8a5c-3bf1734f13f6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "The Phoenix Project is a modern retelling of the bestselling business book The Goal, focusing on software development and delivery. The book follows the story of W. Edwards Deming, the author of Deming's masterwork, as he shared it with the world at the age of ninety-three just before his death. Imagine publishing your magnum opus at that age, just before your death - this gives you a clue as to the kind of man Deming was. Along with The DevOps Handbook, which was written by Gene Kim, Patrick Debois, Jez Humble and the author, the books have sold almost a million copies combined. This modern retelling of The Goal is essential reading for anyone who wants to understand the importance of quality and uniformity in the world of software development and delivery.\n"
          ]
        }
      ],
      "source": [
        "result = qa(\"Summarize the book\", \"refine\", 5)"
      ],
      "id": "fEeLewARsyFB"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MdnUu1vFtTQr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "034efcf8-b9de-48e6-9807-025e2c9cb326"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " This book is a portrait of W. Edwards Deming and John Willis, two soft-spoken experts in this century's biggest advancements. It captures the essence of how to improve most any business, with a focus on quality and understanding of the people responsible for that quality. It also gives a portrait of the hard-working, humble man from Wyoming who helped revolutionize manufacturing in Japan and around the world.\n"
          ]
        }
      ],
      "source": [
        "result = qa(\"Summarize the book\", \"map_reduce\", 5)"
      ],
      "id": "MdnUu1vFtTQr"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "607ea13c"
      },
      "source": [
        "## 10. Language Model Pipeline for Querying Pinecone Datastore\n",
        "This pipeline leverages a Pinecone vector store which contains embedded documents (transcripts, in this case) for efficient querying and retrieval. The main function, get_answer, retrieves relevant documents in response to a prompt and uses a question-answering pipeline to produce a refined answer from these documents."
      ],
      "id": "607ea13c"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "12368e34"
      },
      "outputs": [],
      "source": [
        "from langchain.chains import RetrievalQA\n",
        "from langchain.llms import OpenAI\n",
        "\n",
        "def qa_from_vector_db(query, chain_type, k, openai_api_key=openai.api_key):\n",
        "\n",
        "    # Expose the Pinecone index in a retriever interface\n",
        "    #retriever = index.as_retriever(search_type=\"similarity\", search_kwargs={\"k\": k})\n",
        "    pinecone_vector_store = Pinecone.from_existing_index(index_name, embeddings)\n",
        "\n",
        "    # Create a chain to answer questions\n",
        "    qa_chain = RetrievalQA.from_chain_type(\n",
        "        llm=OpenAI(openai_api_key=openai_api_key),\n",
        "        chain_type=chain_type,\n",
        "        retriever=pinecone_vector_store.as_retriever(),\n",
        "        return_source_documents=True,\n",
        "    )\n",
        "\n",
        "    result = qa_chain({\"query\": query})\n",
        "    print(query)\n",
        "    print(result['result'])\n",
        "    return result"
      ],
      "id": "12368e34"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "829930fd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3e96ba43-7213-4d4c-f4a6-b2c0ef9a369d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "what were the top 5 things about the book\n",
            "\n",
            "\n",
            "1. It is a biography of W. Edwards Deming, a man who published his magnum opus at the age of 93.\n",
            "2. It tells the story behind Deming's magnum opus and provides insight into the power of his work, the concept of quality and uniformity as foundations of commerce, prosperity, and peace.\n",
            "3. It explores the life of an extraordinary man and the impact his work had on the world, including his ideas, quotations, and paraphrases attributed to different thinkers and industry leaders.\n",
            "4. It was published when Deming was 93 years old, just before his death.\n",
            "5. It was written by acclaimed author Michael Lewis, who is known for his biographies, and is available in eBook, Web PDF, and Audio formats.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'query': 'what were the top 5 things about the book',\n",
              " 'result': \"\\n\\n1. It is a biography of W. Edwards Deming, a man who published his magnum opus at the age of 93.\\n2. It tells the story behind Deming's magnum opus and provides insight into the power of his work, the concept of quality and uniformity as foundations of commerce, prosperity, and peace.\\n3. It explores the life of an extraordinary man and the impact his work had on the world, including his ideas, quotations, and paraphrases attributed to different thinkers and industry leaders.\\n4. It was published when Deming was 93 years old, just before his death.\\n5. It was written by acclaimed author Michael Lewis, who is known for his biographies, and is available in eBook, Web PDF, and Audio formats.\",\n",
              " 'source_documents': [Document(page_content='One of my favorite authors is Michael Lewis. When reading Moneyball, for example, you think you’re reading a book about baseball statistics, but by the time you finish, you find that you’ve read a biography of Billy Beane. Similarly, while this book may look like a biography of Deming, it’s the story behind the story of his masterwork, which he shared with the world when he was nine- ty-three years old. Imagine publishing your magnum opus at that age, just before your death. That gives you a clue as to the kind of man you’re dealing with.', metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}),\n",
              "  Document(page_content='One of my favorite authors is Michael Lewis. When reading Moneyball, for example, you think you’re reading a book about baseball statistics, but by the time you finish, you find that you’ve read a biography of Billy Beane. Similarly, while this book may look like a biography of Deming, it’s the story behind the story of his masterwork, which he shared with the world when he was nine- ty-three years old. Imagine publishing your magnum opus at that age, just before your death. That gives you a clue as to the kind of man you’re dealing with.', metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}),\n",
              "  Document(page_content='ISBN: 9781950508839 eBook ISBN: 9781950508846 Web PDF ISBN: 9781950508853 Audio: 9781950508860\\n\\nPublisher’s note to readers: Many of the ideas, quotations, and paraphrases attributed to different thinkers and industry leaders herein are excerpted from informal conversations, correspondence, interviews, conference roundtables, and other forms of oral communication. Although the authors and publisher have made every effort to ensure that the information in this book was correct at press time, the authors and publisher do not assume and hereby disclaim any liability to any party for any loss, damage, or disruption caused by errors or omissions, whether such errors or omissions result from negligence, accident, or any other cause.\\n\\nFor information about special discounts for bulk purchases or for information on booking authors for an event, please visit our website at www.ITRevolution.com.\\n\\nThe right quality and uniformity are foundations\\n\\nof commerce, prosperity, and peace.', metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}),\n",
              "  Document(page_content='ISBN: 9781950508839 eBook ISBN: 9781950508846 Web PDF ISBN: 9781950508853 Audio: 9781950508860\\n\\nPublisher’s note to readers: Many of the ideas, quotations, and paraphrases attributed to different thinkers and industry leaders herein are excerpted from informal conversations, correspondence, interviews, conference roundtables, and other forms of oral communication. Although the authors and publisher have made every effort to ensure that the information in this book was correct at press time, the authors and publisher do not assume and hereby disclaim any liability to any party for any loss, damage, or disruption caused by errors or omissions, whether such errors or omissions result from negligence, accident, or any other cause.\\n\\nFor information about special discounts for bulk purchases or for information on booking authors for an event, please visit our website at www.ITRevolution.com.\\n\\nThe right quality and uniformity are foundations\\n\\nof commerce, prosperity, and peace.', metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'})]}"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ],
      "source": [
        "qa_from_vector_db(\"what were the top 5 things about the book\",\"refine\",6)"
      ],
      "id": "829930fd"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Cj79JH7wokCS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1d134b8f-9b92-4989-a402-8561f72350ec"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tell me more about the book\n",
            " The book is The DevOps Handbook, published in 2016. It was written by Gene Kim, Patrick Debois, Jez Humble, and George Spafford. The book is about software development and delivery, and includes DevOps in the subtitle. It provides a prescriptive solution after reading it. It has sold almost a million copies. The ISBNs are 9781950508839, 9781950508846, 9781950508853, and 9781950508860.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'query': 'Tell me more about the book',\n",
              " 'result': ' The book is The DevOps Handbook, published in 2016. It was written by Gene Kim, Patrick Debois, Jez Humble, and George Spafford. The book is about software development and delivery, and includes DevOps in the subtitle. It provides a prescriptive solution after reading it. It has sold almost a million copies. The ISBNs are 9781950508839, 9781950508846, 9781950508853, and 9781950508860.',\n",
              " 'source_documents': [Document(page_content='One of my favorite authors is Michael Lewis. When reading Moneyball, for example, you think you’re reading a book about baseball statistics, but by the time you finish, you find that you’ve read a biography of Billy Beane. Similarly, while this book may look like a biography of Deming, it’s the story behind the story of his masterwork, which he shared with the world when he was nine- ty-three years old. Imagine publishing your magnum opus at that age, just before your death. That gives you a clue as to the kind of man you’re dealing with.', metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}),\n",
              "  Document(page_content='As was his usual arrangement, Ed signed books before heading off to dinner with those who’d paid for the seminar and select VIPs. As he was sign- ing Doris’s book, he looked up at her with “those steely blue eyes,” as she described them. In his deep, gruff voice, he asked, “Now, you’re the one who was in New Guinea?”\\n\\n“Yes, sir,” she hesitantly replied. “I did that for four years.” “Would you come to dinner with us? I’d like to hear more about that.” Wide eyed, Doris turned to the organizer of the event, Dr. Nancy Mann,*\\n\\nand asked in disbelief, “Can I do that?”', metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}),\n",
              "  Document(page_content='Gene began to discuss a book he, George, and Kevin had been working on for five years. It was a modern retelling of the bestselling business book The Goal, a novel about supply-chain management in a manufacturing setting. These three men gave it a modern lens: a novel about software development and delivery. They called it The Phoenix Project. (He included DevOps in the subtitle, I might proudly add.)\\n\\nIn discussing the book, Gene and I realized that people would want a pre- scriptive solution after reading it. A few years before it was published, Gene, Patrick Debois, and I began working on what would become The DevOps Hand- book. Jez Humble joined us and we published the book in 2016. The books have sold almost a million copies combined.', metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'}),\n",
              "  Document(page_content='ISBN: 9781950508839 eBook ISBN: 9781950508846 Web PDF ISBN: 9781950508853 Audio: 9781950508860\\n\\nPublisher’s note to readers: Many of the ideas, quotations, and paraphrases attributed to different thinkers and industry leaders herein are excerpted from informal conversations, correspondence, interviews, conference roundtables, and other forms of oral communication. Although the authors and publisher have made every effort to ensure that the information in this book was correct at press time, the authors and publisher do not assume and hereby disclaim any liability to any party for any loss, damage, or disruption caused by errors or omissions, whether such errors or omissions result from negligence, accident, or any other cause.\\n\\nFor information about special discounts for bulk purchases or for information on booking authors for an event, please visit our website at www.ITRevolution.com.\\n\\nThe right quality and uniformity are foundations\\n\\nof commerce, prosperity, and peace.', metadata={'source': '/content/gdrive/MyDrive/GAI/transcripts/DEM_Interior.6.24.23.pdf'})]}"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ],
      "source": [
        "qa_from_vector_db(\"Tell me more about the book\",\"stuff\",6)"
      ],
      "id": "Cj79JH7wokCS"
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}