GenAI-Workshop-Demo6-old.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[{"file_id":"1vjgl2OTC_r_tYNa7hnNwp7sfIPMde9e2","timestamp":1709952647262},{"file_id":"1gMGVSHvrHDh8Mf3okBAyK8JMEJJKE4lU","timestamp":1708972784256},{"file_id":"1tRDG7JrUiqRAJOQShKk_3hDNZvOpqN2T","timestamp":1705687451345}]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"96cb24bab89e4c1aad531ccd82d802d7":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_bbe758eaae0941d28d1cc253a6bc0d91","IPY_MODEL_b4c1dad3f66447bc80cefc02d07b70ff","IPY_MODEL_e05eb92d5b744d26859add573c3c6c89"],"layout":"IPY_MODEL_0c2b78cf01244fbe952e018573945abd"}},"bbe758eaae0941d28d1cc253a6bc0d91":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_249d08db554b4cccbc90bca485bd19df","placeholder":"​","style":"IPY_MODEL_0e635f108be3460baf24511f6ac2e2f8","value":"run_evals "}},"b4c1dad3f66447bc80cefc02d07b70ff":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_92c6140c6376408abd85a4b6574b61f8","max":18,"min":0,"orientation":"horizontal","style":"IPY_MODEL_af2df273f1f640df818981293f49f351","value":18}},"e05eb92d5b744d26859add573c3c6c89":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_49056b127534493cb4c75261beac7e50","placeholder":"​","style":"IPY_MODEL_c0a97a26d6ec46dd9c20f13b4489a33d","value":" 18/18 (100.0%) | ⏳ 01:39&lt;00:00 | 15.71s/it"}},"0c2b78cf01244fbe952e018573945abd":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"249d08db554b4cccbc90bca485bd19df":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0e635f108be3460baf24511f6ac2e2f8":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"92c6140c6376408abd85a4b6574b61f8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"af2df273f1f640df818981293f49f351":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"49056b127534493cb4c75261beac7e50":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c0a97a26d6ec46dd9c20f13b4489a33d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"cfe8f751ec884953875987520292a4e8":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_7b1f2fda8793450f9acb18cf220eb604","IPY_MODEL_c8d2d0bb80cc47308c2cd63246f223ab","IPY_MODEL_91cde2e70a8042f7b2066d13b54c8ba1"],"layout":"IPY_MODEL_b47040fdaae7428b90073b418b810cab"}},"7b1f2fda8793450f9acb18cf220eb604":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_93949f65278a42138d9c84a0e656d854","placeholder":"​","style":"IPY_MODEL_9ce2eb3f10bd4cb09d407a8cc0023a28","value":"run_evals "}},"c8d2d0bb80cc47308c2cd63246f223ab":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_59c551bb1e53417795faed73fa63d186","max":36,"min":0,"orientation":"horizontal","style":"IPY_MODEL_8bce30771eab420f90b347243070c277","value":36}},"91cde2e70a8042f7b2066d13b54c8ba1":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7aa5a9456c814b9b9760de59c3973e2b","placeholder":"​","style":"IPY_MODEL_6de72b48ba264c61b5f60a71f0715a7b","value":" 36/36 (100.0%) | ⏳ 00:27&lt;00:00 |  1.68s/it"}},"b47040fdaae7428b90073b418b810cab":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"93949f65278a42138d9c84a0e656d854":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9ce2eb3f10bd4cb09d407a8cc0023a28":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"59c551bb1e53417795faed73fa63d186":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8bce30771eab420f90b347243070c277":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"7aa5a9456c814b9b9760de59c3973e2b":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6de72b48ba264c61b5f60a71f0715a7b":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"cells":[{"cell_type":"markdown","source":["# Evaluation of OpenContext Documentaion with Arize"],"metadata":{"id":"6Yi1T3_JH1ay"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"Sl3lgsKvh47O","executionInfo":{"status":"ok","timestamp":1710435007583,"user_tz":420,"elapsed":33879,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"b17c60b5-b59d-4665-a288-2fd4ddcb1477"},"outputs":[{"output_type":"stream","name":"stdout","text":["\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m809.1/809.1 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m286.1/286.1 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m677.2/677.2 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m22.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.7/119.7 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m23.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m258.8/258.8 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.6/67.6 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m257.5/257.5 kB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.0/53.0 kB\u001b[0m \u001b[31m3.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m138.5/138.5 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.8/77.8 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h"]}],"source":["%pip install --quiet langchain pypdf pymongo langchain-openai tiktoken google-cloud-secret-manager"]},{"cell_type":"code","source":["%pip install --upgrade google-auth"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":421},"id":"A48y0kGZcaGd","executionInfo":{"status":"ok","timestamp":1710435021348,"user_tz":420,"elapsed":13768,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"11cf70ff-b9ee-4075-802f-c7f4e2413f9e"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Requirement already satisfied: google-auth in /usr/local/lib/python3.10/dist-packages (2.27.0)\n","Collecting google-auth\n","  Downloading google_auth-2.28.2-py2.py3-none-any.whl (186 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m186.9/186.9 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth) (5.3.3)\n","Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth) (0.3.0)\n","Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth) (4.9)\n","Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth) (0.5.1)\n","Installing collected packages: google-auth\n","  Attempting uninstall: google-auth\n","    Found existing installation: google-auth 2.27.0\n","    Uninstalling google-auth-2.27.0:\n","      Successfully uninstalled google-auth-2.27.0\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","google-colab 1.0.0 requires google-auth==2.27.0, but you have google-auth 2.28.2 which is incompatible.\u001b[0m\u001b[31m\n","\u001b[0mSuccessfully installed google-auth-2.28.2\n"]},{"output_type":"display_data","data":{"application/vnd.colab-display-data+json":{"pip_warning":{"packages":["google"]},"id":"9087f84f7fc54d488777ff81358f1e9c"}},"metadata":{}}]},{"cell_type":"code","source":["import os\n","\n","%pip show langchain\n","\n","from platform import python_version\n","print(python_version())"],"metadata":{"id":"WlkitefoiJ24","executionInfo":{"status":"ok","timestamp":1710435034186,"user_tz":420,"elapsed":12845,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"23e8698a-3b82-4a2b-e57c-b705539eeaf7"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Name: langchain\n","Version: 0.1.12\n","Summary: Building applications with LLMs through composability\n","Home-page: https://github.com/langchain-ai/langchain\n","Author: \n","Author-email: \n","License: MIT\n","Location: /usr/local/lib/python3.10/dist-packages\n","Requires: aiohttp, async-timeout, dataclasses-json, jsonpatch, langchain-community, langchain-core, langchain-text-splitters, langsmith, numpy, pydantic, PyYAML, requests, SQLAlchemy, tenacity\n","Required-by: \n","3.10.12\n"]}]},{"cell_type":"code","source":["!pip install arize-phoenix"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vRd6LDTiFidh","executionInfo":{"status":"ok","timestamp":1710435142363,"user_tz":420,"elapsed":108179,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"6c44f046-4b29-4228-e105-5879c84f4cd4"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting arize-phoenix\n","  Downloading arize_phoenix-3.14.1-py3-none-any.whl (1.2 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting ddsketch (from arize-phoenix)\n","  Downloading ddsketch-2.0.4-py3-none-any.whl (18 kB)\n","Collecting hdbscan>=0.8.33 (from arize-phoenix)\n","  Downloading hdbscan-0.8.33.tar.gz (5.2 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.2/5.2 MB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n","  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n","  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (3.1.3)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (1.25.2)\n","Collecting openinference-instrumentation-langchain>=0.1.12 (from arize-phoenix)\n","  Downloading openinference_instrumentation_langchain-0.1.12-py3-none-any.whl (13 kB)\n","Collecting openinference-instrumentation-llama-index>=1.2.0 (from arize-phoenix)\n","  Downloading openinference_instrumentation_llama_index-1.2.0-py3-none-any.whl (15 kB)\n","Collecting openinference-instrumentation-openai>=0.1.4 (from arize-phoenix)\n","  Downloading openinference_instrumentation_openai-0.1.4-py3-none-any.whl (21 kB)\n","Collecting openinference-semantic-conventions>=0.1.5 (from arize-phoenix)\n","  Downloading openinference_semantic_conventions-0.1.5-py3-none-any.whl (8.4 kB)\n","Collecting opentelemetry-exporter-otlp (from arize-phoenix)\n","  Downloading opentelemetry_exporter_otlp-1.23.0-py3-none-any.whl (7.0 kB)\n","Collecting opentelemetry-proto (from arize-phoenix)\n","  Downloading opentelemetry_proto-1.23.0-py3-none-any.whl (50 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.8/50.8 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting opentelemetry-sdk (from arize-phoenix)\n","  Downloading opentelemetry_sdk-1.23.0-py3-none-any.whl (105 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.7/105.7 kB\u001b[0m \u001b[31m10.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (1.5.3)\n","Requirement already satisfied: protobuf<5.0,>=3.20 in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (3.20.3)\n","Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (5.9.5)\n","Requirement already satisfied: pyarrow in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (14.0.2)\n","Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (2.31.0)\n","Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (1.2.2)\n","Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (1.11.4)\n","Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (2.4.0)\n","Collecting starlette (from arize-phoenix)\n","  Downloading starlette-0.37.2-py3-none-any.whl (71 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting strawberry-graphql==0.208.2 (from arize-phoenix)\n","  Downloading strawberry_graphql-0.208.2-py3-none-any.whl (269 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m269.0/269.0 kB\u001b[0m \u001b[31m25.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (4.66.2)\n","Requirement already satisfied: typing-extensions>=4.5 in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (4.10.0)\n","Collecting umap-learn (from arize-phoenix)\n","  Downloading umap-learn-0.5.5.tar.gz (90 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.9/90.9 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n","Collecting uvicorn (from arize-phoenix)\n","  Downloading uvicorn-0.28.0-py3-none-any.whl (60 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.6/60.6 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: wrapt in /usr/local/lib/python3.10/dist-packages (from arize-phoenix) (1.14.1)\n","Collecting graphql-core<3.3.0,>=3.2.0 (from strawberry-graphql==0.208.2->arize-phoenix)\n","  Downloading graphql_core-3.2.3-py3-none-any.whl (202 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m202.9/202.9 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.7.0 in /usr/local/lib/python3.10/dist-packages (from strawberry-graphql==0.208.2->arize-phoenix) (2.8.2)\n","Collecting cython<3,>=0.27 (from hdbscan>=0.8.33->arize-phoenix)\n","  Using cached Cython-0.29.37-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (1.9 MB)\n","Requirement already satisfied: joblib>=1.0 in /usr/local/lib/python3.10/dist-packages (from hdbscan>=0.8.33->arize-phoenix) (1.3.2)\n","Collecting opentelemetry-api (from openinference-instrumentation-langchain>=0.1.12->arize-phoenix)\n","  Downloading opentelemetry_api-1.23.0-py3-none-any.whl (58 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.4/58.4 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting opentelemetry-instrumentation (from openinference-instrumentation-langchain>=0.1.12->arize-phoenix)\n","  Downloading opentelemetry_instrumentation-0.44b0-py3-none-any.whl (28 kB)\n","Collecting opentelemetry-semantic-conventions (from openinference-instrumentation-langchain>=0.1.12->arize-phoenix)\n","  Downloading opentelemetry_semantic_conventions-0.44b0-py3-none-any.whl (36 kB)\n","Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->arize-phoenix) (3.3.0)\n","Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from ddsketch->arize-phoenix) (1.16.0)\n","Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->arize-phoenix) (2.1.5)\n","Collecting opentelemetry-exporter-otlp-proto-grpc==1.23.0 (from opentelemetry-exporter-otlp->arize-phoenix)\n","  Downloading opentelemetry_exporter_otlp_proto_grpc-1.23.0-py3-none-any.whl (18 kB)\n","Collecting opentelemetry-exporter-otlp-proto-http==1.23.0 (from opentelemetry-exporter-otlp->arize-phoenix)\n","  Downloading opentelemetry_exporter_otlp_proto_http-1.23.0-py3-none-any.whl (16 kB)\n","Collecting deprecated>=1.2.6 (from opentelemetry-exporter-otlp-proto-grpc==1.23.0->opentelemetry-exporter-otlp->arize-phoenix)\n","  Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n","Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc==1.23.0->opentelemetry-exporter-otlp->arize-phoenix) (1.63.0)\n","Requirement already satisfied: grpcio<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc==1.23.0->opentelemetry-exporter-otlp->arize-phoenix) (1.62.1)\n","Collecting opentelemetry-exporter-otlp-proto-common==1.23.0 (from opentelemetry-exporter-otlp-proto-grpc==1.23.0->opentelemetry-exporter-otlp->arize-phoenix)\n","  Downloading opentelemetry_exporter_otlp_proto_common-1.23.0-py3-none-any.whl (17 kB)\n","Collecting importlib-metadata<7.0,>=6.0 (from opentelemetry-api->openinference-instrumentation-langchain>=0.1.12->arize-phoenix)\n","  Downloading importlib_metadata-6.11.0-py3-none-any.whl (23 kB)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->arize-phoenix) (3.3.2)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->arize-phoenix) (3.6)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->arize-phoenix) (2.0.7)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->arize-phoenix) (2024.2.2)\n","Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->arize-phoenix) (2023.4)\n","Requirement already satisfied: anyio<5,>=3.4.0 in /usr/local/lib/python3.10/dist-packages (from starlette->arize-phoenix) (3.7.1)\n","Requirement already satisfied: numba>=0.51.2 in /usr/local/lib/python3.10/dist-packages (from umap-learn->arize-phoenix) (0.58.1)\n","Collecting pynndescent>=0.5 (from umap-learn->arize-phoenix)\n","  Downloading pynndescent-0.5.11-py3-none-any.whl (55 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.8/55.8 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: click>=7.0 in /usr/local/lib/python3.10/dist-packages (from uvicorn->arize-phoenix) (8.1.7)\n","Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.10/dist-packages (from uvicorn->arize-phoenix) (0.14.0)\n","Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.4.0->starlette->arize-phoenix) (1.3.1)\n","Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.4.0->starlette->arize-phoenix) (1.2.0)\n","Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.2->umap-learn->arize-phoenix) (0.41.1)\n","Requirement already satisfied: setuptools>=16.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation->openinference-instrumentation-langchain>=0.1.12->arize-phoenix) (67.7.2)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0,>=6.0->opentelemetry-api->openinference-instrumentation-langchain>=0.1.12->arize-phoenix) (3.17.0)\n","Building wheels for collected packages: hdbscan, umap-learn\n","  Building wheel for hdbscan (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for hdbscan: filename=hdbscan-0.8.33-cp310-cp310-linux_x86_64.whl size=3039278 sha256=53a34279b856fcd0172d13953eced098fa24b7989f5c0f492ae67efc4f9fa7d8\n","  Stored in directory: /root/.cache/pip/wheels/75/0b/3b/dc4f60b7cc455efaefb62883a7483e76f09d06ca81cf87d610\n","  Building wheel for umap-learn (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for umap-learn: filename=umap_learn-0.5.5-py3-none-any.whl size=86832 sha256=9c0def78e8438f973925a47566923df4ca036903cad16311de02e2d917951917\n","  Stored in directory: /root/.cache/pip/wheels/3a/70/07/428d2b58660a1a3b431db59b806a10da736612ebbc66c1bcc5\n","Successfully built hdbscan umap-learn\n","Installing collected packages: uvicorn, opentelemetry-semantic-conventions, opentelemetry-proto, openinference-semantic-conventions, importlib-metadata, graphql-core, deprecated, ddsketch, cython, strawberry-graphql, starlette, opentelemetry-exporter-otlp-proto-common, opentelemetry-api, pynndescent, opentelemetry-sdk, opentelemetry-instrumentation, hdbscan, umap-learn, opentelemetry-exporter-otlp-proto-http, opentelemetry-exporter-otlp-proto-grpc, openinference-instrumentation-openai, openinference-instrumentation-llama-index, openinference-instrumentation-langchain, opentelemetry-exporter-otlp, arize-phoenix\n","  Attempting uninstall: importlib-metadata\n","    Found existing installation: importlib_metadata 7.0.2\n","    Uninstalling importlib_metadata-7.0.2:\n","      Successfully uninstalled importlib_metadata-7.0.2\n","  Attempting uninstall: cython\n","    Found existing installation: Cython 3.0.9\n","    Uninstalling Cython-3.0.9:\n","      Successfully uninstalled Cython-3.0.9\n","Successfully installed arize-phoenix-3.14.1 cython-0.29.37 ddsketch-2.0.4 deprecated-1.2.14 graphql-core-3.2.3 hdbscan-0.8.33 importlib-metadata-6.11.0 openinference-instrumentation-langchain-0.1.12 openinference-instrumentation-llama-index-1.2.0 openinference-instrumentation-openai-0.1.4 openinference-semantic-conventions-0.1.5 opentelemetry-api-1.23.0 opentelemetry-exporter-otlp-1.23.0 opentelemetry-exporter-otlp-proto-common-1.23.0 opentelemetry-exporter-otlp-proto-grpc-1.23.0 opentelemetry-exporter-otlp-proto-http-1.23.0 opentelemetry-instrumentation-0.44b0 opentelemetry-proto-1.23.0 opentelemetry-sdk-1.23.0 opentelemetry-semantic-conventions-0.44b0 pynndescent-0.5.11 starlette-0.37.2 strawberry-graphql-0.208.2 umap-learn-0.5.5 uvicorn-0.28.0\n"]}]},{"cell_type":"code","source":["from urllib.request import urlopen\n","\n","import nest_asyncio\n","import numpy as np\n","import pandas as pd\n","import phoenix as px\n","from langchain.chains import RetrievalQA\n","from langchain.chat_models import ChatOpenAI\n","from langchain.embeddings import OpenAIEmbeddings\n","from langchain.retrievers import KNNRetriever\n","from phoenix.experimental.evals import (\n","    HallucinationEvaluator,\n","    OpenAIModel,\n","    QAEvaluator,\n","    RelevanceEvaluator,\n","    run_evals,\n",")\n","from phoenix.session.evaluation import get_qa_with_reference, get_retrieved_documents\n","from phoenix.trace import DocumentEvaluations, SpanEvaluations\n","from phoenix.trace.langchain import LangChainInstrumentor\n","from tqdm import tqdm\n","\n","nest_asyncio.apply()  # needed for concurrent evals in notebook environments"],"metadata":{"id":"mDx-ckXTFtSf","executionInfo":{"status":"ok","timestamp":1710435175324,"user_tz":420,"elapsed":32743,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"4870f4e3-398f-4f43-ca4d-0f7749da553c"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["WARNING:phoenix.experimental.evals:Evals are moving out of experimental. Install the evals extra with `pip install arize-phoenix[evals]` and import `phoenix.evals`. For more info, see the [migration guide](https://github.com/Arize-ai/phoenix/blob/main/MIGRATION.md).\n"]}]},{"cell_type":"code","source":["import os\n","from google.cloud import secretmanager\n","from google.colab import auth\n","from google.colab import drive"],"metadata":{"id":"5Ca6p8tnd-wL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["#drive.mount('/content/gdrive')"],"metadata":{"id":"r3S2IMoa2e07"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def load_secrets(secrets_name, project_id):\n","  # Build a client\n","  auth.authenticate_user()\n","  client = secretmanager.SecretManagerServiceClient()\n","  secret_name = secrets_name\n","  # Create path to latest secret\n","  resource_name = f\"projects/{project_id}/secrets/{secret_name}/versions/latest\"\n","  # Get your secret :\n","  response = client.access_secret_version(request={\"name\": resource_name})\n","  secret_string = response.payload.data.decode('UTF-8')\n","  return secret_string"],"metadata":{"id":"sCa55hgW2ikm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["project_id = 'botchagalupep1'\n","openai_api_key = load_secrets(\"openai_api_key\",project_id)\n","os.environ['OPENAI_API_KEY'] = openai_api_key\n","MONGODB_ATLAS_CLUSTER_URI = load_secrets(\"MDB_CLUSTER0_URI\",project_id)\n","langsmith_api_key = load_secrets(\"langsmith_api_key\",project_id)"],"metadata":{"id":"mWn3YgC12nlg"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# connect notebook to langsmith\n","os.environ['LANGCHAIN_TRACING_V2'] = 'true'\n","os.environ['LANGCHAIN_ENDPOINT'] = 'https://api.smith.langchain.com'\n","\n","# This key is sourced from vars.env\n","os.environ['LANGCHAIN_API_KEY'] = langsmith_api_key  # Uncomment and replace  with your actual API key\n","\n","os.environ['LANGCHAIN_PROJECT'] = 'opencontext-1'\n","\n","# To verify, you can print the variables\n","print(os.environ.get('LANGCHAIN_TRACING_V2'))\n","print(os.environ.get('LANGCHAIN_ENDPOINT'))\n","#print(os.environ.get('LANGCHAIN_API_KEY'))  # Uncomment if you want to print your API key (be careful with sharing your notebook)\n","print(os.environ.get('LANGCHAIN_PROJECT'))"],"metadata":{"id":"NpU8CL2r22vD","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1710435214355,"user_tz":420,"elapsed":2,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"3ece5200-018f-48cf-ccc3-e478eb0d8642"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["true\n","https://api.smith.langchain.com\n","opencontext-1\n"]}]},{"cell_type":"code","source":["from pymongo import MongoClient\n","\n","# initialize MongoDB python client\n","client = MongoClient(MONGODB_ATLAS_CLUSTER_URI)\n","\n","DB_NAME = \"Cluster0\"\n","COLLECTION_NAME = \"OpenContext0\"\n","ATLAS_VECTOR_SEARCH_INDEX_NAME = \"vector_index\"\n","\n","MONGODB_COLLECTION = client[DB_NAME][COLLECTION_NAME]"],"metadata":{"id":"aPqh0syziQ2k"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from langchain_community.vectorstores import MongoDBAtlasVectorSearch\n","from langchain_openai import OpenAIEmbeddings\n","\n","vector_search = MongoDBAtlasVectorSearch.from_connection_string(\n","    MONGODB_ATLAS_CLUSTER_URI,\n","    DB_NAME + \".\" + COLLECTION_NAME,\n","    OpenAIEmbeddings(model=\"text-embedding-3-small\",dimensions=1536,disallowed_special=()),\n","    index_name=ATLAS_VECTOR_SEARCH_INDEX_NAME,\n",")"],"metadata":{"id":"7himXISricVE"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["#Similarity Search with Score"],"metadata":{"id":"jddBa70YjpAt"}},{"cell_type":"code","source":["px.close_app()\n","session = px.launch_app()\n","LangChainInstrumentor().instrument()"],"metadata":{"id":"dYv2b5P1F95o","colab":{"base_uri":"https://localhost:8080/","height":88},"executionInfo":{"status":"ok","timestamp":1710435216328,"user_tz":420,"elapsed":917,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"53a2d000-7d88-49df-ee94-5f324685af59"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["No active session to close\n","🌍 To view the Phoenix app in your browser, visit https://itieumewbv1-496ff2e9c6d22116-6006-colab.googleusercontent.com/\n","📺 To view the Phoenix app in a notebook, run `px.active_session().view()`\n","📖 For more information on how to use Phoenix, check out https://docs.arize.com/phoenix\n"]}]},{"cell_type":"markdown","source":[],"metadata":{"id":"LJAL5t9N0uqa"}},{"cell_type":"markdown","source":[],"metadata":{"id":"8fsP9JtfWkqB"}},{"cell_type":"markdown","source":["#Question Answering"],"metadata":{"id":"43GA0yPQjw6T"}},{"cell_type":"markdown","source":["## https://github.com/Arize-ai/phoenix/tree/main/tutorials"],"metadata":{"id":"GUyp708odpt8"}},{"cell_type":"markdown","source":["# Use this example to evaluate your model"],"metadata":{"id":"MbmCqoDzKEtA"}},{"cell_type":"code","source":["queries = [\n","    'What is a CodeCompnent',\n","    'What is a SaaS User',\n","    'What is a Service',\n","    'What is a Location',\n","    'What is a Datacenter',\n","    'List all the entities',\n","    'List code examples',\n","    'List yaml examples names',\n","    'list the yaml example crates-erp'\n","]\n","\n","from langchain.chat_models import ChatOpenAI\n","\n","llm = ChatOpenAI(model_name=\"gpt-3.5-turbo-16k\", temperature=0, max_tokens=5000)\n","\n","qa_chain = RetrievalQA.from_chain_type(\n","    llm, retriever=vector_search.as_retriever(), return_source_documents=True)\n","\n","for query in queries:\n","  answer = qa_chain({\"query\": query})\n","  print(\"Query:\", answer['query'])\n","  print(\"Result:\", answer['result'])"],"metadata":{"id":"GJYM9rSm1CWm","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1710435258915,"user_tz":420,"elapsed":42588,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"d78ab497-1eeb-4bf6-98ec-1a2fe187de85"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The class `langchain_community.chat_models.openai.ChatOpenAI` was deprecated in langchain-community 0.0.10 and will be removed in 0.2.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import ChatOpenAI`.\n","  warn_deprecated(\n","/usr/local/lib/python3.10/dist-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The function `__call__` was deprecated in LangChain 0.1.0 and will be removed in 0.2.0. Use invoke instead.\n","  warn_deprecated(\n"]},{"output_type":"stream","name":"stdout","text":["Query: What is a CodeCompnent\n","Result: A CodeComponent is a software package, library, or other code entity. It is used to describe code-related components in a system. It includes metadata such as name, description, owner, and dependencies, as well as specifications like type, service, and URI. CodeComponents can also have tags, links, and annotations to provide additional information and context.\n","Query: What is a SaaS User\n","Result: A SaaS User refers to a user who utilizes a Software-as-a-Service (SaaS) application. SaaS is a cloud computing model where software is provided as a service over the internet, allowing users to access and use the software through a web browser without the need for installation or maintenance. A SaaS user typically has a registered account with the SaaS provider and can log in to access the features and functionality of the application.\n","Query: What is a Service\n","Result: A Service is a kind of entity that groups together Code Components, Platform Components, and Aux Components. It provides an abstraction layer that allows for insights into the exposed features without requiring too many details about the structure of all components. A Service can have an owner, which is typically a team or a person, and it can also belong to a specific platform. Additionally, a Service can have other components associated with it, such as aux components, platform components, and code components.\n","Query: What is a Location\n","Result: A Location is a marker that references other places to look for OpenContext data. It is a special kind of entity that is used to add supplementary data or entities that cannot be auto-discovered. It contains a list of targets, which can be absolute paths/URLs or relative paths, to specify where to find the OpenContext data.\n","Query: What is a Datacenter\n","Result: A datacenter is a physical or virtual facility that houses computer systems and associated components, such as servers, storage devices, networking equipment, and security devices. It provides the necessary infrastructure and resources for organizations to store, process, manage, and distribute large amounts of data and run their applications and services. Datacenters can be located on-premise within an organization's own facilities or in the cloud, provided by third-party service providers like Amazon Web Services (AWS) or Google Cloud Platform (GCP).\n","Query: List all the entities\n","Result: I'm sorry, but I don't have access to the specific entities in your context. Can you provide more information or specific entities that you would like me to list?\n","Query: List code examples\n","Result: Here are some code examples:\n","\n","1. Crates ERP:\n","```yaml\n","apiVersion: opencontext.com/v1alpha1\n","kind: CodeComponent\n","metadata:\n","  name: crates-erp\n","  description: Distribution Center ERP\n","  title: Crates ERP\n","spec:\n","  type: library\n","  owner: [eng-squirrel]\n","  dependsOn: [code-component:greenhouse-data]\n","  uri: https://example.com/erp\n","```\n","\n","2. Crates Frontend:\n","```yaml\n","apiVersion: opencontext.com/v1alpha1\n","kind: CodeComponent\n","metadata:\n","  name: crates-fe\n","  description: Crates frontend application\n","  title: Crates Frontend\n","spec:\n","  type: package\n","  owner: [eng-squirrel]\n","  service: crates\n","  subcomponentOf: [code-component:crates-erp]\n","  uri: https://example.com/order\n","```\n","\n","3. Dumpster Data Backend:\n","```yaml\n","apiVersion: opencontext.com/v1alpha1\n","kind: CodeComponent\n","metadata:\n","  name: dumpster-data\n","  description: Dumpster data backend\n","  title: Dumpster data\n","spec:\n","  type: repository\n","  owner: [eng-raccoon-aws, eng-raccoon-eco]\n","  uri: https://github.com/example/greenhouse-data\n","```\n","\n","4. Log4j Library:\n","```yaml\n","apiVersion: opencontext.com/v1alpha1\n","kind: CodeComponent\n","metadata:\n","  name: log4j\n","  description: Log4j library\n","  title: Log4j\n","spec:\n","  type: library\n","  owner: [eng-raccoon-aws, eng-raccoon-eco]\n","  service: dumpster-prod\n","  uri: https://github.com/example/greenhouse-data\n","```\n","\n","5. Greenhouse Data Archive:\n","```yaml\n","apiVersion: opencontext.com/v1alpha1\n","kind: CodeComponent\n","metadata:\n","  name: greenhouse-dataset\n","  description: Greenhouse historical data\n","  title: Greenhouse data archive\n","spec:\n","  type: other\n","  otherType: dataset\n","  owner: [prod-squirrel]\n","  service: greenhouse\n","  uri: s3://greenhouse-dataset\n","  dependsOn: [code-component:greenhouse-data]\n","  datacenter: [aws]\n","```\n","\n","Please let me know if you need more examples or if there's anything else I can help with!\n","Query: List yaml examples names\n","Result: The YAML examples provided are named as follows:\n","\n","1. Platform: blue-sky\n","2. Platform: orchard\n","3. Person: paul\n","4. AuxComponent: crates-erp-website\n","5. AuxComponent: crates-docs\n","6. AuxComponent: map-thingy\n","7. AuxComponent: custom-ticketing\n","8. AuxComponent: eng-raccoon-info\n","9. User: paul\n","Query: list the yaml example crates-erp\n","Result: Here is the YAML example for the `crates-erp` component:\n","\n","```yaml\n","apiVersion: opencontext.com/v1alpha1\n","kind: CodeComponent\n","metadata:\n","  name: crates-erp\n","  description: Distribution Center ERP\n","  title: Crates ERP\n","annotations:\n","  github.com/project-slug: scatterly/crates-erp\n","tags:\n","  - java\n","links:\n","  - url: https://cybersquirrel1.com/\n","    title: Cyber Squirrel\n","    icon: dashboard\n","  - url: https://example.com/user\n","    title: Examples Users\n","    icon: user\n","  - url: https://example.com/group\n","    title: Example Group\n","    icon: group\n","  - url: https://example.com/cloud\n","    title: Link with Cloud Icon\n","    icon: cloud\n","  - url: https://example.com/dashboard\n","    title: Dashboard\n","    icon: dashboard\n","  - url: https://example.com/help\n","    title: Support\n","    icon: help\n","  - url: https://example.com/web\n","    title: Website\n","    icon: web\n","  - url: https://example.com/alert\n","    title: Alerts\n","    icon: alert\n","spec:\n","  type: library\n","  owner: [eng-squirrel]\n","  dependsOn: [code-component:greenhouse-data]\n","  uri: https://example.com/erp\n","```\n","\n","Let me know if you need any further assistance!\n"]}]},{"cell_type":"code","source":["queries_df = get_qa_with_reference(px.Client())\n","retrieved_documents_df = get_retrieved_documents(px.Client())"],"metadata":{"id":"wlBbSyHyQ1Fy"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["eval_model = OpenAIModel(\n","    model_name=\"gpt-4-turbo-preview\",\n",")\n","hallucination_evaluator = HallucinationEvaluator(eval_model)\n","qa_correctness_evaluator = QAEvaluator(eval_model)\n","relevance_evaluator = RelevanceEvaluator(eval_model)\n","\n","hallucination_eval_df, qa_correctness_eval_df = run_evals(\n","    dataframe=queries_df,\n","    evaluators=[hallucination_evaluator, qa_correctness_evaluator],\n","    provide_explanation=True,\n",")\n","relevance_eval_df = run_evals(\n","    dataframe=retrieved_documents_df,\n","    evaluators=[relevance_evaluator],\n","    provide_explanation=True,\n",")[0]\n","\n","px.Client().log_evaluations(\n","    SpanEvaluations(eval_name=\"Hallucination\", dataframe=hallucination_eval_df),\n","    SpanEvaluations(eval_name=\"QA Correctness\", dataframe=qa_correctness_eval_df),\n","    DocumentEvaluations(eval_name=\"Relevance\", dataframe=relevance_eval_df),\n",")"],"metadata":{"id":"Tr_nRl46RCg5","colab":{"base_uri":"https://localhost:8080/","height":99,"referenced_widgets":["96cb24bab89e4c1aad531ccd82d802d7","bbe758eaae0941d28d1cc253a6bc0d91","b4c1dad3f66447bc80cefc02d07b70ff","e05eb92d5b744d26859add573c3c6c89","0c2b78cf01244fbe952e018573945abd","249d08db554b4cccbc90bca485bd19df","0e635f108be3460baf24511f6ac2e2f8","92c6140c6376408abd85a4b6574b61f8","af2df273f1f640df818981293f49f351","49056b127534493cb4c75261beac7e50","c0a97a26d6ec46dd9c20f13b4489a33d","cfe8f751ec884953875987520292a4e8","7b1f2fda8793450f9acb18cf220eb604","c8d2d0bb80cc47308c2cd63246f223ab","91cde2e70a8042f7b2066d13b54c8ba1","b47040fdaae7428b90073b418b810cab","93949f65278a42138d9c84a0e656d854","9ce2eb3f10bd4cb09d407a8cc0023a28","59c551bb1e53417795faed73fa63d186","8bce30771eab420f90b347243070c277","7aa5a9456c814b9b9760de59c3973e2b","6de72b48ba264c61b5f60a71f0715a7b"]},"executionInfo":{"status":"ok","timestamp":1710435396130,"user_tz":420,"elapsed":129547,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"dadf6f58-2991-4688-a512-fd2bbec0345a"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["The `model_name` field is deprecated. Use `model` instead.                 This will be removed in a future release.\n"]},{"output_type":"display_data","data":{"text/plain":["run_evals |          | 0/18 (0.0%) | ⏳ 00:00<? | ?it/s"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"96cb24bab89e4c1aad531ccd82d802d7"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["run_evals |          | 0/36 (0.0%) | ⏳ 00:00<? | ?it/s"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cfe8f751ec884953875987520292a4e8"}},"metadata":{}}]},{"cell_type":"code","source":["px.Client().get_evaluations()\n"],"metadata":{"id":"nOxOVwKJ-l7Z"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["trace_df = px.Client().get_spans_dataframe()\n","trace_df"],"metadata":{"id":"llFZb_OC0xoT"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import pandas as pd\n","from phoenix.trace.dsl import SpanQuery\n","\n","query_for_root_span = SpanQuery().where(\n","    \"parent_id is None\",   # Filter for root spans\n",").select(\n","    input=\"input.value\",   # Input contains the user's question\n","    output=\"output.value\", # Output contains the LLM's answer\n",")\n","\n","query_for_retrieved_documents = SpanQuery().where(\n","    \"span_kind == 'RETRIEVER'\",  # Filter for RETRIEVER span\n",").select(\n","    # Rename parent_id as span_id. This turns the parent_id\n","    # values into the index of the output dataframe.\n","    span_id=\"parent_id\",\n",").concat(\n","    \"retrieval.documents\",\n","    reference=\"document.content\",\n",")\n","\n","# Perform an inner join on the two sets of spans.\n","pd.concat(\n","    px.Client().query_spans(\n","        query_for_root_span,\n","        query_for_retrieved_documents,\n","    ),\n","    axis=1,\n","    join=\"inner\",\n",")"],"metadata":{"id":"oLXsPkkX02FA"},"execution_count":null,"outputs":[]}]}