From 4af30e4ff0505cd4865ea7bedb6a5a84916b5720 Mon Sep 17 00:00:00 2001
From: shanurrahman <shanur.cse.nitap@gmail.com>
Date: Mon, 4 Dec 2023 23:48:14 +0300
Subject: [PATCH] Adding some nullability checks

---
 llm-server/routes/_swagger/service.py       | 75 ++++++++++++---------
 llm-server/workers/tasks/reindex_swagger.py | 20 ++++--
 2 files changed, 57 insertions(+), 38 deletions(-)

diff --git a/llm-server/routes/_swagger/service.py b/llm-server/routes/_swagger/service.py
index e7d358098..ac5aa0feb 100644
--- a/llm-server/routes/_swagger/service.py
+++ b/llm-server/routes/_swagger/service.py
@@ -32,37 +32,50 @@ def save_swaggerfile_to_mongo(
 
 def save_swagger_paths_to_qdrant(swagger_doc: ResolvingParser, bot_id: str):
     vector_store = get_vector_store(StoreOptions("apis"))
-
-    # delete documents with metadata in api with the current bot id, before reingesting
-    documents: List[Document] = []
-    paths = swagger_doc.specification.get("paths")
-    for path in paths:
-        operations = paths[path]
-        for method in operations:
-            operation = operations[method]
-            operation["method"] = method
-            operation["path"] = path
-            del operation["responses"]
-            document = Document(
-                page_content=f"{operation['summary']}; {operation['description']}"
-            )
-            document.metadata["bot_id"] = bot_id
-            document.metadata["operation"] = operation
-
-            logger.info(
-                "document before ingestion ---",
-                incident="ingestion_doc",
-                data=document.page_content,
-            )
-            documents.append(document)
-
-    point_ids = vector_store.add_documents(documents)
-    logger.info(
-        "API ingestion for Qdrant",
-        incident="api_ingestion_qdrant",
-        point_ids=point_ids,
-    )
-
+    try:
+        # delete documents with metadata in api with the current bot id, before reingesting
+        documents: List[Document] = []
+        paths = swagger_doc.specification.get("paths", {})
+        
+        for path, operations in paths.items():
+            for method, operation in operations.items():
+                try:
+                    operation["method"] = method
+                    operation["path"] = path
+                    del operation["responses"]
+                    
+                    # Check if "summary" key is present before accessing it
+                    summary = operation.get('summary', '')
+                    description = operation.get('description', '')
+                    
+                    document = Document(
+                        page_content=f"{summary}; {description}"
+                    )
+                    document.metadata["bot_id"] = bot_id
+                    document.metadata["operation"] = operation
+
+                    logger.info(
+                        "document before ingestion ---",
+                        incident="ingestion_doc",
+                        data=document.page_content,
+                    )
+                    documents.append(document)
+                except KeyError as e:
+                    # Handle the specific key error, log, or take necessary action
+                    logger.error(f"KeyError in processing document: {e}")
+
+        point_ids = vector_store.add_documents(documents)
+        logger.info(
+            "API ingestion for Qdrant",
+            incident="api_ingestion_qdrant",
+            point_ids=point_ids,
+        )
+    except KeyError as e:
+        # Handle the specific key error at a higher level if needed
+        logger.error(f"KeyError in processing paths: {e}")
+    except Exception as e:
+        # Handle other exceptions
+        logger.error(f"An error occurred: {e}")
 
 def add_swagger_file(request: Request, id: str) -> Dict[str, str]:
     if request.content_type == "application/json":
diff --git a/llm-server/workers/tasks/reindex_swagger.py b/llm-server/workers/tasks/reindex_swagger.py
index 83421bd5d..d80c8d390 100644
--- a/llm-server/workers/tasks/reindex_swagger.py
+++ b/llm-server/workers/tasks/reindex_swagger.py
@@ -47,7 +47,7 @@ def process_swagger_file(chatbot: Chatbot):
 
         swagger_doc = ResolvingParser(url=swagger_url)
 
-        points = client.scroll(
+        scroll_result = client.scroll(
             collection_name="apis",
             scroll_filter=models.Filter(
                 must_not=[
@@ -55,7 +55,7 @@ def process_swagger_file(chatbot: Chatbot):
                         must=[
                             models.FieldCondition(
                                 key="metadata.bot_id",
-                                match=models.MatchValue(value="bot_id"),
+                                match=models.MatchValue(value=str(chatbot.id)),
                             )
                         ],
                     ),
@@ -63,13 +63,19 @@ def process_swagger_file(chatbot: Chatbot):
             ),
         )
 
-        if points is None:
-            logger.info("Points not found for bot, we will reindex", bot_id=bot_id)
+        if scroll_result is None:
+            logger.info("scroll_result is None. No records found, we will reindex")
             save_swagger_paths_to_qdrant(swagger_doc=swagger_doc, bot_id=bot_id)
-            time.sleep(3)
-
         else:
-            logger.info(f"points already exist for bot : {bot_id}")
+            records, point_id = scroll_result
+
+            if len(records)==0:
+                logger.info("Points not found for bot, we will reindex", bot_id=bot_id)
+                save_swagger_paths_to_qdrant(swagger_doc=swagger_doc, bot_id=bot_id)
+                time.sleep(3)
+
+            else:
+                logger.info(f"points already exist for bot : {bot_id}")
 
     except KeyError as e:
         print(f"Error: Missing key in swagger_file - {e}")