Merge pull request #597 from ScrapeGraphAI/593-abstract-graph-fix-rou…

…nd-4 Abbstract graph fix, round 4
ScrapeGraphAI · Aug 27, 2024 · d45064b · d45064b
2 parents d33b347 + 229d74d
commit d45064b
Show file tree

Hide file tree

Showing 102 changed files with 262 additions and 352 deletions.
diff --git a/examples/anthropic/csv_scraper_graph_multi_haiku.py b/examples/anthropic/csv_scraper_graph_multi_haiku.py
@@ -26,8 +26,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000},
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

diff --git a/examples/anthropic/csv_scraper_haiku.py b/examples/anthropic/csv_scraper_haiku.py
@@ -32,9 +32,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

diff --git a/examples/anthropic/custom_graph_haiku.py b/examples/anthropic/custom_graph_haiku.py
@@ -5,10 +5,9 @@
 import os
 from dotenv import load_dotenv
 
-from langchain_openai import OpenAIEmbeddings
-from langchain_openai import ChatOpenAI
+from langchain_anthropic import ChatAnthropic
 from scrapegraphai.graphs import BaseGraph
-from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode
+from scrapegraphai.nodes import FetchNode, ParseNode, GenerateAnswerNode, RobotsNode
 load_dotenv()
 
 # ************************************************
@@ -19,16 +18,14 @@
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
         "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+    },
 }
 
 # ************************************************
 # Define the graph nodes
 # ************************************************
 
-llm_model = OpenAI(graph_config["llm"])
-embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key)
+llm_model = ChatAnthropic(graph_config["llm"])
 
 # define the nodes for the graph
 robot_node = RobotsNode(
@@ -57,15 +54,6 @@
         "verbose": True,
     }
 )
-rag_node = RAGNode(
-    input="user_prompt & (parsed_doc | doc)",
-    output=["relevant_chunks"],
-    node_config={
-        "llm_model": llm_model,
-        "embedder_model": embedder,
-        "verbose": True,
-    }
-)
 generate_answer_node = GenerateAnswerNode(
     input="user_prompt & (relevant_chunks | parsed_doc | doc)",
     output=["answer"],
@@ -84,14 +72,12 @@
         robot_node,
         fetch_node,
         parse_node,
-        rag_node,
         generate_answer_node,
     ],
     edges=[
         (robot_node, fetch_node),
         (fetch_node, parse_node),
-        (parse_node, rag_node),
-        (rag_node, generate_answer_node)
+        (parse_node, generate_answer_node)
     ],
     entry_point=robot_node
 )

diff --git a/examples/anthropic/json_scraper_haiku.py b/examples/anthropic/json_scraper_haiku.py
@@ -26,9 +26,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

diff --git a/examples/anthropic/json_scraper_multi_haiku.py b/examples/anthropic/json_scraper_multi_haiku.py
@@ -11,9 +11,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 FILE_NAME = "inputs/example.json"

diff --git a/examples/anthropic/pdf_scraper_graph_haiku.py b/examples/anthropic/pdf_scraper_graph_haiku.py
@@ -14,9 +14,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 source = """

diff --git a/examples/anthropic/pdf_scraper_multi_haiku.py b/examples/anthropic/pdf_scraper_multi_haiku.py
@@ -11,9 +11,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ***************

diff --git a/examples/anthropic/scrape_plain_text_haiku.py b/examples/anthropic/scrape_plain_text_haiku.py
@@ -28,9 +28,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

diff --git a/examples/anthropic/script_generator_haiku.py b/examples/anthropic/script_generator_haiku.py
@@ -16,9 +16,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

diff --git a/examples/anthropic/script_multi_generator_haiku.py b/examples/anthropic/script_multi_generator_haiku.py
@@ -16,10 +16,9 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
-        "library": "beautifulsoup"
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
+    "library": "beautifulsoup"
 }
 
 # ************************************************

diff --git a/examples/anthropic/search_graph_haiku.py b/examples/anthropic/search_graph_haiku.py
@@ -15,9 +15,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

diff --git a/examples/anthropic/search_graph_schema_haiku.py b/examples/anthropic/search_graph_schema_haiku.py
@@ -27,8 +27,8 @@ class Dishes(BaseModel):
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000},
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

diff --git a/examples/anthropic/search_link_graph_haiku.py b/examples/anthropic/search_link_graph_haiku.py
@@ -14,23 +14,11 @@
 
 load_dotenv()
 
-llm_model_instance = AzureChatOpenAI(
-    openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
-    azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]
-)
-
-embedder_model_instance = AzureOpenAIEmbeddings(
-    azure_deployment=os.environ["AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"],
-    openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
-)
-
-# ************************************************
-# Create the SmartScraperGraph instance and run it
-# ************************************************
-
 graph_config = {
-    "llm": {"model_instance": llm_model_instance},
-    "embeddings": {"model_instance": embedder_model_instance}
+    "llm": {
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

diff --git a/examples/anthropic/smart_scraper_haiku.py b/examples/anthropic/smart_scraper_haiku.py
@@ -19,9 +19,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 smart_scraper_graph = SmartScraperGraph(

diff --git a/examples/anthropic/smart_scraper_multi_haiku.py b/examples/anthropic/smart_scraper_multi_haiku.py
@@ -17,9 +17,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # *******************************************************

diff --git a/examples/anthropic/smart_scraper_schema_haiku.py b/examples/anthropic/smart_scraper_schema_haiku.py
@@ -33,8 +33,8 @@ class Projects(BaseModel):
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000},
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 smart_scraper_graph = SmartScraperGraph(

diff --git a/examples/anthropic/xml_scraper_graph_multi_haiku.py b/examples/anthropic/xml_scraper_graph_multi_haiku.py
@@ -26,8 +26,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000},
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

diff --git a/examples/anthropic/xml_scraper_haiku.py b/examples/anthropic/xml_scraper_haiku.py
@@ -26,9 +26,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

diff --git a/examples/deepseek/csv_scraper_deepseek.py b/examples/deepseek/csv_scraper_deepseek.py
@@ -27,7 +27,7 @@
 
 graph_config = {
     "llm": {
-        "model": "deepseek-chat",
+        "model": "deepseek/deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
     },

diff --git a/examples/deepseek/csv_scraper_graph_multi_deepseek.py b/examples/deepseek/csv_scraper_graph_multi_deepseek.py
@@ -27,7 +27,7 @@
 
 graph_config = {
     "llm": {
-        "model": "deepseek-chat",
+        "model": "deepseek/deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
     },

diff --git a/examples/deepseek/json_scraper_deepseek.py b/examples/deepseek/json_scraper_deepseek.py
@@ -26,7 +26,7 @@
 
 graph_config = {
     "llm": {
-        "model": "deepseek-chat",
+        "model": "deepseek/deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
     },

diff --git a/examples/deepseek/json_scraper_multi_deepseek.py b/examples/deepseek/json_scraper_multi_deepseek.py
@@ -12,7 +12,7 @@
 
 graph_config = {
     "llm": {
-        "model": "deepseek-chat",
+        "model": "deepseek/deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
     },

diff --git a/examples/deepseek/pdf_scraper_graph_deepseek.py b/examples/deepseek/pdf_scraper_graph_deepseek.py
@@ -17,7 +17,7 @@
 
 graph_config = {
     "llm": {
-        "model": "deepseek-chat",
+        "model": "deepseek/deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
     },

diff --git a/examples/deepseek/pdf_scraper_multi_deepseek.py b/examples/deepseek/pdf_scraper_multi_deepseek.py
@@ -12,7 +12,7 @@
 
 graph_config = {
     "llm": {
-        "model": "deepseek-chat",
+        "model": "deepseek/deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
     },

diff --git a/examples/deepseek/scrape_plain_text_deepseek.py b/examples/deepseek/scrape_plain_text_deepseek.py
@@ -28,7 +28,7 @@
 
 graph_config = {
     "llm": {
-        "model": "deepseek-chat",
+        "model": "deepseek/deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
     },

diff --git a/examples/deepseek/script_generator_deepseek.py b/examples/deepseek/script_generator_deepseek.py
@@ -17,7 +17,7 @@
 
 graph_config = {
     "llm": {
-        "model": "deepseek-chat",
+        "model": "deepseek/deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
     },

diff --git a/examples/deepseek/script_multi_generator_deepseek.py b/examples/deepseek/script_multi_generator_deepseek.py
@@ -17,7 +17,7 @@
 
 graph_config = {
     "llm": {
-        "model": "deepseek-chat",
+        "model": "deepseek/deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
     },

diff --git a/examples/deepseek/search_graph_deepseek.py b/examples/deepseek/search_graph_deepseek.py
@@ -15,7 +15,7 @@
 
 graph_config = {
     "llm": {
-        "model": "deepseek-chat",
+        "model": "deepseek/deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
     },

diff --git a/examples/deepseek/search_graph_schema_deepseek.py b/examples/deepseek/search_graph_schema_deepseek.py
@@ -31,7 +31,7 @@ class Dishes(BaseModel):
 
 graph_config = {
     "llm": {
-        "model": "deepseek-chat",
+        "model": "deepseek/deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
     },

diff --git a/examples/deepseek/search_link_graph_deepseek.py b/examples/deepseek/search_link_graph_deepseek.py
@@ -16,7 +16,7 @@
 
 graph_config = {
     "llm": {
-        "model": "deepseek-chat",
+        "model": "deepseek/deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
     },

diff --git a/examples/deepseek/smart_scraper_deepseek.py b/examples/deepseek/smart_scraper_deepseek.py
@@ -18,7 +18,7 @@
 
 graph_config = {
     "llm": {
-        "model": "deepseek-chat",
+        "model": "deepseek/deepseek-chat",
         "openai_api_key": deepseek_key,
         "openai_api_base": 'https://api.deepseek.com/v1',
     },