diff --git a/.env.template b/.env.template
index 5c4948310..1183505d9 100644
--- a/.env.template
+++ b/.env.template
@@ -78,6 +78,8 @@ KNOWLEDGE_SEARCH_TOP_SIZE=5
#KNOWLEDGE_CHUNK_OVERLAP=50
# Control whether to display the source document of knowledge on the front end.
KNOWLEDGE_CHAT_SHOW_RELATIONS=False
+# Whether to enable Chat Knowledge Search Rewrite Mode
+KNOWLEDGE_SEARCH_REWRITE=False
## EMBEDDING_TOKENIZER - Tokenizer to use for chunking large inputs
## EMBEDDING_TOKEN_LIMIT - Chunk size limit for large inputs
# EMBEDDING_MODEL=all-MiniLM-L6-v2
@@ -92,16 +94,15 @@ KNOWLEDGE_CHAT_SHOW_RELATIONS=False
#*******************************************************************#
-#** DATABASE SETTINGS **#
+#** DB-GPT METADATA DATABASE SETTINGS **#
#*******************************************************************#
### SQLite database (Current default database)
-LOCAL_DB_PATH=data/default_sqlite.db
LOCAL_DB_TYPE=sqlite
### MYSQL database
# LOCAL_DB_TYPE=mysql
# LOCAL_DB_USER=root
-# LOCAL_DB_PASSWORD=aa12345678
+# LOCAL_DB_PASSWORD={your_password}
# LOCAL_DB_HOST=127.0.0.1
# LOCAL_DB_PORT=3306
# LOCAL_DB_NAME=dbgpt
diff --git a/docker/examples/metadata/duckdb2mysql.py b/docker/examples/metadata/duckdb2mysql.py
new file mode 100644
index 000000000..40d68be78
--- /dev/null
+++ b/docker/examples/metadata/duckdb2mysql.py
@@ -0,0 +1,54 @@
+import duckdb
+import pymysql
+
+""" migrate duckdb to mysql"""
+
+mysql_config = {
+ "host": "127.0.0.1",
+ "user": "root",
+ "password": "your_password",
+ "db": "dbgpt",
+ "charset": "utf8mb4",
+ "cursorclass": pymysql.cursors.DictCursor,
+}
+
+duckdb_files_to_tables = {
+ "pilot/message/chat_history.db": "chat_history",
+ "pilot/message/connect_config.db": "connect_config",
+}
+
+conn_mysql = pymysql.connect(**mysql_config)
+
+
+def migrate_table(duckdb_file_path, source_table, destination_table, conn_mysql):
+ conn_duckdb = duckdb.connect(duckdb_file_path)
+ try:
+ cursor = conn_duckdb.cursor()
+ cursor.execute(f"SELECT * FROM {source_table}")
+ column_names = [
+ desc[0] for desc in cursor.description if desc[0].lower() != "id"
+ ]
+ select_columns = ", ".join(column_names)
+
+ cursor.execute(f"SELECT {select_columns} FROM {source_table}")
+ results = cursor.fetchall()
+
+ with conn_mysql.cursor() as cursor_mysql:
+ for row in results:
+ placeholders = ", ".join(["%s"] * len(row))
+ insert_query = f"INSERT INTO {destination_table} ({', '.join(column_names)}) VALUES ({placeholders})"
+ cursor_mysql.execute(insert_query, row)
+ conn_mysql.commit()
+ finally:
+ conn_duckdb.close()
+
+
+try:
+ for duckdb_file, table in duckdb_files_to_tables.items():
+ print(f"Migrating table {table} from {duckdb_file}...")
+ migrate_table(duckdb_file, table, table, conn_mysql)
+ print(f"Table {table} migrated successfully.")
+finally:
+ conn_mysql.close()
+
+print("Migration completed.")
diff --git a/docker/examples/metadata/duckdb2sqlite.py b/docker/examples/metadata/duckdb2sqlite.py
new file mode 100644
index 000000000..cfc269d28
--- /dev/null
+++ b/docker/examples/metadata/duckdb2sqlite.py
@@ -0,0 +1,48 @@
+import duckdb
+import sqlite3
+
+""" migrate duckdb to sqlite"""
+
+duckdb_files_to_tables = {
+ "pilot/message/chat_history.db": "chat_history",
+ "pilot/message/connect_config.db": "connect_config",
+}
+
+sqlite_db_path = "pilot/meta_data/dbgpt.db"
+
+conn_sqlite = sqlite3.connect(sqlite_db_path)
+
+
+def migrate_table(duckdb_file_path, source_table, destination_table, conn_sqlite):
+ conn_duckdb = duckdb.connect(duckdb_file_path)
+ try:
+ cursor_duckdb = conn_duckdb.cursor()
+ cursor_duckdb.execute(f"SELECT * FROM {source_table}")
+ column_names = [
+ desc[0] for desc in cursor_duckdb.description if desc[0].lower() != "id"
+ ]
+ select_columns = ", ".join(column_names)
+
+ cursor_duckdb.execute(f"SELECT {select_columns} FROM {source_table}")
+ results = cursor_duckdb.fetchall()
+
+ cursor_sqlite = conn_sqlite.cursor()
+ for row in results:
+ placeholders = ", ".join(["?"] * len(row))
+ insert_query = f"INSERT INTO {destination_table} ({', '.join(column_names)}) VALUES ({placeholders})"
+ cursor_sqlite.execute(insert_query, row)
+ conn_sqlite.commit()
+ cursor_sqlite.close()
+ finally:
+ conn_duckdb.close()
+
+
+try:
+ for duckdb_file, table in duckdb_files_to_tables.items():
+ print(f"Migrating table {table} from {duckdb_file} to SQLite...")
+ migrate_table(duckdb_file, table, table, conn_sqlite)
+ print(f"Table {table} migrated to SQLite successfully.")
+finally:
+ conn_sqlite.close()
+
+print("Migration to SQLite completed.")
diff --git a/docs/docs/application/advanced_tutorial/rag.md b/docs/docs/application/advanced_tutorial/rag.md
index 49fb831f7..1cfeab5c2 100644
--- a/docs/docs/application/advanced_tutorial/rag.md
+++ b/docs/docs/application/advanced_tutorial/rag.md
@@ -1 +1,111 @@
-# RAG Parameter Adjustment
\ No newline at end of file
+# RAG Parameter Adjustment
+Each knowledge space supports argument customization, including the relevant arguments for vector retrieval and the arguments for knowledge question-answering prompts.
+
+As shown in the figure below, clicking on the "Knowledge" will trigger a pop-up dialog box. Click the "Arguments" button to enter the parameter tuning interface.
+
+
+
+
+
+
+
+
+:::tip Embedding Arguments
+* topk:the top k vectors based on similarity score.
+* recall_score:set a similarity threshold score for the retrieval of similar vectors. between 0 and 1. default 0.3.
+* recall_type:recall type. now nly support topk by vector similarity.
+* model:A model used to create vector representations of text or other data.
+* chunk_size:The size of the data chunks used in processing.default 500.
+* chunk_overlap:The amount of overlap between adjacent data chunks.default 50.
+:::
+
+
+
+
+
+
+:::tip Prompt Arguments
+* scene:A contextual parameter used to define the setting or environment in which the prompt is being used.
+* template:A pre-defined structure or format for the prompt, which can help ensure that the AI system generates responses that are consistent with the desired style or tone.
+* max_token:The maximum number of tokens or words allowed in a prompt.
+:::
+
+
+
+
+
+
+
+:::tip summary arguments
+* max_iteration: summary max iteration call with llm, default 5. the bigger and better for document summary but time will cost longer.
+* concurrency_limit: default summary concurrency call with llm, default 3.
+:::
+
+
+
+
+
+# Knowledge Query Rewrite
+set ``KNOWLEDGE_SEARCH_REWRITE=True`` in ``.env`` file, and restart the server.
+
+```shell
+# Whether to enable Chat Knowledge Search Rewrite Mode
+KNOWLEDGE_SEARCH_REWRITE=True
+```
+
+# Change Vector Database
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+
+
+
+set ``VECTOR_STORE_TYPE`` in ``.env`` file.
+
+```shell
+### Chroma vector db config
+VECTOR_STORE_TYPE=Chroma
+#CHROMA_PERSIST_PATH=/root/DB-GPT/pilot/data
+```
+
+
+
+
+
+set ``VECTOR_STORE_TYPE`` in ``.env`` file
+
+```shell
+### Milvus vector db config
+VECTOR_STORE_TYPE=Milvus
+MILVUS_URL=127.0.0.1
+MILVUS_PORT=19530
+#MILVUS_USERNAME
+#MILVUS_PASSWORD
+#MILVUS_SECURE=
+ ```
+
+
+
+
+set ``VECTOR_STORE_TYPE`` in ``.env`` file
+
+```shell
+### Weaviate vector db config
+VECTOR_STORE_TYPE=Weaviate
+#WEAVIATE_URL=https://kt-region-m8hcy0wc.weaviate.network
+ ```
+
+
+
diff --git a/docs/docs/faq.md b/docs/docs/faq.md
deleted file mode 100644
index 429a49e0d..000000000
--- a/docs/docs/faq.md
+++ /dev/null
@@ -1,2 +0,0 @@
-# FAQ
-If you encounter any problems, you can submit an [issue](https://github.com/eosphoros-ai/DB-GPT/issues) on Github.
\ No newline at end of file
diff --git a/docs/docs/faq/chatdata.md b/docs/docs/faq/chatdata.md
new file mode 100644
index 000000000..a84c44249
--- /dev/null
+++ b/docs/docs/faq/chatdata.md
@@ -0,0 +1,55 @@
+ChatData & ChatDB
+==================================
+ChatData generates SQL from natural language and executes it. ChatDB involves conversing with metadata from the
+Database, including metadata about databases, tables, and
+fields.
+
+### 1.Choose Datasource
+
+If you are using DB-GPT for the first time, you need to add a data source and set the relevant connection information
+for the data source.
+
+```{tip}
+there are some example data in DB-GPT-NEW/DB-GPT/docker/examples
+
+you can execute sql script to generate data.
+```
+
+#### 1.1 Datasource management
+
+
+
+#### 1.2 Connection management
+
+
+
+#### 1.3 Add Datasource
+
+
+
+```{note}
+now DB-GPT support Datasource Type
+
+* Mysql
+* Sqlite
+* DuckDB
+* Clickhouse
+* Mssql
+```
+
+### 2.ChatData
+##### Preview Mode
+After successfully setting up the data source, you can start conversing with the database. You can ask it to generate
+SQL for you or inquire about relevant information on the database's metadata.
+
+
+##### Editor Mode
+In Editor Mode, you can edit your sql and execute it.
+
+
+
+### 3.ChatDB
+
+
+
+
diff --git a/docs/docs/faq/install.md b/docs/docs/faq/install.md
new file mode 100644
index 000000000..8ca0f6ec1
--- /dev/null
+++ b/docs/docs/faq/install.md
@@ -0,0 +1,73 @@
+Installation FAQ
+==================================
+
+
+##### Q1: sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) unable to open database file
+
+make sure you pull latest code or create directory with mkdir pilot/data
+
+##### Q2: The model keeps getting killed.
+
+your GPU VRAM size is not enough, try replace your hardware or replace other llms.
+
+##### Q3: How to access website on the public network
+
+You can try to use gradio's [network](https://github.com/gradio-app/gradio/blob/main/gradio/networking.py) to achieve.
+```python
+import secrets
+from gradio import networking
+token=secrets.token_urlsafe(32)
+local_port=5000
+url = networking.setup_tunnel('0.0.0.0', local_port, token)
+print(f'Public url: {url}')
+time.sleep(60 * 60 * 24)
+```
+
+Open `url` with your browser to see the website.
+
+##### Q4: (Windows) execute `pip install -e .` error
+
+The error log like the following:
+```
+× python setup.py bdist_wheel did not run successfully.
+ │ exit code: 1
+ ╰─> [11 lines of output]
+ running bdist_wheel
+ running build
+ running build_py
+ creating build
+ creating build\lib.win-amd64-cpython-310
+ creating build\lib.win-amd64-cpython-310\cchardet
+ copying src\cchardet\version.py -> build\lib.win-amd64-cpython-310\cchardet
+ copying src\cchardet\__init__.py -> build\lib.win-amd64-cpython-310\cchardet
+ running build_ext
+ building 'cchardet._cchardet' extension
+ error: Microsoft Visual C++ 14.0 or greater is required. Get it with "Microsoft C++ Build Tools": https://visualstudio.microsoft.com/visual-cpp-build-tools/
+ [end of output]
+```
+
+Download and install `Microsoft C++ Build Tools` from [visual-cpp-build-tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/)
+
+
+
+##### Q5: `Torch not compiled with CUDA enabled`
+
+```
+2023-08-19 16:24:30 | ERROR | stderr | raise AssertionError("Torch not compiled with CUDA enabled")
+2023-08-19 16:24:30 | ERROR | stderr | AssertionError: Torch not compiled with CUDA enabled
+```
+
+1. Install [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit-archive)
+2. Reinstall PyTorch [start-locally](https://pytorch.org/get-started/locally/#start-locally) with CUDA support.
+
+
+##### Q6: `How to migrate meta table chat_history and connect_config from duckdb to sqlite`
+```commandline
+ python docker/examples/metadata/duckdb2sqlite.py
+```
+
+##### Q7: `How to migrate meta table chat_history and connect_config from duckdb to mysql`
+```commandline
+1. update your mysql username and password in docker/examples/metadata/duckdb2mysql.py
+2. python docker/examples/metadata/duckdb2mysql.py
+```
diff --git a/docs/docs/faq/kbqa.md b/docs/docs/faq/kbqa.md
new file mode 100644
index 000000000..56137c87a
--- /dev/null
+++ b/docs/docs/faq/kbqa.md
@@ -0,0 +1,70 @@
+KBQA FAQ
+==================================
+
+##### Q1: text2vec-large-chinese not found
+
+make sure you have download text2vec-large-chinese embedding model in right way
+
+```tip
+centos:yum install git-lfs
+ubuntu:apt-get install git-lfs -y
+macos:brew install git-lfs
+```
+```bash
+cd models
+git lfs clone https://huggingface.co/GanymedeNil/text2vec-large-chinese
+```
+
+##### Q2:How to change Vector DB Type in DB-GPT.
+
+Update .env file and set VECTOR_STORE_TYPE.
+
+DB-GPT currently support Chroma(Default), Milvus(>2.1), Weaviate vector database.
+If you want to change vector db, Update your .env, set your vector store type, VECTOR_STORE_TYPE=Chroma (now only support Chroma and Milvus(>2.1), if you set Milvus, please set MILVUS_URL and MILVUS_PORT)
+If you want to support more vector db, you can integrate yourself.[how to integrate](https://db-gpt.readthedocs.io/en/latest/modules/vector.html)
+```commandline
+#*******************************************************************#
+#** VECTOR STORE SETTINGS **#
+#*******************************************************************#
+VECTOR_STORE_TYPE=Chroma
+#MILVUS_URL=127.0.0.1
+#MILVUS_PORT=19530
+#MILVUS_USERNAME
+#MILVUS_PASSWORD
+#MILVUS_SECURE=
+
+#WEAVIATE_URL=https://kt-region-m8hcy0wc.weaviate.network
+```
+##### Q3:When I use vicuna-13b, found some illegal character like this.
+
+
+
+
+Set KNOWLEDGE_SEARCH_TOP_SIZE smaller or set KNOWLEDGE_CHUNK_SIZE smaller, and reboot server.
+
+##### Q4:space add error (pymysql.err.OperationalError) (1054, "Unknown column 'knowledge_space.context' in 'field list'")
+
+1.shutdown dbgpt_server(ctrl c)
+
+2.add column context for table knowledge_space
+
+```commandline
+mysql -h127.0.0.1 -uroot -p {your_password}
+```
+
+3.execute sql ddl
+
+```commandline
+mysql> use knowledge_management;
+mysql> ALTER TABLE knowledge_space ADD COLUMN context TEXT COMMENT "arguments context";
+```
+
+4.restart dbgpt serve
+
+##### Q5:Use Mysql, how to use DB-GPT KBQA
+
+build Mysql KBQA system database schema.
+
+```bash
+$ mysql -h127.0.0.1 -uroot -p{your_password} < ./assets/schema/knowledge_management.sql
+```
\ No newline at end of file
diff --git a/docs/docs/faq/llm.md b/docs/docs/faq/llm.md
new file mode 100644
index 000000000..a0fcaa83e
--- /dev/null
+++ b/docs/docs/faq/llm.md
@@ -0,0 +1,52 @@
+LLM USE FAQ
+==================================
+##### Q1:how to use openai chatgpt service
+change your LLM_MODEL
+````shell
+LLM_MODEL=proxyllm
+````
+
+set your OPENAPI KEY
+
+````shell
+PROXY_API_KEY={your-openai-sk}
+PROXY_SERVER_URL=https://api.openai.com/v1/chat/completions
+````
+
+make sure your openapi API_KEY is available
+
+##### Q2 What difference between `python dbgpt_server --light` and `python dbgpt_server`
+
+:::tip
+python dbgpt_server --light` dbgpt_server does not start the llm service. Users can deploy the llm service separately by using `python llmserver`, and dbgpt_server accesses the llm service through set the LLM_SERVER environment variable in .env. The purpose is to allow for the separate deployment of dbgpt's backend service and llm service.
+
+python dbgpt_server service and the llm service are deployed on the same instance. when dbgpt_server starts the service, it also starts the llm service at the same time.
+:::
+
+##### Q3 how to use MultiGPUs
+
+DB-GPT will use all available gpu by default. And you can modify the setting `CUDA_VISIBLE_DEVICES=0,1` in `.env` file
+to use the specific gpu IDs.
+
+Optionally, you can also specify the gpu ID to use before the starting command, as shown below:
+
+````shell
+# Specify 1 gpu
+CUDA_VISIBLE_DEVICES=0 python3 pilot/server/dbgpt_server.py
+
+# Specify 4 gpus
+CUDA_VISIBLE_DEVICES=3,4,5,6 python3 pilot/server/dbgpt_server.py
+````
+
+You can modify the setting `MAX_GPU_MEMORY=xxGib` in `.env` file to configure the maximum memory used by each GPU.
+
+##### Q4 Not Enough Memory
+
+DB-GPT supported 8-bit quantization and 4-bit quantization.
+
+You can modify the setting `QUANTIZE_8bit=True` or `QUANTIZE_4bit=True` in `.env` file to use quantization(8-bit quantization is enabled by default).
+
+Llama-2-70b with 8-bit quantization can run with 80 GB of VRAM, and 4-bit quantization can run with 48 GB of VRAM.
+
+Note: you need to install the latest dependencies according to [requirements.txt](https://github.com/eosphoros-ai/DB-GPT/blob/main/requirements.txt).
+Note: you need to install the latest dependencies according to [requirements.txt](https://github.com/eosphoros-ai/DB-GPT/blob/main/requirements.txt).
\ No newline at end of file
diff --git a/docs/sidebars.js b/docs/sidebars.js
index ded373e87..996f2d983 100755
--- a/docs/sidebars.js
+++ b/docs/sidebars.js
@@ -200,8 +200,27 @@ const sidebars = {
},
{
- type: "doc",
- id:"faq"
+ type: "category",
+ label: "FAQ",
+ collapsed: true,
+ items: [
+ {
+ type: 'doc',
+ id: 'faq/install',
+ }
+ ,{
+ type: 'doc',
+ id: 'faq/llm',
+ }
+ ,{
+ type: 'doc',
+ id: 'faq/kbqa',
+ }
+ ,{
+ type: 'doc',
+ id: 'faq/chatdata',
+ },
+ ],
},
{
diff --git a/pilot/base_modules/meta_data/meta_data.py b/pilot/base_modules/meta_data/meta_data.py
index 5b79a38a5..4ec80492a 100644
--- a/pilot/base_modules/meta_data/meta_data.py
+++ b/pilot/base_modules/meta_data/meta_data.py
@@ -14,7 +14,7 @@
logger = logging.getLogger(__name__)
-
+# DB-GPT meta_data database config, now support mysql and sqlite
CFG = Config()
default_db_path = os.path.join(os.getcwd(), "meta_data")
@@ -26,6 +26,7 @@
db_path = default_db_path + f"/{db_name}.db"
connection = sqlite3.connect(db_path)
+
if CFG.LOCAL_DB_TYPE == "mysql":
engine_temp = create_engine(
f"mysql+pymysql://"
@@ -81,18 +82,10 @@
alembic_cfg.set_main_option("script_location", default_db_path + "/alembic")
-# 将模型和会话传递给Alembic配置
alembic_cfg.attributes["target_metadata"] = Base.metadata
alembic_cfg.attributes["session"] = session
-# # 创建表
-# Base.metadata.create_all(engine)
-#
-# # 删除表
-# Base.metadata.drop_all(engine)
-
-
def ddl_init_and_upgrade(disable_alembic_upgrade: bool):
"""Initialize and upgrade database metadata
@@ -105,10 +98,6 @@ def ddl_init_and_upgrade(disable_alembic_upgrade: bool):
)
return
- # Base.metadata.create_all(bind=engine)
- # 生成并应用迁移脚本
- # command.upgrade(alembic_cfg, 'head')
- # subprocess.run(["alembic", "revision", "--autogenerate", "-m", "Added account table"])
with engine.connect() as connection:
alembic_cfg.attributes["connection"] = connection
heads = command.heads(alembic_cfg)
diff --git a/pilot/configs/config.py b/pilot/configs/config.py
index 25e1bac53..8fcce75cb 100644
--- a/pilot/configs/config.py
+++ b/pilot/configs/config.py
@@ -104,13 +104,6 @@ def __init__(self) -> None:
self.use_mac_os_tts = False
self.use_mac_os_tts = os.getenv("USE_MAC_OS_TTS")
- # milvus or zilliz cloud configuration
- self.milvus_addr = os.getenv("MILVUS_ADDR", "localhost:19530")
- self.milvus_username = os.getenv("MILVUS_USERNAME")
- self.milvus_password = os.getenv("MILVUS_PASSWORD")
- self.milvus_collection = os.getenv("MILVUS_COLLECTION", "dbgpt")
- self.milvus_secure = os.getenv("MILVUS_SECURE", "False").lower() == "true"
-
self.authorise_key = os.getenv("AUTHORISE_COMMAND_KEY", "y")
self.exit_key = os.getenv("EXIT_KEY", "n")
self.image_provider = os.getenv("IMAGE_PROVIDER", True)
@@ -190,7 +183,7 @@ def __init__(self) -> None:
self.LOCAL_DB_PASSWORD = os.getenv("LOCAL_DB_PASSWORD", "aa123456")
self.LOCAL_DB_POOL_SIZE = int(os.getenv("LOCAL_DB_POOL_SIZE", 10))
- self.CHAT_HISTORY_STORE_TYPE = os.getenv("CHAT_HISTORY_STORE_TYPE", "duckdb")
+ self.CHAT_HISTORY_STORE_TYPE = os.getenv("CHAT_HISTORY_STORE_TYPE", "db")
### LLM Model Service Configuration
self.LLM_MODEL = os.getenv("LLM_MODEL", "vicuna-13b-v1.5")
@@ -232,10 +225,18 @@ def __init__(self) -> None:
self.KNOWLEDGE_CHUNK_SIZE = int(os.getenv("KNOWLEDGE_CHUNK_SIZE", 100))
self.KNOWLEDGE_CHUNK_OVERLAP = int(os.getenv("KNOWLEDGE_CHUNK_OVERLAP", 50))
self.KNOWLEDGE_SEARCH_TOP_SIZE = int(os.getenv("KNOWLEDGE_SEARCH_TOP_SIZE", 5))
+ # default recall similarity score, between 0 and 1
+ self.KNOWLEDGE_SEARCH_RECALL_SCORE = float(
+ os.getenv("KNOWLEDGE_SEARCH_RECALL_SCORE", 0.3)
+ )
self.KNOWLEDGE_SEARCH_MAX_TOKEN = int(
os.getenv("KNOWLEDGE_SEARCH_MAX_TOKEN", 2000)
)
- ### Control whether to display the source document of knowledge on the front end.
+ # Whether to enable Chat Knowledge Search Rewrite Mode
+ self.KNOWLEDGE_SEARCH_REWRITE = (
+ os.getenv("KNOWLEDGE_SEARCH_REWRITE", "False").lower() == "true"
+ )
+ # Control whether to display the source document of knowledge on the front end.
self.KNOWLEDGE_CHAT_SHOW_RELATIONS = (
os.getenv("KNOWLEDGE_CHAT_SHOW_RELATIONS", "False").lower() == "true"
)
diff --git a/pilot/connections/manages/connect_config_db.py b/pilot/connections/manages/connect_config_db.py
index 602ff719b..b6ab6940e 100644
--- a/pilot/connections/manages/connect_config_db.py
+++ b/pilot/connections/manages/connect_config_db.py
@@ -1,5 +1,4 @@
-from typing import List
-from sqlalchemy import Column, Integer, String, Index, DateTime, func, Boolean, Text
+from sqlalchemy import Column, Integer, String, Index, Text, text
from sqlalchemy import UniqueConstraint
from pilot.base_modules.meta_data.base_dao import BaseDao
@@ -12,15 +11,18 @@
class ConnectConfigEntity(Base):
+ """db connect config entity"""
+
__tablename__ = "connect_config"
id = Column(
Integer, primary_key=True, autoincrement=True, comment="autoincrement id"
)
+
db_type = Column(String(255), nullable=False, comment="db type")
db_name = Column(String(255), nullable=False, comment="db name")
db_path = Column(String(255), nullable=True, comment="file db path")
db_host = Column(String(255), nullable=True, comment="db connect host(not file db)")
- db_port = Column(String(255), nullable=True, comment="db cnnect port(not file db)")
+ db_port = Column(String(255), nullable=True, comment="db connect port(not file db)")
db_user = Column(String(255), nullable=True, comment="db user")
db_pwd = Column(String(255), nullable=True, comment="db password")
comment = Column(Text, nullable=True, comment="db comment")
@@ -29,10 +31,13 @@ class ConnectConfigEntity(Base):
__table_args__ = (
UniqueConstraint("db_name", name="uk_db"),
Index("idx_q_db_type", "db_type"),
+ {"mysql_charset": "utf8mb4", "mysql_collate": "utf8mb4_unicode_ci"},
)
class ConnectConfigDao(BaseDao[ConnectConfigEntity]):
+ """db connect config dao"""
+
def __init__(self):
super().__init__(
database=META_DATA_DATABASE,
@@ -42,6 +47,7 @@ def __init__(self):
)
def update(self, entity: ConnectConfigEntity):
+ """update db connect info"""
session = self.get_session()
try:
updated = session.merge(entity)
@@ -51,6 +57,7 @@ def update(self, entity: ConnectConfigEntity):
session.close()
def delete(self, db_name: int):
+ """ "delete db connect info"""
session = self.get_session()
if db_name is None:
raise Exception("db_name is None")
@@ -61,10 +68,177 @@ def delete(self, db_name: int):
session.commit()
session.close()
- def get_by_name(self, db_name: str) -> ConnectConfigEntity:
+ def get_by_names(self, db_name: str) -> ConnectConfigEntity:
+ """get db connect info by name"""
session = self.get_session()
db_connect = session.query(ConnectConfigEntity)
db_connect = db_connect.filter(ConnectConfigEntity.db_name == db_name)
result = db_connect.first()
session.close()
return result
+
+ def add_url_db(
+ self,
+ db_name,
+ db_type,
+ db_host: str,
+ db_port: int,
+ db_user: str,
+ db_pwd: str,
+ comment: str = "",
+ ):
+ """
+ add db connect info
+ Args:
+ db_name: db name
+ db_type: db type
+ db_host: db host
+ db_port: db port
+ db_user: db user
+ db_pwd: db password
+ comment: comment
+ """
+ try:
+ session = self.get_session()
+
+ from sqlalchemy import text
+
+ insert_statement = text(
+ """
+ INSERT INTO connect_config (
+ db_name, db_type, db_path, db_host, db_port, db_user, db_pwd, comment
+ ) VALUES (
+ :db_name, :db_type, :db_path, :db_host, :db_port, :db_user, :db_pwd, :comment
+ )
+ """
+ )
+
+ params = {
+ "db_name": db_name,
+ "db_type": db_type,
+ "db_path": "",
+ "db_host": db_host,
+ "db_port": db_port,
+ "db_user": db_user,
+ "db_pwd": db_pwd,
+ "comment": comment,
+ }
+ session.execute(insert_statement, params)
+ session.commit()
+ session.close()
+ except Exception as e:
+ print("add db connect info error！" + str(e))
+
+ def update_db_info(
+ self,
+ db_name,
+ db_type,
+ db_path: str = "",
+ db_host: str = "",
+ db_port: int = 0,
+ db_user: str = "",
+ db_pwd: str = "",
+ comment: str = "",
+ ):
+ """update db connect info"""
+ old_db_conf = self.get_db_config(db_name)
+ if old_db_conf:
+ try:
+ session = self.get_session()
+ if not db_path:
+ update_statement = text(
+ f"UPDATE connect_config set db_type='{db_type}', db_host='{db_host}', db_port={db_port}, db_user='{db_user}', db_pwd='{db_pwd}', comment='{comment}' where db_name='{db_name}'"
+ )
+ else:
+ update_statement = text(
+ f"UPDATE connect_config set db_type='{db_type}', db_path='{db_path}', comment='{comment}' where db_name='{db_name}'"
+ )
+ session.execute(update_statement)
+ session.commit()
+ session.close()
+ except Exception as e:
+ print("edit db connect info error！" + str(e))
+ return True
+ raise ValueError(f"{db_name} not have config info!")
+
+ def add_file_db(self, db_name, db_type, db_path: str, comment: str = ""):
+ """add file db connect info"""
+ try:
+ session = self.get_session()
+ insert_statement = text(
+ """
+ INSERT INTO connect_config(
+ db_name, db_type, db_path, db_host, db_port, db_user, db_pwd, comment
+ ) VALUES (
+ :db_name, :db_type, :db_path, :db_host, :db_port, :db_user, :db_pwd, :comment
+ )
+ """
+ )
+ params = {
+ "db_name": db_name,
+ "db_type": db_type,
+ "db_path": db_path,
+ "db_host": "",
+ "db_port": 0,
+ "db_user": "",
+ "db_pwd": "",
+ "comment": comment,
+ }
+
+ session.execute(insert_statement, params)
+
+ session.commit()
+ session.close()
+ except Exception as e:
+ print("add db connect info error！" + str(e))
+
+ def get_db_config(self, db_name):
+ """get db config by name"""
+ session = self.get_session()
+ if db_name:
+ select_statement = text(
+ """
+ SELECT
+ *
+ FROM
+ connect_config
+ WHERE
+ db_name = :db_name
+ """
+ )
+ params = {"db_name": db_name}
+ result = session.execute(select_statement, params)
+
+ else:
+ raise ValueError("Cannot get database by name" + db_name)
+
+ fields = [field[0] for field in result.cursor.description]
+ row_dict = {}
+ row_1 = list(result.cursor.fetchall()[0])
+ for i, field in enumerate(fields):
+ row_dict[field] = row_1[i]
+ return row_dict
+
+ def get_db_list(self):
+ """get db list"""
+ session = self.get_session()
+ result = session.execute(text("SELECT * FROM connect_config"))
+
+ fields = [field[0] for field in result.cursor.description]
+ data = []
+ for row in result.cursor.fetchall():
+ row_dict = {}
+ for i, field in enumerate(fields):
+ row_dict[field] = row[i]
+ data.append(row_dict)
+ return data
+
+ def delete_db(self, db_name):
+ """delete db connect info"""
+ session = self.get_session()
+ delete_statement = text("""DELETE FROM connect_config where db_name=:db_name""")
+ params = {"db_name": db_name}
+ session.execute(delete_statement, params)
+ session.commit()
+ session.close()
+ return True
diff --git a/pilot/connections/manages/connection_manager.py b/pilot/connections/manages/connection_manager.py
index 473449f94..261f9b5cf 100644
--- a/pilot/connections/manages/connection_manager.py
+++ b/pilot/connections/manages/connection_manager.py
@@ -2,6 +2,7 @@
import asyncio
from pilot.configs.config import Config
+from pilot.connections import ConnectConfigDao
from pilot.connections.manages.connect_storage_duckdb import DuckdbConnectConfig
from pilot.common.schema import DBType
from pilot.component import SystemApp, ComponentType
@@ -28,6 +29,8 @@
class ConnectManager:
+ """db connect manager"""
+
def get_all_subclasses(self, cls):
subclasses = cls.__subclasses__()
for subclass in subclasses:
@@ -49,90 +52,81 @@ def get_cls_by_dbtype(self, db_type):
if cls.db_type == db_type:
result = cls
if not result:
- raise ValueError("Unsupport Db Type！" + db_type)
+ raise ValueError("Unsupported Db Type！" + db_type)
return result
def __init__(self, system_app: SystemApp):
- self.storage = DuckdbConnectConfig()
+ """metadata database management initialization"""
+ # self.storage = DuckdbConnectConfig()
+ self.storage = ConnectConfigDao()
self.db_summary_client = DBSummaryClient(system_app)
# self.__load_config_db()
- def __load_config_db(self):
- if CFG.LOCAL_DB_HOST:
- # default mysql
- if CFG.LOCAL_DB_NAME:
- self.storage.add_url_db(
- CFG.LOCAL_DB_NAME,
- DBType.Mysql.value(),
- CFG.LOCAL_DB_HOST,
- CFG.LOCAL_DB_PORT,
- CFG.LOCAL_DB_USER,
- CFG.LOCAL_DB_PASSWORD,
- "",
- )
- else:
- # get all default mysql database
- default_mysql = Database.from_uri(
- "mysql+pymysql://"
- + CFG.LOCAL_DB_USER
- + ":"
- + CFG.LOCAL_DB_PASSWORD
- + "@"
- + CFG.LOCAL_DB_HOST
- + ":"
- + str(CFG.LOCAL_DB_PORT),
- engine_args={
- "pool_size": CFG.LOCAL_DB_POOL_SIZE,
- "pool_recycle": 3600,
- "echo": True,
- },
- )
- # default_mysql = MySQLConnect.from_uri(
- # "mysql+pymysql://"
- # + CFG.LOCAL_DB_USER
- # + ":"
- # + CFG.LOCAL_DB_PASSWORD
- # + "@"
- # + CFG.LOCAL_DB_HOST
- # + ":"
- # + str(CFG.LOCAL_DB_PORT),
- # engine_args={"pool_size": 10, "pool_recycle": 3600, "echo": True},
- # )
- dbs = default_mysql.get_database_list()
- for name in dbs:
- self.storage.add_url_db(
- name,
- DBType.Mysql.value(),
- CFG.LOCAL_DB_HOST,
- CFG.LOCAL_DB_PORT,
- CFG.LOCAL_DB_USER,
- CFG.LOCAL_DB_PASSWORD,
- "",
- )
- db_type = DBType.of_db_type(CFG.LOCAL_DB_TYPE)
- if db_type.is_file_db():
- db_name = CFG.LOCAL_DB_NAME
- db_type = CFG.LOCAL_DB_TYPE
- db_path = CFG.LOCAL_DB_PATH
- if not db_type:
- # Default file database type
- db_type = DBType.DuckDb.value()
- if not db_name:
- db_type, db_name = self._parse_file_db_info(db_type, db_path)
- if db_name:
- print(
- f"Add file db, db_name: {db_name}, db_type: {db_type}, db_path: {db_path}"
- )
- self.storage.add_file_db(db_name, db_type, db_path)
-
- def _parse_file_db_info(self, db_type: str, db_path: str):
- if db_type is None or db_type == DBType.DuckDb.value():
- # file db is duckdb
- db_name = self.storage.get_file_db_name(db_path)
- db_type = DBType.DuckDb.value()
- else:
- db_name = DBType.parse_file_db_name_from_path(db_type, db_path)
- return db_type, db_name
+ # def __load_config_db(self):
+ # if CFG.LOCAL_DB_HOST:
+ # # default mysql
+ # if CFG.LOCAL_DB_NAME:
+ # self.storage.add_url_db(
+ # CFG.LOCAL_DB_NAME,
+ # DBType.Mysql.value(),
+ # CFG.LOCAL_DB_HOST,
+ # CFG.LOCAL_DB_PORT,
+ # CFG.LOCAL_DB_USER,
+ # CFG.LOCAL_DB_PASSWORD,
+ # "",
+ # )
+ # else:
+ # # get all default mysql database
+ # default_mysql = Database.from_uri(
+ # "mysql+pymysql://"
+ # + CFG.LOCAL_DB_USER
+ # + ":"
+ # + CFG.LOCAL_DB_PASSWORD
+ # + "@"
+ # + CFG.LOCAL_DB_HOST
+ # + ":"
+ # + str(CFG.LOCAL_DB_PORT),
+ # engine_args={
+ # "pool_size": CFG.LOCAL_DB_POOL_SIZE,
+ # "pool_recycle": 3600,
+ # "echo": True,
+ # },
+ # )
+ # dbs = default_mysql.get_database_list()
+ # for name in dbs:
+ # self.storage.add_url_db(
+ # name,
+ # DBType.Mysql.value(),
+ # CFG.LOCAL_DB_HOST,
+ # CFG.LOCAL_DB_PORT,
+ # CFG.LOCAL_DB_USER,
+ # CFG.LOCAL_DB_PASSWORD,
+ # "",
+ # )
+ # db_type = DBType.of_db_type(CFG.LOCAL_DB_TYPE)
+ # if db_type.is_file_db():
+ # db_name = CFG.LOCAL_DB_NAME
+ # db_type = CFG.LOCAL_DB_TYPE
+ # db_path = CFG.LOCAL_DB_PATH
+ # if not db_type:
+ # # Default file database type
+ # db_type = DBType.DuckDb.value()
+ # if not db_name:
+ # db_type, db_name = self._parse_file_db_info(db_type, db_path)
+ # if db_name:
+ # print(
+ # f"Add file db, db_name: {db_name}, db_type: {db_type}, db_path: {db_path}"
+ # )
+ # self.storage.add_file_db(db_name, db_type, db_path)
+
+ # def _parse_file_db_info(self, db_type: str, db_path: str):
+ # if db_type is None or db_type == DBType.DuckDb.value():
+ # # file db is duckdb
+ # db_name = self.storage.get_file_db_name(db_path)
+ # db_type = DBType.DuckDb.value()
+ # else:
+ # db_name = DBType.parse_file_db_name_from_path(db_type, db_path)
+ # return db_type, db_name
def get_connect(self, db_name):
db_config = self.storage.get_db_config(db_name)
@@ -178,7 +172,7 @@ def get_db_list(self):
return self.storage.get_db_list()
def get_db_names(self):
- return self.storage.get_db_names()
+ return self.storage.get_by_name()
def delete_db(self, db_name: str):
return self.storage.delete_db(db_name)
diff --git a/pilot/embedding_engine/embedding_engine.py b/pilot/embedding_engine/embedding_engine.py
index ce739c3bb..687137f9d 100644
--- a/pilot/embedding_engine/embedding_engine.py
+++ b/pilot/embedding_engine/embedding_engine.py
@@ -16,6 +16,27 @@ class EmbeddingEngine:
2.similar_search: similarity search from vector_store
how to use reference:https://db-gpt.readthedocs.io/en/latest/modules/knowledge.html
how to integrate:https://db-gpt.readthedocs.io/en/latest/modules/knowledge/pdf/pdf_embedding.html
+ Example:
+ .. code-block:: python
+ embedding_model = "your_embedding_model"
+ vector_store_type = "Chroma"
+ chroma_persist_path = "your_persist_path"
+ vector_store_config = {
+ "vector_store_name": "document_test",
+ "vector_store_type": vector_store_type,
+ "chroma_persist_path": chroma_persist_path,
+ }
+
+ # it can be .md,.pdf,.docx, .csv, .html
+ document_path = "your_path/test.md"
+ embedding_engine = EmbeddingEngine(
+ knowledge_source=document_path,
+ knowledge_type=KnowledgeType.DOCUMENT.value,
+ model_name=embedding_model,
+ vector_store_config=vector_store_config,
+ )
+ # embedding document content to vector store
+ embedding_engine.knowledge_embedding()
"""
def __init__(
@@ -74,7 +95,8 @@ def init_knowledge_embedding(self):
)
def similar_search(self, text, topk):
- """vector db similar search
+ """vector db similar search in vector database.
+ Return topk docs.
Args:
- text: query text
- topk: top k
@@ -84,8 +106,22 @@ def similar_search(self, text, topk):
)
# https://github.com/chroma-core/chroma/issues/657
ans = vector_client.similar_search(text, topk)
- # except NotEnoughElementsException:
- # ans = vector_client.similar_search(text, 1)
+ return ans
+
+ def similar_search_with_scores(self, text, topk, score_threshold: float = 0.3):
+ """
+ similar_search_with_score in vector database.
+ Return docs and relevance scores in the range [0, 1].
+ Args:
+ doc(str): query text
+ topk(int): return docs nums. Defaults to 4.
+ score_threshold(float): score_threshold: Optional, a floating point value between 0 to 1 to
+ filter the resulting set of retrieved docs,0 is dissimilar, 1 is most similar.
+ """
+ vector_client = VectorStoreConnector(
+ self.vector_store_config["vector_store_type"], self.vector_store_config
+ )
+ ans = vector_client.similar_search_with_scores(text, topk, score_threshold)
return ans
def vector_exist(self):
diff --git a/pilot/graph_engine/__init__.py b/pilot/rag/__init__.py
similarity index 100%
rename from pilot/graph_engine/__init__.py
rename to pilot/rag/__init__.py
diff --git a/pilot/rag/extracter/__init__.py b/pilot/rag/extracter/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pilot/rag/extracter/base.py b/pilot/rag/extracter/base.py
new file mode 100644
index 000000000..6ace7872a
--- /dev/null
+++ b/pilot/rag/extracter/base.py
@@ -0,0 +1,19 @@
+from abc import abstractmethod, ABC
+from typing import List, Dict
+
+from langchain.schema import Document
+
+
+class Extractor(ABC):
+ """Extractor Base class, it's apply for Summary Extractor, Keyword Extractor, Triplets Extractor, Question Extractor, etc."""
+
+ def __init__(self):
+ pass
+
+ @abstractmethod
+ def extract(self, chunks: List[Document]) -> List[Dict]:
+ """Extracts chunks.
+
+ Args:
+ nodes (Sequence[Document]): nodes to extract metadata from
+ """
diff --git a/pilot/rag/extracter/summary.py b/pilot/rag/extracter/summary.py
new file mode 100644
index 000000000..5eb09d65a
--- /dev/null
+++ b/pilot/rag/extracter/summary.py
@@ -0,0 +1,95 @@
+from typing import List, Dict
+
+from langchain.schema import Document
+
+from pilot.common.llm_metadata import LLMMetadata
+from pilot.rag.extracter.base import Extractor
+
+
+class SummaryExtractor(Extractor):
+ """Summary Extractor, it can extract document summary."""
+
+ def __init__(self, model_name: str = None, llm_metadata: LLMMetadata = None):
+ self.model_name = (model_name,)
+ self.llm_metadata = (llm_metadata or LLMMetadata,)
+
+ async def extract(self, chunks: List[Document]) -> str:
+ """async document extract summary
+ Args:
+ - model_name: str
+ - chunk_docs: List[Document]
+ """
+ texts = [doc.page_content for doc in chunks]
+ from pilot.common.prompt_util import PromptHelper
+
+ prompt_helper = PromptHelper()
+ from pilot.scene.chat_knowledge.summary.prompt import prompt
+
+ texts = prompt_helper.repack(prompt_template=prompt.template, text_chunks=texts)
+ return await self._mapreduce_extract_summary(
+ docs=texts, model_name=self.model_name, llm_metadata=self.llm_metadata
+ )
+
+ async def _mapreduce_extract_summary(
+ self,
+ docs,
+ model_name,
+ llm_metadata: LLMMetadata,
+ ):
+ """Extract summary by mapreduce mode
+ map -> multi async call llm to generate summary
+ reduce -> merge the summaries by map process
+ Args:
+ docs:List[str]
+ model_name:model name str
+ llm_metadata:LLMMetadata
+ Returns:
+ Document: refine summary context document.
+ """
+ from pilot.scene.base import ChatScene
+ from pilot.common.chat_util import llm_chat_response_nostream
+ import uuid
+
+ tasks = []
+ if len(docs) == 1:
+ return docs[0]
+ else:
+ max_iteration = (
+ llm_metadata.max_chat_iteration
+ if len(docs) > llm_metadata.max_chat_iteration
+ else len(docs)
+ )
+ for doc in docs[0:max_iteration]:
+ chat_param = {
+ "chat_session_id": uuid.uuid1(),
+ "current_user_input": "",
+ "select_param": doc,
+ "model_name": model_name,
+ "model_cache_enable": True,
+ }
+ tasks.append(
+ llm_chat_response_nostream(
+ ChatScene.ExtractSummary.value(), **{"chat_param": chat_param}
+ )
+ )
+ from pilot.common.chat_util import run_async_tasks
+
+ summary_iters = await run_async_tasks(
+ tasks=tasks, concurrency_limit=llm_metadata.concurrency_limit
+ )
+ summary_iters = list(
+ filter(
+ lambda content: "LLMServer Generate Error" not in content,
+ summary_iters,
+ )
+ )
+ from pilot.common.prompt_util import PromptHelper
+ from pilot.scene.chat_knowledge.summary.prompt import prompt
+
+ prompt_helper = PromptHelper()
+ summary_iters = prompt_helper.repack(
+ prompt_template=prompt.template, text_chunks=summary_iters
+ )
+ return await self._mapreduce_extract_summary(
+ summary_iters, model_name, max_iteration, llm_metadata.concurrency_limit
+ )
diff --git a/pilot/rag/graph_engine/__init__.py b/pilot/rag/graph_engine/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pilot/graph_engine/graph_engine.py b/pilot/rag/graph_engine/graph_engine.py
similarity index 66%
rename from pilot/graph_engine/graph_engine.py
rename to pilot/rag/graph_engine/graph_engine.py
index bea5f3123..c7f880da0 100644
--- a/pilot/graph_engine/graph_engine.py
+++ b/pilot/rag/graph_engine/graph_engine.py
@@ -6,8 +6,8 @@
from pilot.embedding_engine import KnowledgeType
from pilot.embedding_engine.knowledge_type import get_knowledge_embedding
-from pilot.graph_engine.index_struct import KG
-from pilot.graph_engine.node import TextNode
+from pilot.rag.graph_engine.index_struct import KG
+from pilot.rag.graph_engine.node import TextNode
from pilot.utils import utils
logger = logging.getLogger(__name__)
@@ -121,64 +121,9 @@ def _build_index_from_docs(self, documents: List[Document]) -> KG:
self.graph_store.upsert_triplet(*triplet)
index_struct.add_node([subj, obj], text_node)
return index_struct
- # num_threads = 5
- # chunk_size = (
- # len(documents)
- # if (len(documents) < num_threads)
- # else len(documents) // num_threads
- # )
- #
- # import concurrent
- # triples = []
- # future_tasks = []
- # with concurrent.futures.ThreadPoolExecutor() as executor:
- # for i in range(num_threads):
- # start = i * chunk_size
- # end = start + chunk_size if i < num_threads - 1 else None
- # # doc = documents[start:end]
- # future_tasks.append(
- # executor.submit(
- # self._extract_triplets_task,
- # documents[start:end],
- # index_struct,
- # )
- # )
- # # for doc in documents[start:end]:
- # # future_tasks.append(
- # # executor.submit(
- # # self._extract_triplets_task,
- # # doc,
- # # index_struct,
- # # )
- # # )
- #
- # # result = [future.result() for future in future_tasks]
- # completed_futures, _ = concurrent.futures.wait(future_tasks, return_when=concurrent.futures.ALL_COMPLETED)
- # for future in completed_futures:
- # # 获取已完成的future的结果并添加到results列表中
- # result = future.result()
- # triplets.extend(result)
- # print(f"total triplets-{triples}")
- # for triplet in triplets:
- # subj, _, obj = triplet
- # self.graph_store.upsert_triplet(*triplet)
- # # index_struct.add_node([subj, obj], text_node)
- # return index_struct
- # for doc in documents:
- # triplets = self._extract_triplets(doc.page_content)
- # if len(triplets) == 0:
- # continue
- # text_node = TextNode(text=doc.page_content, metadata=doc.metadata)
- # logger.info(f"extracted knowledge triplets: {triplets}")
- # for triplet in triplets:
- # subj, _, obj = triplet
- # self.graph_store.upsert_triplet(*triplet)
- # index_struct.add_node([subj, obj], text_node)
- #
- # return index_struct
def search(self, query):
- from pilot.graph_engine.graph_search import RAGGraphSearch
+ from pilot.rag.graph_engine.graph_search import RAGGraphSearch
graph_search = RAGGraphSearch(graph_engine=self)
return graph_search.search(query)
@@ -200,8 +145,3 @@ def _extract_triplets_task(self, docs, index_struct):
)
triple_results.extend(triplets)
return triple_results
- # for triplet in triplets:
- # subj, _, obj = triplet
- # self.graph_store.upsert_triplet(*triplet)
- # self.graph_store.upsert_triplet(*triplet)
- # index_struct.add_node([subj, obj], text_node)
diff --git a/pilot/graph_engine/graph_factory.py b/pilot/rag/graph_engine/graph_factory.py
similarity index 93%
rename from pilot/graph_engine/graph_factory.py
rename to pilot/rag/graph_engine/graph_factory.py
index 3a8b99c17..e0897e387 100644
--- a/pilot/graph_engine/graph_factory.py
+++ b/pilot/rag/graph_engine/graph_factory.py
@@ -20,7 +20,7 @@ def __init__(
super().__init__(system_app=system_app)
self._default_model_name = default_model_name
self.kwargs = kwargs
- from pilot.graph_engine.graph_engine import RAGGraphEngine
+ from pilot.rag.graph_engine.graph_engine import RAGGraphEngine
self.rag_engine = RAGGraphEngine(model_name="proxyllm")
diff --git a/pilot/graph_engine/graph_search.py b/pilot/rag/graph_engine/graph_search.py
similarity index 97%
rename from pilot/graph_engine/graph_search.py
rename to pilot/rag/graph_engine/graph_search.py
index 9419a4979..4a3f44f68 100644
--- a/pilot/graph_engine/graph_search.py
+++ b/pilot/rag/graph_engine/graph_search.py
@@ -6,8 +6,8 @@
from langchain.schema import Document
-from pilot.graph_engine.node import BaseNode, TextNode, NodeWithScore
-from pilot.graph_engine.search import BaseSearch, SearchMode
+from pilot.rag.graph_engine.node import BaseNode, TextNode, NodeWithScore
+from pilot.rag.graph_engine.search import BaseSearch, SearchMode
logger = logging.getLogger(__name__)
DEFAULT_NODE_SCORE = 1000.0
@@ -45,7 +45,7 @@ def __init__(
**kwargs: Any,
) -> None:
"""Initialize params."""
- from pilot.graph_engine.graph_engine import RAGGraphEngine
+ from pilot.rag.graph_engine.graph_engine import RAGGraphEngine
self.graph_engine: RAGGraphEngine = graph_engine
self.model_name = model_name or self.graph_engine.model_name
diff --git a/pilot/graph_engine/index_struct.py b/pilot/rag/graph_engine/index_struct.py
similarity index 98%
rename from pilot/graph_engine/index_struct.py
rename to pilot/rag/graph_engine/index_struct.py
index edc47a7ac..0bd8d8667 100644
--- a/pilot/graph_engine/index_struct.py
+++ b/pilot/rag/graph_engine/index_struct.py
@@ -12,8 +12,8 @@
from dataclasses_json import DataClassJsonMixin
-from pilot.graph_engine.index_type import IndexStructType
-from pilot.graph_engine.node import TextNode, BaseNode
+from pilot.rag.graph_engine.index_type import IndexStructType
+from pilot.rag.graph_engine.node import TextNode, BaseNode
# TODO: legacy backport of old Node class
Node = TextNode
diff --git a/pilot/graph_engine/index_type.py b/pilot/rag/graph_engine/index_type.py
similarity index 100%
rename from pilot/graph_engine/index_type.py
rename to pilot/rag/graph_engine/index_type.py
diff --git a/pilot/graph_engine/kv_index.py b/pilot/rag/graph_engine/kv_index.py
similarity index 100%
rename from pilot/graph_engine/kv_index.py
rename to pilot/rag/graph_engine/kv_index.py
diff --git a/pilot/graph_engine/node.py b/pilot/rag/graph_engine/node.py
similarity index 100%
rename from pilot/graph_engine/node.py
rename to pilot/rag/graph_engine/node.py
diff --git a/pilot/graph_engine/search.py b/pilot/rag/graph_engine/search.py
similarity index 100%
rename from pilot/graph_engine/search.py
rename to pilot/rag/graph_engine/search.py
diff --git a/pilot/rag/retriever/__init__.py b/pilot/rag/retriever/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pilot/rag/retriever/reinforce.py b/pilot/rag/retriever/reinforce.py
new file mode 100644
index 000000000..f73c7fd86
--- /dev/null
+++ b/pilot/rag/retriever/reinforce.py
@@ -0,0 +1,53 @@
+from typing import List
+
+from pilot.scene.base import ChatScene
+from pilot.scene.base_chat import BaseChat
+
+
+class QueryReinforce:
+ """
+ query reinforce, include query rewrite, query correct
+ """
+
+ def __init__(
+ self, query: str = None, model_name: str = None, llm_chat: BaseChat = None
+ ):
+ """query reinforce
+ Args:
+ - query: str, user query
+ - model_name: str, llm model name
+ """
+ self.query = query
+ self.model_name = model_name
+ self.llm_chat = llm_chat
+
+ async def rewrite(self) -> List[str]:
+ """query rewrite"""
+ from pilot.common.chat_util import llm_chat_response_nostream
+ import uuid
+
+ chat_param = {
+ "chat_session_id": uuid.uuid1(),
+ "current_user_input": self.query,
+ "select_param": 2,
+ "model_name": self.model_name,
+ "model_cache_enable": False,
+ }
+ tasks = [
+ llm_chat_response_nostream(
+ ChatScene.QueryRewrite.value(), **{"chat_param": chat_param}
+ )
+ ]
+ from pilot.common.chat_util import run_async_tasks
+
+ queries = await run_async_tasks(tasks=tasks, concurrency_limit=1)
+ queries = list(
+ filter(
+ lambda content: "LLMServer Generate Error" not in content,
+ queries,
+ )
+ )
+ return queries[0]
+
+ def correct(self) -> List[str]:
+ pass
diff --git a/pilot/rag/retriever/rerank.py b/pilot/rag/retriever/rerank.py
new file mode 100644
index 000000000..dd48be82b
--- /dev/null
+++ b/pilot/rag/retriever/rerank.py
@@ -0,0 +1,87 @@
+from abc import ABC
+from typing import List, Tuple, Optional
+
+
+class Ranker(ABC):
+ """Base Ranker"""
+
+ def __init__(self, topk: int, rank_fn: Optional[callable] = None):
+ """
+ abstract base ranker
+ Args:
+ topk: int
+ rank_fn: Optional[callable]
+ """
+ self.topk = topk
+ self.rank_fn = rank_fn
+
+ def rank(self, candidates_with_scores: List, topk: int):
+ """rank algorithm implementation return topk documents by candidates similarity score
+ Args:
+ candidates_with_scores: List[Tuple]
+ topk: int
+ Return:
+ List[Document]
+ """
+
+ pass
+
+ def _filter(self, candidates_with_scores: List):
+ """filter duplicate candidates documents"""
+ candidates_with_scores = sorted(
+ candidates_with_scores, key=lambda x: x[1], reverse=True
+ )
+ visited_docs = set()
+ new_candidates = []
+ for candidate_doc, score in candidates_with_scores:
+ if candidate_doc.page_content not in visited_docs:
+ new_candidates.append((candidate_doc, score))
+ visited_docs.add(candidate_doc.page_content)
+ return new_candidates
+
+
+class DefaultRanker(Ranker):
+ """Default Ranker"""
+
+ def __init__(self, topk: int, rank_fn: Optional[callable] = None):
+ super().__init__(topk, rank_fn)
+
+ def rank(self, candidates_with_scores: List[Tuple]):
+ """Default rank algorithm implementation
+ return topk documents by candidates similarity score
+ Args:
+ candidates_with_scores: List[Tuple]
+ Return:
+ List[Document]
+ """
+ candidates_with_scores = self._filter(candidates_with_scores)
+ if self.rank_fn is not None:
+ candidates_with_scores = self.rank_fn(candidates_with_scores)
+ else:
+ candidates_with_scores = sorted(
+ candidates_with_scores, key=lambda x: x[1], reverse=True
+ )
+ return [
+ (candidate_doc, score) for candidate_doc, score in candidates_with_scores
+ ][: self.topk]
+
+
+class RRFRanker(Ranker):
+ """RRF(Reciprocal Rank Fusion) Ranker"""
+
+ def __init__(self, topk: int, rank_fn: Optional[callable] = None):
+ super().__init__(topk, rank_fn)
+
+ def rank(self, candidates_with_scores: List):
+ """RRF rank algorithm implementation
+ This code implements an algorithm called Reciprocal Rank Fusion (RRF), is a method for combining multiple result sets with different relevance indicators into a single result set. RRF requires no tuning, and the different relevance indicators do not have to be related to each other to achieve high-quality results.
+ RRF uses the following formula to determine the score for ranking each document:
+ score = 0.0
+ for q in queries:
+ if d in result(q):
+ score += 1.0 / ( k + rank( result(q), d ) )
+ return score
+ reference:https://www.elastic.co/guide/en/elasticsearch/reference/current/rrf.html
+ """
+ # it will be implemented soon when multi recall is implemented
+ return candidates_with_scores
diff --git a/pilot/scene/base.py b/pilot/scene/base.py
index 9d9b66962..893bccdb0 100644
--- a/pilot/scene/base.py
+++ b/pilot/scene/base.py
@@ -106,6 +106,9 @@ class ChatScene(Enum):
ExtractEntity = Scene(
"extract_entity", "Extract Entity", "Extract Entity", ["Extract Select"], True
)
+ QueryRewrite = Scene(
+ "query_rewrite", "query_rewrite", "query_rewrite", ["query_rewrite"], True
+ )
@staticmethod
def of_mode(mode):
diff --git a/pilot/scene/base_chat.py b/pilot/scene/base_chat.py
index 864eb34b6..070c27b76 100644
--- a/pilot/scene/base_chat.py
+++ b/pilot/scene/base_chat.py
@@ -203,7 +203,7 @@ async def stream_call(self):
payload = await self.__call_base()
self.skip_echo_len = len(payload.get("prompt").replace("", " ")) + 11
- logger.info(f"Requert: \n{payload}")
+ logger.info(f"payload request: \n{payload}")
ai_response_text = ""
span = root_tracer.start_span(
"BaseChat.stream_call", metadata=self._get_span_metadata(payload)
@@ -214,7 +214,7 @@ async def stream_call(self):
async for output in await self._model_stream_operator.call_stream(
call_data={"data": payload}
):
- ### Plug-in research in result generation
+ # Plugin research in result generation
msg = self.prompt_template.output_parser.parse_model_stream_resp_ex(
output, self.skip_echo_len
)
@@ -227,7 +227,7 @@ async def stream_call(self):
span.end()
except Exception as e:
print(traceback.format_exc())
- logger.error("model response parase failed！" + str(e))
+ logger.error("model response parse failed！" + str(e))
self.current_message.add_view_message(
f"""ERROR!{str(e)}\n {ai_response_text} """
)
diff --git a/pilot/scene/chat_factory.py b/pilot/scene/chat_factory.py
index 10a588c04..256cd3ebe 100644
--- a/pilot/scene/chat_factory.py
+++ b/pilot/scene/chat_factory.py
@@ -18,6 +18,7 @@ def get_implementation(chat_mode, **kwargs):
from pilot.scene.chat_knowledge.extract_entity.chat import ExtractEntity
from pilot.scene.chat_knowledge.summary.chat import ExtractSummary
from pilot.scene.chat_knowledge.refine_summary.chat import ExtractRefineSummary
+ from pilot.scene.chat_knowledge.rewrite.chat import QueryRewrite
from pilot.scene.chat_data.chat_excel.excel_analyze.chat import ChatExcel
from pilot.scene.chat_agent.chat import ChatAgent
diff --git a/pilot/scene/chat_knowledge/rewrite/__init__.py b/pilot/scene/chat_knowledge/rewrite/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/pilot/scene/chat_knowledge/rewrite/chat.py b/pilot/scene/chat_knowledge/rewrite/chat.py
new file mode 100644
index 000000000..48d575abc
--- /dev/null
+++ b/pilot/scene/chat_knowledge/rewrite/chat.py
@@ -0,0 +1,36 @@
+from typing import Dict
+
+from pilot.scene.base_chat import BaseChat
+from pilot.scene.base import ChatScene
+from pilot.configs.config import Config
+
+from pilot.scene.chat_knowledge.rewrite.prompt import prompt
+
+CFG = Config()
+
+
+class QueryRewrite(BaseChat):
+ chat_scene: str = ChatScene.QueryRewrite.value()
+
+ """query rewrite by llm"""
+
+ def __init__(self, chat_param: Dict):
+ """ """
+ chat_param["chat_mode"] = ChatScene.QueryRewrite
+ super().__init__(
+ chat_param=chat_param,
+ )
+
+ self.nums = chat_param["select_param"]
+ self.current_user_input = chat_param["current_user_input"]
+
+ async def generate_input_values(self):
+ input_values = {
+ "nums": self.nums,
+ "original_query": self.current_user_input,
+ }
+ return input_values
+
+ @property
+ def chat_type(self) -> str:
+ return ChatScene.QueryRewrite.value
diff --git a/pilot/scene/chat_knowledge/rewrite/out_parser.py b/pilot/scene/chat_knowledge/rewrite/out_parser.py
new file mode 100644
index 000000000..9a3ce5092
--- /dev/null
+++ b/pilot/scene/chat_knowledge/rewrite/out_parser.py
@@ -0,0 +1,47 @@
+import logging
+from typing import List, Tuple
+
+from pilot.out_parser.base import BaseOutputParser, T
+from pilot.configs.config import Config
+
+CFG = Config()
+
+
+logger = logging.getLogger(__name__)
+
+
+class QueryRewriteParser(BaseOutputParser):
+ def __init__(self, sep: str, is_stream_out: bool):
+ super().__init__(sep=sep, is_stream_out=is_stream_out)
+
+ def parse_prompt_response(self, response, max_length: int = 128):
+ lowercase = True
+ try:
+ results = []
+ response = response.strip()
+
+ if response.startswith("queries:"):
+ response = response[len("queries:") :]
+
+ queries = response.split(",")
+ if len(queries) == 1:
+ queries = response.split("，")
+ if len(queries) == 1:
+ queries = response.split("?")
+ if len(queries) == 1:
+ queries = response.split("？")
+ for k in queries:
+ rk = k
+ if lowercase:
+ rk = rk.lower()
+ s = rk.strip()
+ if s == "":
+ continue
+ results.append(s)
+ except Exception as e:
+ logger.error(f"parse query rewrite prompt_response error: {e}")
+ return []
+ return results
+
+ def parse_view_response(self, speak, data) -> str:
+ return data
diff --git a/pilot/scene/chat_knowledge/rewrite/prompt.py b/pilot/scene/chat_knowledge/rewrite/prompt.py
new file mode 100644
index 000000000..4df4f08c8
--- /dev/null
+++ b/pilot/scene/chat_knowledge/rewrite/prompt.py
@@ -0,0 +1,47 @@
+from pilot.prompts.prompt_new import PromptTemplate
+from pilot.configs.config import Config
+from pilot.scene.base import ChatScene
+from pilot.common.schema import SeparatorStyle
+
+from .out_parser import QueryRewriteParser
+
+CFG = Config()
+
+
+PROMPT_SCENE_DEFINE = """You are a helpful assistant that generates multiple search queries based on a single input query."""
+
+_DEFAULT_TEMPLATE_ZH = """请根据原问题优化生成{nums}个相关的搜索查询，这些查询应与原始查询相似并且是人们可能会提出的可回答的搜索问题。请勿使用任何示例中提到的内容，确保所有生成的查询均独立于示例，仅基于提供的原始查询。请按照以下逗号分隔的格式提供: 'queries：'：
+"original_query：{original_query}\n"
+"queries：\n"
+"""
+
+_DEFAULT_TEMPLATE_EN = """
+Generate {nums} search queries related to: {original_query}, Provide following comma-separated format: 'queries: '\n":
+ "original query:: {original_query}\n"
+ "queries:\n"
+"""
+
+_DEFAULT_TEMPLATE = (
+ _DEFAULT_TEMPLATE_EN if CFG.LANGUAGE == "en" else _DEFAULT_TEMPLATE_ZH
+)
+
+PROMPT_RESPONSE = """"""
+
+PROMPT_SEP = SeparatorStyle.SINGLE.value
+
+PROMPT_NEED_NEED_STREAM_OUT = True
+
+prompt = PromptTemplate(
+ template_scene=ChatScene.QueryRewrite.value(),
+ input_variables=["nums", "original_query"],
+ response_format=None,
+ template_define=PROMPT_SCENE_DEFINE,
+ template=_DEFAULT_TEMPLATE + PROMPT_RESPONSE,
+ stream_out=PROMPT_NEED_NEED_STREAM_OUT,
+ output_parser=QueryRewriteParser(
+ sep=PROMPT_SEP, is_stream_out=PROMPT_NEED_NEED_STREAM_OUT
+ ),
+)
+
+CFG.prompt_template_registry.register(prompt, is_default=True)
+from ..v1 import prompt_chatglm
diff --git a/pilot/scene/chat_knowledge/v1/chat.py b/pilot/scene/chat_knowledge/v1/chat.py
index a0c15e658..0bb8cf846 100644
--- a/pilot/scene/chat_knowledge/v1/chat.py
+++ b/pilot/scene/chat_knowledge/v1/chat.py
@@ -1,18 +1,22 @@
import json
import os
+from functools import reduce
from typing import Dict, List
-from pilot.component import ComponentType
from pilot.scene.base_chat import BaseChat
from pilot.scene.base import ChatScene
from pilot.configs.config import Config
from pilot.configs.model_config import (
- KNOWLEDGE_UPLOAD_ROOT_PATH,
EMBEDDING_MODEL_CONFIG,
)
from pilot.scene.chat_knowledge.v1.prompt import prompt
+from pilot.server.knowledge.chunk_db import DocumentChunkDao, DocumentChunkEntity
+from pilot.server.knowledge.document_db import (
+ KnowledgeDocumentDao,
+ KnowledgeDocumentEntity,
+)
from pilot.server.knowledge.service import KnowledgeService
from pilot.utils.executor_utils import blocking_func_to_async
from pilot.utils.tracer import root_tracer, trace
@@ -47,6 +51,11 @@ def __init__(self, chat_param: Dict):
if self.space_context is None
else int(self.space_context["embedding"]["topk"])
)
+ self.recall_score = (
+ CFG.KNOWLEDGE_SEARCH_RECALL_SCORE
+ if self.space_context is None
+ else float(self.space_context["embedding"]["recall_score"])
+ )
self.max_token = (
CFG.KNOWLEDGE_SEARCH_MAX_TOKEN
if self.space_context is None or self.space_context.get("prompt") is None
@@ -65,11 +74,16 @@ def __init__(self, chat_param: Dict):
embedding_factory=embedding_factory,
)
self.prompt_template.template_is_strict = False
+ self.relations = None
+ self.chunk_dao = DocumentChunkDao()
+ document_dao = KnowledgeDocumentDao()
+ documents = document_dao.get_documents(
+ query=KnowledgeDocumentEntity(space=self.knowledge_space)
+ )
+ if len(documents) > 0:
+ self.document_ids = [document.id for document in documents]
async def stream_call(self):
- input_values = await self.generate_input_values()
- # Source of knowledge file
- relations = input_values.get("relations")
last_output = None
async for output in super().stream_call():
last_output = output
@@ -78,76 +92,118 @@ async def stream_call(self):
if (
CFG.KNOWLEDGE_CHAT_SHOW_RELATIONS
and last_output
- and type(relations) == list
- and len(relations) > 0
+ and type(self.relations) == list
+ and len(self.relations) > 0
and hasattr(last_output, "text")
):
last_output.text = (
- last_output.text + "\n\nrelations:\n\n" + ",".join(relations)
+ last_output.text + "\n\nrelations:\n\n" + ",".join(self.relations)
)
- reference = f"\n\n{self.parse_source_view(self.sources)}"
+ reference = f"\n\n{self.parse_source_view(self.chunks_with_score)}"
last_output = last_output + reference
yield last_output
def stream_call_reinforce_fn(self, text):
"""return reference"""
- return text + f"\n\n{self.parse_source_view(self.sources)}"
+ return text + f"\n\n{self.parse_source_view(self.chunks_with_score)}"
@trace()
async def generate_input_values(self) -> Dict:
if self.space_context and self.space_context.get("prompt"):
self.prompt_template.template_define = self.space_context["prompt"]["scene"]
self.prompt_template.template = self.space_context["prompt"]["template"]
- docs = await blocking_func_to_async(
- self._executor,
- self.knowledge_embedding_client.similar_search,
- self.current_user_input,
- self.top_k,
- )
- self.sources = _merge_by_key(
- list(map(lambda doc: doc.metadata, docs)), "source"
- )
+ from pilot.rag.retriever.reinforce import QueryReinforce
- if not docs or len(docs) == 0:
+ # query reinforce, get similar queries
+ query_reinforce = QueryReinforce(
+ query=self.current_user_input, model_name=self.llm_model
+ )
+ queries = []
+ if CFG.KNOWLEDGE_SEARCH_REWRITE:
+ queries = await query_reinforce.rewrite()
+ print("rewrite queries:", queries)
+ queries.append(self.current_user_input)
+ from pilot.common.chat_util import run_async_tasks
+
+ # similarity search from vector db
+ tasks = [self.execute_similar_search(query) for query in queries]
+ docs_with_scores = await run_async_tasks(tasks=tasks, concurrency_limit=1)
+ candidates_with_scores = reduce(lambda x, y: x + y, docs_with_scores)
+ # candidates document rerank
+ from pilot.rag.retriever.rerank import DefaultRanker
+
+ ranker = DefaultRanker(self.top_k)
+ candidates_with_scores = ranker.rank(candidates_with_scores)
+ self.chunks_with_score = []
+ if not candidates_with_scores or len(candidates_with_scores) == 0:
print("no relevant docs to retrieve")
context = "no relevant docs to retrieve"
else:
- context = [d.page_content for d in docs]
+ self.chunks_with_score = []
+ for d, score in candidates_with_scores:
+ chucks = self.chunk_dao.get_document_chunks(
+ query=DocumentChunkEntity(content=d.page_content),
+ document_ids=self.document_ids,
+ )
+ if len(chucks) > 0:
+ self.chunks_with_score.append((chucks[0], score))
+
+ context = [doc.page_content for doc, _ in candidates_with_scores]
+
context = context[: self.max_token]
- relations = list(
- set([os.path.basename(str(d.metadata.get("source", ""))) for d in docs])
+ self.relations = list(
+ set(
+ [
+ os.path.basename(str(d.metadata.get("source", "")))
+ for d, _ in candidates_with_scores
+ ]
+ )
)
input_values = {
"context": context,
"question": self.current_user_input,
- "relations": relations,
+ "relations": self.relations,
}
return input_values
- def parse_source_view(self, sources: List):
+ def parse_source_view(self, chunks_with_score: List):
"""
- build knowledge reference view message to web
- {
- "title":"References",
- "references":[{
- "name":"aa.pdf",
- "pages":["1","2","3"]
- }]
- }
+ format knowledge reference view message to web
+
"""
- references = {"title": "References", "references": []}
- for item in sources:
- reference = {}
- source = item["source"] if "source" in item else ""
- reference["name"] = source
- pages = item["pages"] if "pages" in item else []
- if len(pages) > 0:
- reference["pages"] = pages
- references["references"].append(reference)
- html = (
- f"""{json.dumps(references, ensure_ascii=False)}"""
- )
- return html
+ import xml.etree.ElementTree as ET
+
+ references_ele = ET.Element("references")
+ title = "References"
+ references_ele.set("title", title)
+ references_dict = {}
+ for chunk, score in chunks_with_score:
+ doc_name = chunk.doc_name
+ if doc_name not in references_dict:
+ references_dict[doc_name] = {
+ "name": doc_name,
+ "chunks": [
+ {
+ "id": chunk.id,
+ "content": chunk.content,
+ "meta_info": chunk.meta_info,
+ "recall_score": score,
+ }
+ ],
+ }
+ else:
+ references_dict[doc_name]["chunks"].append(
+ {
+ "id": chunk.id,
+ "content": chunk.content,
+ "meta_info": chunk.meta_info,
+ "recall_score": score,
+ }
+ )
+ references_list = list(references_dict.values())
+ references_ele.set("references", json.dumps(references_list))
+ html = ET.tostring(references_ele, encoding="utf-8")
+ return html.decode("utf-8")
@property
def chat_type(self) -> str:
@@ -157,26 +213,12 @@ def get_space_context(self, space_name):
service = KnowledgeService()
return service.get_space_context(space_name)
-
-def _merge_by_key(data, key):
- result = {}
- for item in data:
- if item.get(key):
- item_key = os.path.basename(item.get(key))
- if item_key in result:
- if "pages" in result[item_key] and "page" in item:
- result[item_key]["pages"].append(str(item["page"]))
- elif "page" in item:
- result[item_key]["pages"] = [
- result[item_key]["pages"],
- str(item["page"]),
- ]
- else:
- if "page" in item:
- result[item_key] = {
- "source": item_key,
- "pages": [str(item["page"])],
- }
- else:
- result[item_key] = {"source": item_key}
- return list(result.values())
+ async def execute_similar_search(self, query):
+ """execute similarity search"""
+ return await blocking_func_to_async(
+ self._executor,
+ self.knowledge_embedding_client.similar_search_with_scores,
+ query,
+ self.top_k,
+ self.recall_score,
+ )
diff --git a/pilot/scene/operator/_experimental.py b/pilot/scene/operator/_experimental.py
index f0ee06179..1f5ba87d2 100644
--- a/pilot/scene/operator/_experimental.py
+++ b/pilot/scene/operator/_experimental.py
@@ -184,7 +184,6 @@ async def map(self, input_value: ChatContext) -> ChatContext:
from pilot.configs.model_config import EMBEDDING_MODEL_CONFIG
from pilot.embedding_engine.embedding_engine import EmbeddingEngine
from pilot.embedding_engine.embedding_factory import EmbeddingFactory
- from pilot.scene.chat_knowledge.v1.chat import _merge_by_key
# TODO, decompose the current operator into some atomic operators
knowledge_space = input_value.select_param
@@ -223,7 +222,6 @@ async def map(self, input_value: ChatContext) -> ChatContext:
input_value.current_user_input,
top_k,
)
- sources = _merge_by_key(list(map(lambda doc: doc.metadata, docs)), "source")
if not docs or len(docs) == 0:
print("no relevant docs to retrieve")
context = "no relevant docs to retrieve"
diff --git a/pilot/server/knowledge/chunk_db.py b/pilot/server/knowledge/chunk_db.py
index 1df3cf895..5110f34fc 100644
--- a/pilot/server/knowledge/chunk_db.py
+++ b/pilot/server/knowledge/chunk_db.py
@@ -61,7 +61,9 @@ def create_documents_chunks(self, documents: List):
session.commit()
session.close()
- def get_document_chunks(self, query: DocumentChunkEntity, page=1, page_size=20):
+ def get_document_chunks(
+ self, query: DocumentChunkEntity, page=1, page_size=20, document_ids=None
+ ):
session = self.get_session()
document_chunks = session.query(DocumentChunkEntity)
if query.id is not None:
@@ -74,6 +76,10 @@ def get_document_chunks(self, query: DocumentChunkEntity, page=1, page_size=20):
document_chunks = document_chunks.filter(
DocumentChunkEntity.doc_type == query.doc_type
)
+ if query.content is not None:
+ document_chunks = document_chunks.filter(
+ DocumentChunkEntity.content == query.content
+ )
if query.doc_name is not None:
document_chunks = document_chunks.filter(
DocumentChunkEntity.doc_name == query.doc_name
@@ -82,6 +88,10 @@ def get_document_chunks(self, query: DocumentChunkEntity, page=1, page_size=20):
document_chunks = document_chunks.filter(
DocumentChunkEntity.meta_info == query.meta_info
)
+ if document_ids is not None:
+ document_chunks = document_chunks.filter(
+ DocumentChunkEntity.document_id.in_(document_ids)
+ )
document_chunks = document_chunks.order_by(DocumentChunkEntity.id.asc())
document_chunks = document_chunks.offset((page - 1) * page_size).limit(
@@ -116,12 +126,6 @@ def get_document_chunks_count(self, query: DocumentChunkEntity):
session.close()
return count
- # def update_knowledge_document(self, document:KnowledgeDocumentEntity):
- # session = self.get_session()
- # updated_space = session.merge(document)
- # session.commit()
- # return updated_space.id
-
def delete(self, document_id: int):
session = self.get_session()
if document_id is None:
diff --git a/pilot/server/knowledge/service.py b/pilot/server/knowledge/service.py
index fac027920..a42396582 100644
--- a/pilot/server/knowledge/service.py
+++ b/pilot/server/knowledge/service.py
@@ -56,7 +56,12 @@ class SyncStatus(Enum):
FINISHED = "FINISHED"
-# @singleton
+# default summary max iteration call with llm.
+DEFAULT_SUMMARY_MAX_ITERATION = 5
+# default summary concurrency call with llm.
+DEFAULT_SUMMARY_CONCURRENCY_LIMIT = 3
+
+
class KnowledgeService:
"""KnowledgeService
Knowledge Management Service:
@@ -425,7 +430,7 @@ def async_knowledge_graph(self, chunk_docs, doc):
f"async_knowledge_graph, doc:{doc.doc_name}, chunk_size:{len(chunk_docs)}, begin embedding to graph store"
)
try:
- from pilot.graph_engine.graph_factory import RAGGraphFactory
+ from pilot.rag.graph_engine.graph_factory import RAGGraphFactory
rag_engine = CFG.SYSTEM_APP.get_component(
ComponentType.RAG_GRAPH_DEFAULT.value, RAGGraphFactory
@@ -502,7 +507,7 @@ def _build_default_context(self):
context_template = {
"embedding": {
"topk": CFG.KNOWLEDGE_SEARCH_TOP_SIZE,
- "recall_score": 0.0,
+ "recall_score": CFG.KNOWLEDGE_SEARCH_RECALL_SCORE,
"recall_type": "TopK",
"model": EMBEDDING_MODEL_CONFIG[CFG.EMBEDDING_MODEL].rsplit("/", 1)[-1],
"chunk_size": CFG.KNOWLEDGE_CHUNK_SIZE,
@@ -514,8 +519,8 @@ def _build_default_context(self):
"template": _DEFAULT_TEMPLATE,
},
"summary": {
- "max_iteration": 5,
- "concurrency_limit": 3,
+ "max_iteration": DEFAULT_SUMMARY_MAX_ITERATION,
+ "concurrency_limit": DEFAULT_SUMMARY_CONCURRENCY_LIMIT,
},
}
context_template_string = json.dumps(context_template, indent=4)
diff --git a/pilot/server/static/404.html b/pilot/server/static/404.html
index 49f15e208..edc7eb0c5 100644
--- a/pilot/server/static/404.html
+++ b/pilot/server/static/404.html
@@ -1 +1 @@
-404: This page could not be found
\ No newline at end of file
diff --git a/pilot/server/static/404/index.html b/pilot/server/static/404/index.html
index 49f15e208..edc7eb0c5 100644
--- a/pilot/server/static/404/index.html
+++ b/pilot/server/static/404/index.html
@@ -1 +1 @@
-404: This page could not be found
\ No newline at end of file
diff --git a/pilot/server/static/chat/[scene]/[id]/index.html b/pilot/server/static/chat/[scene]/[id]/index.html
index a94476e87..5b9f827bf 100644
--- a/pilot/server/static/chat/[scene]/[id]/index.html
+++ b/pilot/server/static/chat/[scene]/[id]/index.html
@@ -1 +1 @@
-
\ No newline at end of file
diff --git a/pilot/server/static/chat/index.html b/pilot/server/static/chat/index.html
index b05c6f855..78763b50b 100644
--- a/pilot/server/static/chat/index.html
+++ b/pilot/server/static/chat/index.html
@@ -1 +1 @@
-
\ No newline at end of file
diff --git a/pilot/server/static/database/index.html b/pilot/server/static/database/index.html
index 296b16484..5d3bfa3e1 100644
--- a/pilot/server/static/database/index.html
+++ b/pilot/server/static/database/index.html
@@ -1 +1 @@
-
Powerful, scalable, secure relational database system by Microsoft.
DuckDB
In-memory analytical database with efficient query processing.
Sqlite
Lightweight embedded relational database with simplicity and portability.
ClickHouse
Columnar database for high-performance analytics and real-time queries.
Oracle
Robust, scalable, secure relational database widely used in enterprises.
Access
Easy-to-use relational database for small-scale applications by Microsoft.
MongoDB
Flexible, scalable NoSQL document database for web and mobile apps.
DB2
Scalable, secure relational database system developed by IBM.
HBase
Distributed, scalable NoSQL database for large structured/semi-structured data.
Redis
Fast, versatile in-memory data structure store as cache, DB, or broker.
Cassandra
Scalable, fault-tolerant distributed NoSQL database for large data.
Couchbase
High-performance NoSQL document database with distributed architecture.
PostgreSQL
Powerful open-source relational database with extensibility and SQL standards.
Spark
Unified engine for large-scale data analytics.
Space
knowledge analytics.
\ No newline at end of file
diff --git a/pilot/server/static/index.html b/pilot/server/static/index.html
index 36a2e799c..c7f985e24 100644
--- a/pilot/server/static/index.html
+++ b/pilot/server/static/index.html
@@ -1 +1 @@
-
\ No newline at end of file
diff --git a/pilot/server/static/knowledge/chunk/index.html b/pilot/server/static/knowledge/chunk/index.html
index a92646d2a..7c01d201d 100644
--- a/pilot/server/static/knowledge/chunk/index.html
+++ b/pilot/server/static/knowledge/chunk/index.html
@@ -1 +1 @@
-
\ No newline at end of file
diff --git a/pilot/server/static/knowledge/index.html b/pilot/server/static/knowledge/index.html
index a3e590620..805d08a6c 100644
--- a/pilot/server/static/knowledge/index.html
+++ b/pilot/server/static/knowledge/index.html
@@ -1 +1 @@
-
\ No newline at end of file
diff --git a/pilot/server/static/models/index.html b/pilot/server/static/models/index.html
index a2a4dc193..b6cf8a989 100644
--- a/pilot/server/static/models/index.html
+++ b/pilot/server/static/models/index.html
@@ -1 +1 @@
-
\ No newline at end of file
diff --git a/pilot/server/static/prompt/index.html b/pilot/server/static/prompt/index.html
index 70551f6a7..83429b9a0 100644
--- a/pilot/server/static/prompt/index.html
+++ b/pilot/server/static/prompt/index.html
@@ -1 +1 @@
-
