Merge branch 'eosphoros-ai:main' into main

.env.template

@@ -1,16 +1,11 @@
 #*******************************************************************#
 #**             DB-GPT  - GENERAL SETTINGS                        **#  
 #*******************************************************************#
-## DISABLED_COMMAND_CATEGORIES - The list of categories of commands that are disabled. Each of the below are an option:
-## pilot.commands.query_execute
-
-## For example, to disable coding related features, uncomment the next line
-# DISABLED_COMMAND_CATEGORIES=   
 
 #*******************************************************************#
 #**                        Webserver Port                         **#
 #*******************************************************************#
-WEB_SERVER_PORT=7860
+# DBGPT_WEBSERVER_PORT=5670
 
 #*******************************************************************#
 #***                       LLM PROVIDER                          ***#
@@ -21,15 +16,15 @@ WEB_SERVER_PORT=7860
 #*******************************************************************#
 #**                         LLM MODELS                            **#
 #*******************************************************************#
-# LLM_MODEL, see /pilot/configs/model_config.LLM_MODEL_CONFIG
+# LLM_MODEL, see dbgpt/configs/model_config.LLM_MODEL_CONFIG
 LLM_MODEL=vicuna-13b-v1.5
 ## LLM model path, by default, DB-GPT will read the model path from LLM_MODEL_CONFIG based on the LLM_MODEL.
 ## Of course you can specify your model path according to LLM_MODEL_PATH
 ## In DB-GPT, the priority from high to low to read model path:
 ##    1. environment variable with key: {LLM_MODEL}_MODEL_PATH (Avoid multi-model conflicts)
 ##    2. environment variable with key: MODEL_PATH
 ##    3. environment variable with key: LLM_MODEL_PATH
-##    4. the config in /pilot/configs/model_config.LLM_MODEL_CONFIG
+##    4. the config in dbgpt/configs/model_config.LLM_MODEL_CONFIG
 # LLM_MODEL_PATH=/app/models/vicuna-13b-v1.5
 # LLM_PROMPT_TEMPLATE=vicuna_v1.1
 MODEL_SERVER=http://127.0.0.1:8000
@@ -47,7 +42,7 @@ QUANTIZE_8bit=True
 # PROXYLLM_BACKEND=
 
 ### You can configure parameters for a specific model with {model name}_{config key}=xxx
-### See /pilot/model/parameter.py
+### See dbgpt/model/parameter.py
 ## prompt template for current model
 # llama_cpp_prompt_template=vicuna_v1.1
 ## llama-2-70b must be 8
@@ -75,6 +70,11 @@ EMBEDDING_MODEL=text2vec
 #EMBEDDING_MODEL=bge-large-zh
 KNOWLEDGE_CHUNK_SIZE=500
 KNOWLEDGE_SEARCH_TOP_SIZE=5
+KNOWLEDGE_GRAPH_SEARCH_TOP_SIZE=50
+## Maximum number of chunks to load at once, if your single document is too large,
+## you can set this value to a higher value for better performance.
+## if out of memory when load large document, you can set this value to a lower value.
+# KNOWLEDGE_MAX_CHUNKS_ONCE_LOAD=10
 #KNOWLEDGE_CHUNK_OVERLAP=50
 # Control whether to display the source document of knowledge on the front end.
 KNOWLEDGE_CHAT_SHOW_RELATIONS=False
@@ -86,12 +86,36 @@ KNOWLEDGE_SEARCH_REWRITE=False
 # EMBEDDING_TOKENIZER=all-MiniLM-L6-v2
 # EMBEDDING_TOKEN_LIMIT=8191
 
-## Openai embedding model, See /pilot/model/parameter.py
+## Openai embedding model, See dbgpt/model/parameter.py
 # EMBEDDING_MODEL=proxy_openai
 # proxy_openai_proxy_server_url=https://api.openai.com/v1
 # proxy_openai_proxy_api_key={your-openai-sk}
 # proxy_openai_proxy_backend=text-embedding-ada-002
 
+## Common HTTP embedding model
+# EMBEDDING_MODEL=proxy_http_openapi
+# proxy_http_openapi_proxy_server_url=http://localhost:8100/api/v1/embeddings
+# proxy_http_openapi_proxy_api_key=1dce29a6d66b4e2dbfec67044edbb924
+# proxy_http_openapi_proxy_backend=text2vec
+
+#*******************************************************************#
+#**                         RERANK SETTINGS                       **#
+#*******************************************************************#
+## Rerank model
+# RERANK_MODEL=bge-reranker-base
+## If you not set RERANK_MODEL_PATH, DB-GPT will read the model path from EMBEDDING_MODEL_CONFIG based on the RERANK_MODEL.
+# RERANK_MODEL_PATH=
+## The number of rerank results to return
+# RERANK_TOP_K=3
+
+## Common HTTP rerank model
+# RERANK_MODEL=rerank_proxy_http_openapi
+# rerank_proxy_http_openapi_proxy_server_url=http://127.0.0.1:8100/api/v1/beta/relevance
+# rerank_proxy_http_openapi_proxy_api_key={your-api-key}
+# rerank_proxy_http_openapi_proxy_backend=bge-reranker-base
+
+
+
 
 #*******************************************************************#
 #**                  DB-GPT METADATA DATABASE SETTINGS            **#
@@ -114,30 +138,13 @@ LOCAL_DB_TYPE=sqlite
 #*******************************************************************#
 EXECUTE_LOCAL_COMMANDS=False
 
-
-
-#*******************************************************************#
-#**                  ALLOWLISTED PLUGINS                          **#
-#*******************************************************************#
-
-#ALLOWLISTED_PLUGINS - Sets the listed plugins that are allowed (Example: plugin1,plugin2,plugin3)
-#DENYLISTED_PLUGINS - Sets the listed plugins that are not allowed (Example: plugin1,plugin2,plugin3)
-ALLOWLISTED_PLUGINS=
-DENYLISTED_PLUGINS=
-
-
 #*******************************************************************#
-#**                 CHAT PLUGIN SETTINGS                          **#
+#**            VECTOR STORE / KNOWLEDGE GRAPH SETTINGS            **#
 #*******************************************************************#
-# CHAT_MESSAGES_ENABLED - Enable chat messages (Default: False)
-# CHAT_MESSAGES_ENABLED=False
-
+VECTOR_STORE_TYPE=Chroma
+GRAPH_STORE_TYPE=TuGraph
 
-#*******************************************************************#
-#**                  VECTOR STORE SETTINGS                       **#
-#*******************************************************************#
 ### Chroma vector db config
-VECTOR_STORE_TYPE=Chroma
 #CHROMA_PERSIST_PATH=/root/DB-GPT/pilot/data
 
 ### Milvus vector db config
@@ -152,9 +159,26 @@ VECTOR_STORE_TYPE=Chroma
 #VECTOR_STORE_TYPE=Weaviate
 #WEAVIATE_URL=https://kt-region-m8hcy0wc.weaviate.network
 
+## ElasticSearch vector db config
+#VECTOR_STORE_TYPE=ElasticSearch
+ElasticSearch_URL=127.0.0.1
+ElasticSearch_PORT=9200
+ElasticSearch_USERNAME=elastic
+ElasticSearch_PASSWORD=i=+iLw9y0Jduq86XTi6W
+
+### TuGraph config
+#TUGRAPH_HOST=127.0.0.1
+#TUGRAPH_PORT=7687
+#TUGRAPH_USERNAME=admin
+#TUGRAPH_PASSWORD=73@TuGraph
+#TUGRAPH_VERTEX_TYPE=entity
+#TUGRAPH_EDGE_TYPE=relation
+#TUGRAPH_EDGE_NAME_KEY=label
+
 #*******************************************************************#
 #**                  WebServer Language Support                   **#
 #*******************************************************************#
+# en, zh, fr, ja, ko, ru
 LANGUAGE=en
 #LANGUAGE=zh
 
@@ -180,7 +204,7 @@ TONGYI_PROXY_API_KEY={your-tongyi-sk}
 ## Baidu wenxin
 #WEN_XIN_MODEL_VERSION={version}
 #WEN_XIN_API_KEY={your-wenxin-sk}
-#WEN_XIN_SECRET_KEY={your-wenxin-sct}
+#WEN_XIN_API_SECRET={your-wenxin-sct}
 
 ## Zhipu
 #ZHIPU_MODEL_VERSION={version}
@@ -197,6 +221,20 @@ TONGYI_PROXY_API_KEY={your-tongyi-sk}
 #XUNFEI_SPARK_API_KEY={your_api_key}
 #XUNFEI_SPARK_API_SECRET={your_api_secret}
 
+## Yi Proxyllm, https://platform.lingyiwanwu.com/docs
+#YI_MODEL_VERSION=yi-34b-chat-0205
+#YI_API_BASE=https://api.lingyiwanwu.com/v1
+#YI_API_KEY={your-yi-api-key}
+
+## Moonshot Proxyllm, https://platform.moonshot.cn/docs/
+# MOONSHOT_MODEL_VERSION=moonshot-v1-8k
+# MOONSHOT_API_BASE=https://api.moonshot.cn/v1
+# MOONSHOT_API_KEY={your-moonshot-api-key}
+
+## Deepseek Proxyllm, https://platform.deepseek.com/api-docs/
+# DEEPSEEK_MODEL_VERSION=deepseek-chat
+# DEEPSEEK_API_BASE=https://api.deepseek.com/v1
+# DEEPSEEK_API_KEY={your-deepseek-api-key}
 
 
 #*******************************************************************#
@@ -219,4 +257,20 @@ SUMMARY_CONFIG=FAST
 # FATAL, ERROR, WARNING, WARNING, INFO, DEBUG, NOTSET
 DBGPT_LOG_LEVEL=INFO
 # LOG dir, default: ./logs
-#DBGPT_LOG_DIR=
+#DBGPT_LOG_DIR=
+
+
+#*******************************************************************#
+#**                         API_KEYS                              **#
+#*******************************************************************#
+# API_KEYS - The list of API keys that are allowed to access the API. Each of the below are an option, separated by commas.
+# API_KEYS=dbgpt
+
+
+#*******************************************************************#
+#**                     Application Config                        **#
+#*******************************************************************#
+## Non-streaming scene retries
+# DBGPT_APP_SCENE_NON_STREAMING_RETRIES_BASE=1
+## Non-streaming scene parallelism
+# DBGPT_APP_SCENE_NON_STREAMING_PARALLELISM_BASE=1

.flake8

@@ -0,0 +1,41 @@
+[flake8]
+exclude =
+  .eggs/
+  build/
+  */tests/*
+  *_private
+max-line-length = 88
+inline-quotes = "
+ignore =
+  C408
+  C417
+  E121
+  E123
+  E126
+  E203
+  E226
+  E231
+  E24
+  E704
+  W503
+  W504
+  W605
+  I
+  N
+  B001
+  B002
+  B003
+  B004
+  B005
+  B007
+  B008
+  B009
+  B010
+  B011
+  B012
+  B013
+  B014
+  B015
+  B016
+  B017
+avoid-escape = no

.github/PULL_REQUEST_TEMPLATE/pull_request_template.md

@@ -0,0 +1,20 @@
+# Description
+
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
+
+# How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
+
+# Snapshots:
+
+Include snapshots for easier review.
+
+# Checklist:
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have already rebased the commits and make the commit message conform to the project standard.
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] Any dependent changes have been merged and published in downstream modules

.github/pull_request_template.md

@@ -0,0 +1,20 @@
+# Description
+
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
+
+# How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
+
+# Snapshots:
+
+Include snapshots for easier review.
+
+# Checklist:
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have already rebased the commits and make the commit message conform to the project standard.
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] Any dependent changes have been merged and published in downstream modules

.github/workflows/close-issue.yml

@@ -0,0 +1,22 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "00 21 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v5
+        with:
+          days-before-issue-stale: 30
+          days-before-issue-close: 7 
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue has been marked as `stale`, because it has been over 30 days without any activity."
+          close-issue-message: "This issue bas been closed, because it has been marked as `stale` and there has been no activity for over 7 days."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pylint.yml

@@ -16,17 +16,16 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.10", "3.11"]
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -U black isort
-    - name: check the code lint
-      run: |
-        black . --check
+    - name: Install dependencies and setup environment
+      run: make setup
+    - name: Check Python code style
+      run: make fmt-check
+    - name: Check Python code type
+      run: make mypy

.github/workflows/sync-docs.yaml

@@ -0,0 +1,20 @@
+name: Trigger Auto Publish
+
+on:
+  push:
+    tags:
+      - "*"
+
+jobs:
+  trigger-api:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Trigger Publish API
+        run: |
+          curl -X POST ${{secrets.PUBLISH_SECRET_API}} \
+          -H "Content-Type: application/json" \
+          -d '{"tag": "${{ github.ref }}"}'