pre release (#151)

docker/bisheng/config/config.yaml

@@ -7,7 +7,7 @@ redis_url: "redis://redis:6379/0"
 
 environment:
   env: dev
-  uns_support: ['png','jpg','jpeg','bmp','doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx', 'txt', 'md', 'html', 'pdf']
+  uns_support: ['png','jpg','jpeg','bmp','doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx', 'txt', 'md', 'html', 'pdf', 'csv', 'tiff']
 
 # admin 用户配置
 admin:

src/backend/bisheng/api/v1/callback.py

@@ -91,8 +91,8 @@ async def on_text(self, text: str, **kwargs: Any) -> Any:
         receiver = kwargs.get('receiver')
         if kwargs.get('sender'):
             log = ChatResponse(message=text, type='end',
-                               sender=sender, recevier=receiver)
-            start = ChatResponse(type='start', sender=sender, recevier=receiver)
+                               sender=sender, receiver=receiver)
+            start = ChatResponse(type='start', sender=sender, receiver=receiver)
 
             if receiver and receiver.get('is_self'):
                 await self.websocket.send_json(log.dict())

src/backend/bisheng/api/v1/knowledge.py

@@ -114,26 +114,32 @@ async def process_knowledge(*,
     for path in file_path:
         filepath, file_name = path.split('_', 1)
         md5_ = filepath.rsplit('/', 1)[1]
-        db_file = KnowledgeFile(knowledge_id=knowledge_id,
-                                file_name=file_name,
-                                status=1,
-                                md5=md5_,
+        # 是否包含重复文件
+        repeat = session.exec(select(KnowledgeFile
+                                     ).where(KnowledgeFile.md5 == md5_, KnowledgeFile.status == 2,
+                                             KnowledgeFile.knowledge_id == knowledge_id)).all()
+        status = 3 if repeat else 1
+        remark = 'file repeat' if repeat else ''
+        db_file = KnowledgeFile(knowledge_id=knowledge_id, file_name=file_name,
+                                status=status, md5=md5_, remark=remark,
                                 user_id=payload.get('user_id'))
+
         session.add(db_file)
         session.commit()
         session.refresh(db_file)
         files.append(db_file)
         file_paths.append(filepath)
         logger.info(f'fileName={file_name} col={collection_name}')
 
-    asyncio.create_task(
-        addEmbedding(collection_name=collection_name,
-                     model=knowledge.model,
-                     chunk_size=chunk_size,
-                     separator=separator,
-                     chunk_overlap=chunk_overlap,
-                     file_paths=file_paths,
-                     knowledge_files=files))
+    if not repeat:
+        asyncio.create_task(
+            addEmbedding(collection_name=collection_name,
+                         model=knowledge.model,
+                         chunk_size=chunk_size,
+                         separator=separator,
+                         chunk_overlap=chunk_overlap,
+                         file_paths=file_paths,
+                         knowledge_files=files))
 
     knowledge.update_time = db_file.create_time
     session.add(knowledge)
@@ -383,7 +389,6 @@ async def addEmbedding(collection_name, model: str, chunk_size: int, separator:
             setattr(db_file, 'remark', str(e)[:500])
             session.add(db_file)
             session.commit()
-            raise e
 
 
 def _read_chunk_text(input_file, file_name, size, chunk_overlap, separator):

src/backend/bisheng/api/v1/schemas.py

@@ -67,7 +67,7 @@ class ChatMessage(BaseModel):
     """Chat message schema."""
 
     is_bot: bool = False
-    message: Union[str, None, dict] = None
+    message: Union[str, None, dict] = ''
     type: str = 'human'
     category: str = 'processing'
     intermediate_steps: str = None
@@ -76,7 +76,7 @@ class ChatMessage(BaseModel):
     message_id: int = None
     source: bool = False
     sender: str = None
-    recevier: dict = None
+    receiver: dict = None
 
 
 class ChatResponse(ChatMessage):

src/backend/bisheng/api/v1/user.py

@@ -32,19 +32,21 @@ async def regist(*, session: Session = Depends(get_session), user: UserCreate):
     if not admin:
         db_user_role = UserRole(user_id=db_user.user_id, role_id=1)
         db_user.user_id = 1
-    else:
-        # 默认加入普通用户
-        db_user_role = UserRole(user_id=db_user.user_id, role_id=2)
+
     # check if user already exist
-    db_user = session.exec(select(User).where(User.user_name == user.user_name)).all()
-    if db_user:
+    name_user = session.exec(select(User).where(User.user_name == user.user_name)).all()
+    if name_user:
         raise HTTPException(status_code=500, detail='账号已存在')
     else:
         try:
             user.password = md5_hash(user.password)
             session.add(db_user)
             session.flush()
-            session.add(db_user_role)
+            session.refresh(db_user)
+            # 默认加入普通用户
+            if db_user != 1:
+                db_user_role = UserRole(user_id=db_user.user_id, role_id=2)
+                session.add(db_user_role)
             session.commit()
         except Exception as e:
             session.rollback()