diff --git a/appbuilder/core/_exception.py b/appbuilder/core/_exception.py index ca90a19e..2a38cd9d 100644 --- a/appbuilder/core/_exception.py +++ b/appbuilder/core/_exception.py @@ -74,15 +74,16 @@ class TypeNotSupportedException(BaseRPCException): class AppBuilderServerException(BaseRPCException): r"""AppBuilderServerException represent backend server failed response. """ + description: str = "Interal Server Error" + code: int = 500 def __init__(self, request_id="", code="", message="", service_err_code="", service_err_message=""): - r"""__init__ a AppBuilderServerException instance. - :param request_id: str, request unique id. - :param code: str, backend . - :rtype: - """ - super().__init__("request_id={}, code={}, message={}, service_err_code={}, service_err_message={} ".format( - request_id, code, message, service_err_code, service_err_message)) + self.description = "request_id={}, code={}, message={}, service_err_code={}, service_err_message={} ".format( + request_id, code, message, service_err_code, service_err_message) + self.code = code if code else self.code + + def __str__(self): + return self.description class InvalidRequestArgumentError(BaseRPCException): diff --git a/appbuilder/core/agent.py b/appbuilder/core/agent.py index d0efb11d..c6828e61 100644 --- a/appbuilder/core/agent.py +++ b/appbuilder/core/agent.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import sys +import copy import os import logging import uuid @@ -246,7 +247,11 @@ def warp(): app_builder_token = request.headers["X-Appbuilder-Token"] self.component.set_secret_key_and_gateway(secret_key=app_builder_token) except appbuilder.core._exception.BaseRPCException as e: + logging.error(f"failed to verify. err={e}", exc_info=True) raise BadRequest("X-Appbuilder-Token invalid") + except Exception as e: + logging.error(f"failed to verify. err={e}", exc_info=True) + raise e else: pass @@ -276,34 +281,29 @@ def warp(): def gen_sse_resp(stream_message): with app.app_context(): try: - iterator = iter(stream_message.content) - prev_result = { - "content": next(iterator), - "extra": stream_message.extra if hasattr(stream_message, "extra") else {}, - "token_usage": stream_message.token_usage if hasattr(stream_message, "token_usage") else {}, - } - for sub_content in iterator: + content_iterator = iter(stream_message.content) + prev_content = next(content_iterator) + prev_result = copy.deepcopy(stream_message) + prev_result.content = prev_content + for sub_content in content_iterator: logging.info(f"[request_id={request_id}, session_id={session_id}] streaming_result={prev_result}") yield "data: " + json.dumps({ "code": 0, "message": "", "result": { "session_id": session_id, "is_completion": False, - "answer_message": prev_result + "answer_message": json.loads(prev_result.json(exclude_none=True)) } }, ensure_ascii=False) + "\n\n" - prev_result = { - "content": sub_content, - "extra": stream_message.extra if hasattr(stream_message, "extra") else {}, - "token_usage": stream_message.token_usage if hasattr(stream_message, "token_usage") else {}, - } + prev_result = copy.deepcopy(stream_message) + prev_result.content = sub_content logging.info(f"[request_id={request_id}, session_id={session_id}] streaming_result={prev_result}") yield "data: " + json.dumps({ "code": 0, "message": "", "result": { "session_id": session_id, "is_completion": True, - "answer_message": prev_result + "answer_message": json.loads(prev_result.json(exclude_none=True)) } }, ensure_ascii=False) + "\n\n" self.user_session._post_append() @@ -317,7 +317,7 @@ def gen_sse_resp(stream_message): {'Content-Type': 'text/event-stream; charset=utf-8'}, ) else: - blocking_result = json.loads(answer.json(exclude_none=True)) + blocking_result = json.loads(copy.deepcopy(answer).json(exclude_none=True)) logging.info(f"[request_id={request_id}, session_id={session_id}] blocking_result={blocking_result}") self.user_session._post_append() return { diff --git a/appbuilder/core/component.py b/appbuilder/core/component.py index 7aba7783..31256c92 100644 --- a/appbuilder/core/component.py +++ b/appbuilder/core/component.py @@ -19,7 +19,7 @@ from pydantic import BaseModel from typing import Dict, List, Optional, Any - +from appbuilder.core.utils import ttl_lru_cache from appbuilder.core._client import HTTPClient from appbuilder.core.message import Message @@ -81,7 +81,8 @@ def __init__( self.lazy_certification = lazy_certification if not self.lazy_certification: self.set_secret_key_and_gateway(self.secret_key, self.gateway) - + + @ttl_lru_cache(seconds_to_live=1 * 60 * 60) # 1h def set_secret_key_and_gateway(self, secret_key: Optional[str] = None, gateway: str = ""): self.secret_key = secret_key self.gateway = gateway diff --git a/appbuilder/core/components/excel2figure/component.py b/appbuilder/core/components/excel2figure/component.py index 5b72c8e3..e74befde 100644 --- a/appbuilder/core/components/excel2figure/component.py +++ b/appbuilder/core/components/excel2figure/component.py @@ -25,7 +25,7 @@ from appbuilder.core._exception import AppBuilderServerException, ModelNotSupportedException from appbuilder.core.component import Component, ComponentArguments from appbuilder.core.message import Message -from appbuilder.core.utils import ModelInfo +from appbuilder.core.utils import ModelInfo, ttl_lru_cache class Excel2FigureArgs(ComponentArguments): @@ -74,11 +74,13 @@ def __init__( self._check_model_and_get_model_url(self.model, self.model_type) self.server_sub_path = "/v1/ai_engine/copilot_engine/v1/api/agent/excel2figure" + @ttl_lru_cache(seconds_to_live=1 * 60 * 60) # 1h def set_secret_key_and_gateway(self, secret_key: Optional[str] = None, gateway: str = ""): super(Excel2Figure, self).set_secret_key_and_gateway( secret_key=secret_key, gateway=gateway) self.__class__.model_info = ModelInfo(client=self.http_client) + @ttl_lru_cache(seconds_to_live=1 * 60 * 60) # 1h def _check_model_and_get_model_url(self, model, model_type): if model and model in self.excluded_models: raise ModelNotSupportedException(f"Model {model} not supported") @@ -221,8 +223,8 @@ def tool_eval( 'text': [result_msg.content], } except Exception as e: - result = f'绘制图表时发生错误：{e}' - logging.error(result, exc_info=True) + raise RuntimeError(f'绘制图表时发生错误：{e}') + if streaming: yield result else: diff --git a/appbuilder/core/components/llms/base.py b/appbuilder/core/components/llms/base.py index 29d789e7..11b4c124 100644 --- a/appbuilder/core/components/llms/base.py +++ b/appbuilder/core/components/llms/base.py @@ -17,6 +17,8 @@ from enum import Enum import logging import requests +import copy +import collections.abc from appbuilder.core.constants import GATEWAY_URL, GATEWAY_INNER_URL from pydantic import BaseModel, Field, ValidationError, HttpUrl, validator from pydantic.types import confloat @@ -27,7 +29,7 @@ from typing import Dict, List, Optional, Any from appbuilder.core.component import ComponentArguments -from appbuilder.core.utils import ModelInfo +from appbuilder.core.utils import ModelInfo, ttl_lru_cache from appbuilder.utils.sse_util import SSEClient from appbuilder.core._exception import AppBuilderServerException, ModelNotSupportedException @@ -40,6 +42,16 @@ class LLMMessage(Message): def __str__(self): return f"Message(name={self.name}, content={self.content}, " \ f"mtype={self.mtype}, extra={self.extra}, token_usage={self.token_usage})" + + def __deepcopy__(self, memo): + new_instance = self.__class__() + memo[id(self)] = new_instance + for k, v in self.__dict__.items(): + if k == "content" and isinstance(v, collections.abc.Iterator): + pass + else: + setattr(new_instance, k, copy.deepcopy(v, memo)) + return new_instance class CompletionRequest(object): @@ -270,11 +282,13 @@ def __init__( if not lazy_certification: self._check_model_and_get_model_url(self.model_name, self.model_type) + @ttl_lru_cache(seconds_to_live=1 * 60 * 60) # 1h def set_secret_key_and_gateway(self, secret_key: Optional[str] = None, gateway: str = ""): super(CompletionBaseComponent, self).set_secret_key_and_gateway( secret_key=secret_key, gateway=gateway) self.__class__.model_info = ModelInfo(client=self.http_client) + @ttl_lru_cache(seconds_to_live=1 * 60 * 60) # 1h def _check_model_and_get_model_url(self, model, model_type): if model and model in self.excluded_models: raise ModelNotSupportedException(f"Model {model} not supported") diff --git a/appbuilder/core/message.py b/appbuilder/core/message.py index ed9d073d..022bf70e 100644 --- a/appbuilder/core/message.py +++ b/appbuilder/core/message.py @@ -16,14 +16,14 @@ import uuid -from pydantic import BaseModel +from pydantic import BaseModel, Extra from typing import Optional, TypeVar, Generic _T = TypeVar("_T") -class Message(BaseModel, Generic[_T]): +class Message(BaseModel, Generic[_T], extra=Extra.allow): content: Optional[_T] = {} name: Optional[str] = "msg" mtype: Optional[str] = "dict" diff --git a/appbuilder/core/utils.py b/appbuilder/core/utils.py index fa6af8f3..47d7445b 100644 --- a/appbuilder/core/utils.py +++ b/appbuilder/core/utils.py @@ -11,12 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import time import itertools from typing import List from urllib.parse import urlparse from appbuilder.core._client import HTTPClient from appbuilder.core._exception import TypeNotSupportedException, ModelNotSupportedException from appbuilder.utils.model_util import GetModelListRequest, Models, RemoteModelCollector +from functools import lru_cache def utils_get_user_agent(): @@ -75,6 +77,20 @@ def is_url(string): return all([result.scheme, result.netloc]) +def ttl_lru_cache(seconds_to_live: int, maxsize: int = 128): + """ + Time aware lru caching + """ + def wrapper(func): + @lru_cache(maxsize) + def inner(__ttl, *args, **kwargs): + # Note that __ttl is not passed down to func, + # as it's only used to trigger cache miss after some time + return func(*args, **kwargs) + return lambda *args, **kwargs: inner(time.time() // seconds_to_live, *args, **kwargs) + return wrapper + + class ModelInfo: """ 模型信息类 """