diff --git a/.gitignore b/.gitignore index c6aa115..cd9b201 100644 --- a/.gitignore +++ b/.gitignore @@ -139,4 +139,7 @@ dmypy.json *.ini -**/multicache_serving.py \ No newline at end of file +**/multicache_serving.py +**/modelcache_serving.py + +**/model/ \ No newline at end of file diff --git a/model/clip_zh/__init__.py b/model/clip_zh/__init__.py new file mode 100644 index 0000000..b5dea3f --- /dev/null +++ b/model/clip_zh/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" + Alipay.com Inc. + Copyright (c) 2004-2023 All Rights Reserved. + ------------------------------------------------------ + File Name : __init__.py.py + Author : fuhui.phe + Create Time : 2024/5/7 14:05 + Description : description what the main function of this file + Change Activity: + version0 : 2024/5/7 14:05 by fuhui.phe init +""" diff --git a/modelcache/adapter/adapter_query.py b/modelcache/adapter/adapter_query.py index 934c644..c32b7fe 100644 --- a/modelcache/adapter/adapter_query.py +++ b/modelcache/adapter/adapter_query.py @@ -30,10 +30,12 @@ def adapt_query(cache_data_convert, *args, **kwargs): report_func=chat_cache.report.embedding, )(pre_embedding_data) + # print('embedding_data: {}'.format(embedding_data)) + if cache_enable: cache_data_list = time_cal( chat_cache.data_manager.search, - func_name="milvus_search", + func_name="vector_search", report_func=chat_cache.report.search, )( embedding_data, @@ -41,6 +43,7 @@ def adapt_query(cache_data_convert, *args, **kwargs): top_k=kwargs.pop("top_k", -1), model=model ) + print('cache_data_list: {}'.format(cache_data_list)) cache_answers = [] cache_questions = [] cache_ids = [] @@ -78,8 +81,8 @@ def adapt_query(cache_data_convert, *args, **kwargs): return for cache_data in cache_data_list: + print('cache_data: {}'.format(cache_data)) primary_id = cache_data[1] - start_time = time.time() ret = chat_cache.data_manager.get_scalar_data( cache_data, extra_param=context.get("get_scalar_data", None) ) diff --git a/modelcache/core.py b/modelcache/core.py index f5484e3..bd57029 100644 --- a/modelcache/core.py +++ b/modelcache/core.py @@ -4,7 +4,7 @@ from modelcache.processor.post import first from modelcache.similarity_evaluation import ExactMatchEvaluation from modelcache.similarity_evaluation import SimilarityEvaluation -from modelcache.embedding.string import to_embeddings as string_embedding +from modelcache.embedding.string_text import to_embeddings as string_embedding from modelcache.report import Report from modelcache.config import Config from modelcache.utils.cache_func import cache_all diff --git a/modelcache/embedding/clip.py b/modelcache/embedding/clip.py new file mode 100644 index 0000000..cbe45ff --- /dev/null +++ b/modelcache/embedding/clip.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +import os +import torch +from modelcache.embedding.base import BaseEmbedding +from modelscope.utils.constant import Tasks +from modelscope.pipelines import pipeline +from modelscope.preprocessors.image import load_image + + +# def mean_pooling(model_output, attention_mask): +# token_embeddings = model_output[0] # First element of model_output contains all token embeddings +# input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() +# return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) + + +class ClipAudio(BaseEmbedding): + def __init__(self, model: str = "sentence-transformers/all-MiniLM-L6-v2"): + # current_dir = os.path.dirname(os.path.abspath(__file__)) + # parent_dir = os.path.dirname(current_dir) + # model_dir = os.path.dirname(parent_dir) + # model = os.path.join(model_dir, 'model/text2vec-base-chinese/') + + self.clip_pipeline = pipeline(task=Tasks.multi_modal_embedding, + model='damo/multi-modal_clip-vit-base-patch16_zh', model_revision='v1.0.1') + + self.__dimension = 1024 + + def to_embeddings(self, data_dict, **_): + text_list = data_dict['text'] + image_data = data_dict['image'] + + img_data = None + txt_data = None + + if image_data: + input_img = load_image(image_data) + # 2D Tensor, [图片数, 特征维度] + img_embedding = self.clip_pipeline.forward({'img': input_img})['img_embedding'].tolist()[0] if input_img else [] + print('img_embedding: {}'.format(img_embedding)) + else: + raise ValueError('image_data is None, please check!') + + if text_list and len(text_list) > 0: + # 2D Tensor, [文本数, 特征维度] + text_embedding = self.clip_pipeline.forward({'text': text_list})['text_embedding'].tolist()[0] if text_list else [] + print('text_embedding: {}'.format(text_embedding)) + else: + raise ValueError('text_list is None, please check!') + + return {'image_embedding': img_embedding, 'text_embeddings': text_embedding} + + # return {'image_embedding': img_feats, 'text_embeddings': txt_feats} + # input_texts = ["杰尼龟", "妙蛙种子", "小火龙", "皮卡丘"] + # input_img = load_image( + # 'https://clip-cn-beijing.oss-cn-beijing.aliyuncs.com/pokemon.jpeg') + + # img_embedding = self.clip_pipeline.forward({'img': input_img})['img_embedding'] # 2D Tensor, [图片数, 特征维度] + # print('img_embedding: {}'.format(img_embedding)) + # text_embedding = self.clip_pipeline.forward({'text': input_texts})['text_embedding'] # 2D Tensor, [文本数, 特征维度] + + + # return embedding_array + + def post_proc(self, token_embeddings, inputs): + attention_mask = inputs["attention_mask"] + input_mask_expanded = ( + attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() + ) + sentence_embs = torch.sum( + token_embeddings * input_mask_expanded, 1 + ) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) + return sentence_embs + + @property + def dimension(self): + """Embedding dimension. + + :return: embedding dimension + """ + return self.__dimension + + +# if __name__ == '__main__': +# clip_vec = ClipAudio() +# text_list = ['hello', '你好'] +# text = ['###'.join(text_list)] +# image = 'https://clip-cn-beijing.oss-cn-beijing.aliyuncs.com/pokemon.jpeg' +# data_dict = {'text': text, 'image': image} +# resp = clip_vec.to_embeddings(data_dict) +# print('resp: {}'.format(resp)) diff --git a/modelcache/embedding/clip_demo.py b/modelcache/embedding/clip_demo.py new file mode 100644 index 0000000..f6e157b --- /dev/null +++ b/modelcache/embedding/clip_demo.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +""" + Alipay.com Inc. + Copyright (c) 2004-2023 All Rights Reserved. + ------------------------------------------------------ + File Name : clip_demo.py + Author : fuhui.phe + Create Time : 2024/5/7 11:58 + Description : description what the main function of this file + Change Activity: + version0 : 2024/5/7 11:58 by fuhui.phe init +""" +import torch +from modelscope.utils.constant import Tasks +from modelscope.pipelines import pipeline +from modelscope.preprocessors.image import load_image + + +pipeline = pipeline(task=Tasks.multi_modal_embedding, + model='damo/multi-modal_clip-vit-base-patch16_zh', model_revision='v1.0.1') + +# pipeline = pipeline(task=Tasks.multi_modal_embedding, +# model='/Users/penghongen/PycharmProjects/CodeFuse-ModelCache/model/clip_zh', model_revision='v1.0.1') + +# pipeline = pipeline(task=Tasks.multi_modal_embedding, model='/Users/penghongen/PycharmProjects/CodeFuse-ModelCache/model/clip_zh') + + +input_img = load_image('https://clip-cn-beijing.oss-cn-beijing.aliyuncs.com/pokemon.jpeg') # 支持皮卡丘示例图片路径/本地图片 返回PIL.Image + + +input_texts = ["杰尼龟", "妙蛙种子", "小火龙", "皮卡丘"] + +# 支持一张图片(PIL.Image)或多张图片(List[PIL.Image])输入，输出归一化特征向量 +img_embedding = pipeline.forward({'img': input_img})['img_embedding'] # 2D Tensor, [图片数, 特征维度] +print('img_embedding: {}'.format(img_embedding)) + +# 支持一条文本(str)或多条文本(List[str])输入，输出归一化特征向量 +text_embedding = pipeline.forward({'text': input_texts})['text_embedding'] # 2D Tensor, [文本数, 特征维度] + +# 计算图文相似度 +with torch.no_grad(): + # 计算内积得到logit，考虑模型temperature + logits_per_image = (img_embedding / pipeline.model.temperature) @ text_embedding.t() + # 根据logit计算概率分布 + probs = logits_per_image.softmax(dim=-1).cpu().numpy() + +print("图文匹配概率:", probs) + + diff --git a/modelcache/embedding/string.py b/modelcache/embedding/string_text.py similarity index 100% rename from modelcache/embedding/string.py rename to modelcache/embedding/string_text.py diff --git a/modelcache/embedding/timm.py b/modelcache/embedding/timm_embedding.py similarity index 100% rename from modelcache/embedding/timm.py rename to modelcache/embedding/timm_embedding.py diff --git a/requirements.txt b/requirements.txt index 3bf85e6..78d682e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ pymilvus==2.3.1 PyMySQL==1.1.0 Requests==2.31.0 torch==2.1.0 -transformers==4.34.1 +transformers==4.38.2 faiss-cpu==1.7.4 redis==5.0.1 - +modelscope==1.14.0