From d3da17eae1bc8ff1c9926b8cc265f28c9539a30b Mon Sep 17 00:00:00 2001 From: Ekaterina Aidova Date: Mon, 20 Jan 2025 11:38:16 +0400 Subject: [PATCH] prioretize config model type under path-based task determination (#1587) CVS-160764 --- .../llm_bench/llm_bench_utils/config_class.py | 2 +- .../llm_bench/llm_bench_utils/model_utils.py | 27 +++++++++---------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/tools/llm_bench/llm_bench_utils/config_class.py b/tools/llm_bench/llm_bench_utils/config_class.py index 76b56fb4c5..e7ebad9700 100644 --- a/tools/llm_bench/llm_bench_utils/config_class.py +++ b/tools/llm_bench/llm_bench_utils/config_class.py @@ -53,7 +53,7 @@ USE_CASES = { 'image_gen': ['stable-diffusion-', 'ssd-', 'tiny-sd', 'small-sd', 'lcm-', 'sdxl', 'dreamlike', "flux"], - "vlm": ["llava", "llava-next", "qwen2-vl", "llava-qwen2", "internvl-chat", "minicpmv", "phi3-v"], + "vlm": ["llava", "llava-next", "qwen2-vl", "llava-qwen2", "internvl-chat", "minicpmv", "phi3-v", "minicpm-v"], 'speech2text': ['whisper'], 'image_cls': ['vit'], 'code_gen': ['replit', 'codegen2', 'codegen', 'codet5', "stable-code"], diff --git a/tools/llm_bench/llm_bench_utils/model_utils.py b/tools/llm_bench/llm_bench_utils/model_utils.py index ae61cf96dd..324a67bc2a 100644 --- a/tools/llm_bench/llm_bench_utils/model_utils.py +++ b/tools/llm_bench/llm_bench_utils/model_utils.py @@ -186,22 +186,10 @@ def analyze_args(args): def get_use_case(model_name_or_path): - # 1. try to get use_case from model name - path = os.path.normpath(model_name_or_path) - model_names = path.split(os.sep) - for model_name in reversed(model_names): - for case, model_ids in USE_CASES.items(): - for model_id in model_ids: - if model_name.lower().startswith(model_id): - log.info(f'==SUCCESS FOUND==: use_case: {case}, model_type: {model_name}') - return case, model_name - - # 2. try to get use_case from model config - try: - config_file = Path(model_name_or_path) / "config.json" + config_file = Path(model_name_or_path) / "config.json" + config = None + if config_file.exists(): config = json.loads(config_file.read_text()) - except Exception: - config = None if (Path(model_name_or_path) / "model_index.json").exists(): diffusers_config = json.loads((Path(model_name_or_path) / "model_index.json").read_text()) pipe_type = diffusers_config.get("_class_name") @@ -214,6 +202,15 @@ def get_use_case(model_name_or_path): if config.get("model_type").lower().replace('_', '-').startswith(model_id): log.info(f'==SUCCESS FOUND==: use_case: {case}, model_type: {model_id}') return case, model_ids[idx] + # try to get use_case from model name + path = os.path.normpath(model_name_or_path) + model_names = path.split(os.sep) + for model_name in reversed(model_names): + for case, model_ids in USE_CASES.items(): + for model_id in model_ids: + if model_name.lower().startswith(model_id): + log.info(f'==SUCCESS FOUND==: use_case: {case}, model_type: {model_name}') + return case, model_name raise RuntimeError('==Failure FOUND==: no use_case found')