From 542b578e1c06be722f3005912232c9561c0e9aaf Mon Sep 17 00:00:00 2001
From: chengtao-lv <897674362@qq.com>
Date: Sat, 23 Nov 2024 23:14:32 +0800
Subject: [PATCH] update lvlm

---
 .../methods/Awq/awq_w_only_vlm.yml            | 46 +++++++++++++++++++
 llmc/models/internvl2.py                      |  1 +
 llmc/models/llava.py                          |  1 +
 llmc/models/qwen2vl.py                        |  1 +
 4 files changed, 49 insertions(+)
 create mode 100644 configs/quantization/methods/Awq/awq_w_only_vlm.yml

diff --git a/configs/quantization/methods/Awq/awq_w_only_vlm.yml b/configs/quantization/methods/Awq/awq_w_only_vlm.yml
new file mode 100644
index 00000000..2cfdefdc
--- /dev/null
+++ b/configs/quantization/methods/Awq/awq_w_only_vlm.yml
@@ -0,0 +1,46 @@
+base:
+    seed: &seed 42
+model:
+    type: model_type
+    path: model path
+    tokenizer_mode: slow
+    torch_dtype: auto
+calib:
+    name: vlm_datastes
+    type: img_txt
+    download: False
+    path: calib data path
+    n_samples: 128
+    bs: -1
+    seq_len: 512
+    preproc: vlm_general
+    padding: True
+    seed: *seed
+eval:
+    eval_pos: [pretrain, fake_quant]
+    type: img_txt
+    name: MME
+    download: False
+    path: MME dataset path
+    bs: 16
+    inference_per_block: False
+quant:
+    method: Awq
+    quant_objects: [vision, language] # default is [language]
+    weight:
+        bit: 4
+        symmetric: False
+        granularity: per_group
+        group_size: 128
+    special:
+        trans: True
+        # The options for "trans_version" include "v1" and "v2".
+        # But their results don't differ significantly.
+        trans_version: v2
+        weight_clip: True
+        # For 2-bit quantization, setting "clip_sym: False" will yield better results.
+        clip_sym: True
+save:
+    save_trans: False
+    save_fake: False
+    save_path: /path/to/save/
diff --git a/llmc/models/internvl2.py b/llmc/models/internvl2.py
index b02a6621..94a54176 100644
--- a/llmc/models/internvl2.py
+++ b/llmc/models/internvl2.py
@@ -252,5 +252,6 @@ def get_vision_subsets_in_block(self, block):
                 'inspect': block.mlp.fc2,
                 'has_kwargs': False,
                 'is_mlp': True,
+                'do_trans': False
             },
         ]
diff --git a/llmc/models/llava.py b/llmc/models/llava.py
index b25ec0c9..4fde059f 100644
--- a/llmc/models/llava.py
+++ b/llmc/models/llava.py
@@ -124,5 +124,6 @@ def get_vision_subsets_in_block(self, block):
                 'inspect': block.mlp.fc2,
                 'has_kwargs': False,
                 'is_mlp': True,
+                'do_trans': False
             },
         ]
diff --git a/llmc/models/qwen2vl.py b/llmc/models/qwen2vl.py
index 1f5cd231..b629ca61 100644
--- a/llmc/models/qwen2vl.py
+++ b/llmc/models/qwen2vl.py
@@ -154,6 +154,7 @@ def get_vision_subsets_in_block(self, block):
                 'inspect': block.mlp.fc2,
                 'has_kwargs': False,
                 'is_mlp': True,
+                'do_trans': False
             },
         ]