4bit/llava-v1.5-13b-3GB

aifrenz · Oct 14, 2023 · b03d4b1 · b03d4b1
1 parent 59c9646
commit b03d4b1
Showing 1 changed file with 16 additions and 1 deletion.
diff --git a/LLaVA_13b_4bit_colab.ipynb b/LLaVA_13b_4bit_colab.ipynb
@@ -52,7 +52,22 @@
         "    '--model-path', '4bit/llava-v1.5-13b-3GB',\n",
         "    '--load-4bit'\n",
         "]\n",
-        "threading.Thread(target=lambda: subprocess.run(command, check=True, shell=False), daemon=True).start()"
+        "threading.Thread(target=lambda: subprocess.run(command, check=True, shell=False), daemon=True).start()\n",
+        "\n",
+        "# from transformers import AutoTokenizer, BitsAndBytesConfig\n",
+        "# from llava.model import LlavaLlamaForCausalLM\n",
+        "# import torch\n",
+        "# model_path = \"4bit/llava-v1.5-13b-3GB\"\n",
+        "# kwargs = {\"device_map\": \"auto\"}\n",
+        "# kwargs['load_in_4bit'] = True\n",
+        "# kwargs['quantization_config'] = BitsAndBytesConfig(\n",
+        "#     load_in_4bit=True,\n",
+        "#     bnb_4bit_compute_dtype=torch.float16,\n",
+        "#     bnb_4bit_use_double_quant=True,\n",
+        "#     bnb_4bit_quant_type='nf4'\n",
+        "# )\n",
+        "# model = LlavaLlamaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)\n",
+        "# tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)"
       ]
     },
     {