test

aifrenz · Oct 13, 2023 · 6d69b6e · 6d69b6e
1 parent 8d9bf84
commit 6d69b6e
Showing 1 changed file with 24 additions and 9 deletions.
diff --git a/LLaVA_13b_8bit_colab.ipynb b/LLaVA_13b_8bit_colab.ipynb
@@ -79,15 +79,30 @@
       "metadata": {},
       "outputs": [],
       "source": [
-        "from llava.model.builder import load_pretrained_model\n",
-        "model_path = \"4bit/llava-v1.5-13b-5GB\"\n",
-        "tokenizer, model, image_processor, context_len = load_pretrained_model(\n",
-        "    model_path=model_path,\n",
-        "    model_base=None,\n",
-        "    model_name=model_path.split(\"/\")[-1],\n",
-        "    load_8bit=True,\n",
-        "    load_4bit=False\n",
-        ")\n",
+        "# from llava.model.builder import load_pretrained_model\n",
+        "# model_path = \"4bit/llava-v1.5-13b-5GB\"\n",
+        "# tokenizer, model, image_processor, context_len = load_pretrained_model(\n",
+        "#     model_path=model_path,\n",
+        "#     model_base=None,\n",
+        "#     model_name=model_path.split(\"/\")[-1],\n",
+        "#     load_8bit=True,\n",
+        "#     load_4bit=False\n",
+        "# )\n",
+        "\n",
+        "from transformers import AutoTokenizer\n",
+        "from llava.model import LlavaLlamaForCausalLM\n",
+        "import torch\n",
+        "\n",
+        "model_path = \"4bit/llava-v1.5-13b-4GB-8bit\"\n",
+        "tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)\n",
+        "model = LlavaLlamaForCausalLM.from_pretrained(model_path)\n",
+        "# model = LlavaLlamaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, load_in_8bit=True, device_map=\"auto\")\n",
+        "\n",
+        "vision_tower = model.get_vision_tower()\n",
+        "if not vision_tower.is_loaded:\n",
+        "    vision_tower.load_model()\n",
+        "vision_tower.to(device='cpu', dtype=torch.float16)\n",
+        "image_processor = vision_tower.image_processor\n",
         "\n",
         "import requests\n",
         "from PIL import Image\n",