There is some problem with the LLaVA_13b_8bit_colab.ipynb

[xinsir6]

I use the ipynb scripts and load the model but always got Error:

The complete information is :
NotImplementedError Traceback (most recent call last)
Cell In[2], line 10
7 model_path = "/dfs/comicai/qi.xin/models/llava-v1.5-13b-5GB"
8 # model_path = "4bit/llava-v1.5-13b-4GB-8bit"
9 # model = LlavaLlamaForCausalLM.from_pretrained(model_path)
---> 10 model = LlavaLlamaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, load_in_8bit=True, device_map="auto")
12 kwargs = {"device_map": "auto"}
13 kwargs['load_in_4bit'] = True

File ~/anaconda3/envs/llava/lib/python3.10/site-packages/transformers/modeling_utils.py:2903, in PreTrainedModel.from_pretrained(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, *model_args, **kwargs)
2893 if dtype_orig is not None:
2894 torch.set_default_dtype(dtype_orig)
2896 (
2897 model,
2898 missing_keys,
2899 unexpected_keys,
2900 mismatched_keys,
2901 offload_index,
2902 error_msgs,
-> 2903 ) = cls._load_pretrained_model(
2904 model,
2905 state_dict,
2906 loaded_state_dict_keys, # XXX: rename?
2907 resolved_archive_file,
2908 pretrained_model_name_or_path,
2909 ignore_mismatched_sizes=ignore_mismatched_sizes,
2910 sharded_metadata=sharded_metadata,
2911 _fast_init=_fast_init,
2912 low_cpu_mem_usage=low_cpu_mem_usage,
2913 device_map=device_map,
2914 offload_folder=offload_folder,
2915 offload_state_dict=offload_state_dict,
2916 dtype=torch_dtype,
2917 is_quantized=(load_in_8bit or load_in_4bit),
2918 keep_in_fp32_modules=keep_in_fp32_modules,
2919 )
2921 model.is_loaded_in_4bit = load_in_4bit
2922 model.is_loaded_in_8bit = load_in_8bit

File ~/anaconda3/envs/llava/lib/python3.10/site-packages/transformers/modeling_utils.py:3260, in PreTrainedModel._load_pretrained_model(cls, model, state_dict, loaded_keys, resolved_archive_file, pretrained_model_name_or_path, ignore_mismatched_sizes, sharded_metadata, _fast_init, low_cpu_mem_usage, device_map, offload_folder, offload_state_dict, dtype, is_quantized, keep_in_fp32_modules)
3250 mismatched_keys += _find_mismatched_keys(
3251 state_dict,
3252 model_state_dict,
(...)
3256 ignore_mismatched_sizes,
3257 )
3259 if low_cpu_mem_usage:
-> 3260 new_error_msgs, offload_index, state_dict_index = _load_state_dict_into_meta_model(
3261 model_to_load,
3262 state_dict,
3263 loaded_keys,
3264 start_prefix,
3265 expected_keys,
3266 device_map=device_map,
3267 offload_folder=offload_folder,
3268 offload_index=offload_index,
3269 state_dict_folder=state_dict_folder,
3270 state_dict_index=state_dict_index,
3271 dtype=dtype,
3272 is_quantized=is_quantized,
3273 is_safetensors=is_safetensors,
3274 keep_in_fp32_modules=keep_in_fp32_modules,
3275 )
3276 error_msgs += new_error_msgs
3277 else:

File ~/anaconda3/envs/llava/lib/python3.10/site-packages/transformers/modeling_utils.py:725, in _load_state_dict_into_meta_model(model, state_dict, loaded_state_dict_keys, start_prefix, expected_keys, device_map, offload_folder, offload_index, state_dict_folder, state_dict_index, dtype, is_quantized, is_safetensors, keep_in_fp32_modules)
722 fp16_statistics = None
724 if "SCB" not in param_name:
--> 725 set_module_quantized_tensor_to_device(
726 model, param_name, param_device, value=param, fp16_statistics=fp16_statistics
727 )
729 return error_msgs, offload_index, state_dict_index

File ~/anaconda3/envs/llava/lib/python3.10/site-packages/transformers/utils/bitsandbytes.py:77, in set_module_quantized_tensor_to_device(module, tensor_name, device, value, fp16_statistics)
75 new_value = old_value.to(device)
76 elif isinstance(value, torch.Tensor):
---> 77 new_value = value.to("cpu")
78 if value.dtype == torch.int8:
79 is_8bit_serializable = version.parse(importlib.metadata.version("bitsandbytes")) > version.parse(
80 "0.37.2"
81 )

NotImplementedError: Cannot copy out of meta tensor; no data!

So it there some problem the model in huggingface?

[steinhaug]

I'm quite sure there is a problem with the model from huggingface, I get the same problems trying to load it.

llava-v1.5-7b and llava-v1.5-13b-3GB works fine.