多卡微调时提示显存不足/ CUDA Out of Memory Error

[Chenhong-Zhang]

我使用了8*RTX 3080 进行训练，每张卡有10GB显存。
调用以下微调命令：
OMP_NUM_THREADS=1 torchrun --standalone --nnodes=1 --nproc_per_node=8 finetune_hf.py /home/usr/IRE/data /home/usr/IRE/chatglm3-6b configs/lora.yaml configs/ds_zero_2.json
报错提示显存不足：

OutOfMemoryError: CUDA out of memory. Tried to allocate 108.00 MiB. GPU 0 has a total capacity of 9.77 GiB of which 19.31 MiB is free. Including non-PyTorch memory, this process has 9.75 GiB memory in use. Of the allocated memory 9.54 GiB is allocated
by PyTorch, and 1.04 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management

观察Traceback发现是在模型to device过程出现了错误：

╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ /home/slm/ChatGLM3/finetune_demo/finetune_hf.py:516 in main │
│ │
│ 513 │ ] │
│ 514 │ model.gradient_checkpointing_enable() │
│ 515 │ model.enable_input_require_grads() │
│ ❱ 516 │ trainer = Seq2SeqTrainer( │
│ 517 │ │ model=model, │
│ 518 │ │ args=ft_config.training_args, │
│ 519 │ │ data_collator=DataCollatorForSeq2Seq( │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/transformers/trainer_ │
│ seq2seq.py:57 in init │
│ │
│ 54 │ │ optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR] = (N │
│ 55 │ │ preprocess_logits_for_metrics: Optional[Callable[[torch.Tensor, torch.Tensor], t │
│ 56 │ ): │
│ ❱ 57 │ │ super().init( │
│ 58 │ │ │ model=model, │
│ 59 │ │ │ args=args, │
│ 60 │ │ │ data_collator=data_collator, │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/transformers/trainer. │
│ py:495 in init │
│ │
│ 492 │ │ │ self.place_model_on_device │
│ 493 │ │ │ and not getattr(model, "quantization_method", None) == QuantizationMethod.BI │
│ 494 │ │ ): │
│ ❱ 495 │ │ │ self._move_model_to_device(model, args.device) │
│ 496 │ │ │
│ 497 │ │ # Force n_gpu to 1 to avoid DataParallel as MP will manage the GPUs │
│ 498 │ │ if self.is_model_parallel: │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/transformers/trainer. │
│ py:736 in _move_model_to_device │
│ │
│ 733 │ │ self.callback_handler.remove_callback(callback) │
│ 734 │ │
│ 735 │ def _move_model_to_device(self, model, device): │
│ ❱ 736 │ │ model = model.to(device) │
│ 737 │ │ # Moving a model to an XLA device disconnects the tied weights, so we have to re │
│ 738 │ │ if self.args.parallel_mode == ParallelMode.TPU and hasattr(model, "tie_weights") │
│ 739 │ │ │ model.tie_weights() │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/torch/nn/modules/modu │
│ le.py:1152 in to │
│ │
│ 1149 │ │ │ │ │ │ │ non_blocking, memory_format=convert_to_format) │
│ 1150 │ │ │ return t.to(device, dtype if t.is_floating_point() or t.is_complex() else No │
│ 1151 │ │ │
│ ❱ 1152 │ │ return self._apply(convert) │
│ 1153 │ │
│ 1154 │ def register_full_backward_pre_hook( │
│ 1155 │ │ self, │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/torch/nn/modules/modu │
│ le.py:802 in _apply │
│ │
│ 799 │ def _apply(self, fn, recurse=True): │
│ 800 │ │ if recurse: │
│ 801 │ │ │ for module in self.children(): │
│ ❱ 802 │ │ │ │ module._apply(fn) │
│ 803 │ │ │
│ 804 │ │ def compute_should_use_set_data(tensor, tensor_applied): │
│ 805 │ │ │ if torch._has_compatible_shallow_copy_type(tensor, tensor_applied): │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/torch/nn/modules/modu │
│ le.py:802 in _apply │
│ │
│ 799 │ def _apply(self, fn, recurse=True): │
│ 800 │ │ if recurse: │
│ 801 │ │ │ for module in self.children(): │
│ ❱ 802 │ │ │ │ module._apply(fn) │
│ 803 │ │ │
│ 804 │ │ def compute_should_use_set_data(tensor, tensor_applied): │
│ 805 │ │ │ if torch._has_compatible_shallow_copy_type(tensor, tensor_applied): │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/torch/nn/modules/modu │
│ le.py:802 in _apply │
│ │
│ 799 │ def _apply(self, fn, recurse=True): │
│ 800 │ │ if recurse: │
│ 801 │ │ │ for module in self.children(): │
│ ❱ 802 │ │ │ │ module._apply(fn) │
│ 803 │ │ │
│ 804 │ │ def compute_should_use_set_data(tensor, tensor_applied): │
│ 805 │ │ │ if torch._has_compatible_shallow_copy_type(tensor, tensor_applied): │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/torch/nn/modules/modu │
│ le.py:802 in _apply │
│ │
│ 799 │ def _apply(self, fn, recurse=True): │
│ 800 │ │ if recurse: │
│ 801 │ │ │ for module in self.children(): │
│ ❱ 802 │ │ │ │ module._apply(fn) │
│ 803 │ │ │
│ 804 │ │ def compute_should_use_set_data(tensor, tensor_applied): │
│ 805 │ │ │ if torch._has_compatible_shallow_copy_type(tensor, tensor_applied): │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/torch/nn/modules/modu │
│ le.py:802 in _apply │
│ │
│ 799 │ def _apply(self, fn, recurse=True): │
│ 800 │ │ if recurse: │
│ 801 │ │ │ for module in self.children(): │
│ ❱ 802 │ │ │ │ module._apply(fn) │
│ 803 │ │ │
│ 804 │ │ def compute_should_use_set_data(tensor, tensor_applied): │
│ 805 │ │ │ if torch._has_compatible_shallow_copy_type(tensor, tensor_applied): │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/torch/nn/modules/modu │
│ le.py:802 in _apply │
│ │
│ 799 │ def _apply(self, fn, recurse=True): │
│ 800 │ │ if recurse: │
│ 801 │ │ │ for module in self.children(): │
│ ❱ 802 │ │ │ │ module._apply(fn) │
│ 803 │ │ │
│ 804 │ │ def compute_should_use_set_data(tensor, tensor_applied): │
│ 805 │ │ │ if torch._has_compatible_shallow_copy_type(tensor, tensor_applied): │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/torch/nn/modules/modu │
│ le.py:802 in _apply │
│ │
│ 799 │ def _apply(self, fn, recurse=True): │
│ 800 │ │ if recurse: │
│ 801 │ │ │ for module in self.children(): │
│ ❱ 802 │ │ │ │ module._apply(fn) │
│ 803 │ │ │
│ 804 │ │ def compute_should_use_set_data(tensor, tensor_applied): │
│ 805 │ │ │ if torch._has_compatible_shallow_copy_type(tensor, tensor_applied): │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/torch/nn/modules/modu │
│ le.py:802 in _apply │
│ │
│ 799 │ def _apply(self, fn, recurse=True): │
│ 800 │ │ if recurse: │
│ 801 │ │ │ for module in self.children(): │
│ ❱ 802 │ │ │ │ module._apply(fn) │
│ 803 │ │ │
│ 804 │ │ def compute_should_use_set_data(tensor, tensor_applied): │
│ 805 │ │ │ if torch._has_compatible_shallow_copy_type(tensor, tensor_applied): │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/torch/nn/modules/modu │
│ le.py:825 in _apply │
│ │
│ 822 │ │ │ # track autograd history of param_applied, so we have to use │
│ 823 │ │ │ # with torch.no_grad(): │
│ 824 │ │ │ with torch.no_grad(): │
│ ❱ 825 │ │ │ │ param_applied = fn(param) │
│ 826 │ │ │ should_use_set_data = compute_should_use_set_data(param, param_applied) │
│ 827 │ │ │ if should_use_set_data: │
│ 828 │ │ │ │ param.data = param_applied │
│ │
│ /home/slm/.pyenv/versions/3.11.0/envs/LLM_311/lib/python3.11/site-packages/torch/nn/modules/modu │
│ le.py:1150 in convert │
│ │
│ 1147 │ │ │ if convert_to_format is not None and t.dim() in (4, 5): │
│ 1148 │ │ │ │ return t.to(device, dtype if t.is_floating_point() or t.is_complex() els │
│ 1149 │ │ │ │ │ │ │ non_blocking, memory_format=convert_to_format) │
│ ❱ 1150 │ │ │ return t.to(device, dtype if t.is_floating_point() or t.is_complex() else No │
│ 1151 │ │ │
│ 1152 │ │ return self._apply(convert) │
│ 1153 │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
但是根据官方文档，LORA 微调1张显卡占用 14082MiB 显存，那8张3080显卡应该是够的啊？
请问各位大佬，是否我哪里参数配置存在问题导致微调失败？

[zhaohai7206]

跟你同样的问题，也是lora训练的时候报cuda oom，但是提示信息里面显示需要171 T 的内存，不知道是哪里配置问题，用swift可以进行微调，但是还没找到合并成新模型，并且能够进行推理的方案

[Chenhong-Zhang]

改用Ptuning 微调，同时在Main函数里增加模型量化，可以进行微调
model = model.quantize(4)