diff --git a/llmc/compression/quantization/quarot.py b/llmc/compression/quantization/quarot.py index fc0073a4..386bc206 100644 --- a/llmc/compression/quantization/quarot.py +++ b/llmc/compression/quantization/quarot.py @@ -27,7 +27,7 @@ def preprocess(self): self.model.get_head_layers()[0].weight, self.model.get_embed_layers()[0].weight, ): - logger.info('Tie weight! Skip rotating head layer!') + logger.info('Tie weight! Copy embed_layer for head_layer!') del self.model.get_head_layers()[0].weight w = self.model.get_embed_layers()[0].weight.clone() self.model.get_head_layers()[0].weight = nn.Parameter(w)