From 84ddfb286808cd16d0e96e87168542b13f4217ca Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 26 May 2025 22:39:20 +0300 Subject: [PATCH] SDNQ fix lora apply --- modules/lora/lora_apply.py | 4 ++-- modules/model_quant_sdnq.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index 370beabea..67af8974a 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -80,7 +80,7 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. try: t0 = time.time() if hasattr(self, "sdnq_decompressor"): - weight = self.sdnq_decompressor.to(devices.device)(self.weight.to(devices.device), skip_int8_matmul=True) + weight = self.sdnq_decompressor.to(devices.device)(self.weight.to(devices.device), skip_int8_matmul=self.sdnq_decompressor.use_int8_matmul) else: weight = self.weight.to(devices.device) # must perform calc on gpu due to performance updown, ex_bias = module.calc_updown(weight) @@ -143,7 +143,7 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G from modules.model_quant_sdnq import sdnq_quantize_layer num_bits = self.sdnq_decompressor.num_bits is_asym_mode = self.sdnq_decompressor.is_asym_mode - dequant_weight = self.sdnq_decompressor.to(devices.device)(model_weights.to(devices.device), skip_int8_matmul=True) + dequant_weight = self.sdnq_decompressor.to(devices.device)(model_weights.to(devices.device), skip_int8_matmul=self.sdnq_decompressor.use_int8_matmul) new_weight = dequant_weight.to(devices.device, dtype=torch.float32) + lora_weights.to(devices.device, dtype=torch.float32) self.weight = torch.nn.Parameter(new_weight, requires_grad=False) self.sdnq_decompressor = None diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 5a3f1b81b..dda1d28d5 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -117,6 +117,7 @@ def sdnq_quantize_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c compressed_weight_shape=compressed_weight.shape, result_dtype=torch_dtype, result_shape=result_shape, + use_int8_matmul=use_int8_matmul, ) else: decompressor = INT4SymmetricWeightsDecompressor( @@ -124,6 +125,7 @@ def sdnq_quantize_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c compressed_weight_shape=compressed_weight.shape, result_dtype=torch_dtype, result_shape=result_shape, + use_int8_matmul=use_int8_matmul, ) else: if is_asym_mode: @@ -132,12 +134,14 @@ def sdnq_quantize_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c zero_point=zero_point.data, result_dtype=torch_dtype, result_shape=result_shape, + use_int8_matmul=use_int8_matmul, ) else: decompressor = INT8SymmetricWeightsDecompressor( scale=scale.data, result_dtype=torch_dtype, result_shape=result_shape, + use_int8_matmul=use_int8_matmul, ) compressed_weight = decompressor.pack_weight(compressed_weight).to(return_device) @@ -516,6 +520,7 @@ class INT8AsymmetricWeightsDecompressor(torch.nn.Module): zero_point: torch.Tensor, result_dtype: torch.dtype, result_shape: torch.Size, + use_int8_matmul: bool = False, ): super().__init__() self.num_bits = 8 @@ -524,6 +529,7 @@ class INT8AsymmetricWeightsDecompressor(torch.nn.Module): self.zero_point = zero_point self.result_dtype = result_dtype self.result_shape = result_shape + self.use_int8_matmul = use_int8_matmul def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: if debug: @@ -541,6 +547,7 @@ class INT8SymmetricWeightsDecompressor(torch.nn.Module): scale: torch.Tensor, result_dtype: torch.dtype, result_shape: torch.Size, + use_int8_matmul: bool = False, ): super().__init__() self.num_bits = 8 @@ -548,6 +555,7 @@ class INT8SymmetricWeightsDecompressor(torch.nn.Module): self.scale = scale self.result_dtype = result_dtype self.result_shape = result_shape + self.use_int8_matmul = use_int8_matmul def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: if debug: @@ -567,6 +575,7 @@ class INT4AsymmetricWeightsDecompressor(torch.nn.Module): compressed_weight_shape: torch.Size, result_dtype: torch.dtype, result_shape: torch.Size, + use_int8_matmul: bool = False, ): super().__init__() self.num_bits = 4 @@ -576,6 +585,7 @@ class INT4AsymmetricWeightsDecompressor(torch.nn.Module): self.compressed_weight_shape = compressed_weight_shape self.result_dtype = result_dtype self.result_shape = result_shape + self.use_int8_matmul = use_int8_matmul def pack_weight(self, weight: torch.Tensor, **kwargs) -> torch.Tensor: if debug: @@ -594,6 +604,7 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): compressed_weight_shape: torch.Size, result_dtype: torch.dtype, result_shape: torch.Size, + use_int8_matmul: bool = False, ): super().__init__() self.num_bits = 4 @@ -602,6 +613,7 @@ class INT4SymmetricWeightsDecompressor(torch.nn.Module): self.compressed_weight_shape = compressed_weight_shape self.result_dtype = result_dtype self.result_shape = result_shape + self.use_int8_matmul = use_int8_matmul def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: if debug: