Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-18 16:54:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

SDNQ use the model quant params instead of user settings on Lora

Browse Source
This commit is contained in:
Disty0
2025-11-10 00:12:38 +03:00
parent 55658bf76c
commit 6f33ec3357
5 changed files with 80 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/lora/lora_apply.py
+19 -15
View File
@@ -167,23 +167,27 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G
try:
from modules.sdnq import sdnq_quantize_layer
if hasattr(self, "sdnq_dequantizer_backup"):
weights_dtype = self.sdnq_dequantizer_backup.weights_dtype
use_svd = bool(self.sdnq_svd_up_backup is not None)
dequantize_fp32 = bool(self.sdnq_scale_backup.dtype == torch.float32)
sdnq_dequantizer = self.sdnq_dequantizer_backup
dequant_weight = self.sdnq_dequantizer_backup.to(devices.device)(
model_weights.to(devices.device),
self.sdnq_scale_backup.to(devices.device),
self.sdnq_zero_point_backup.to(devices.device) if self.sdnq_zero_point_backup is not None else None,
self.sdnq_svd_up_backup.to(devices.device) if self.sdnq_svd_up_backup is not None else None,
self.sdnq_svd_down_backup.to(devices.device) if self.sdnq_svd_down_backup is not None else None,
self.sdnq_svd_up_backup.to(devices.device) if use_svd else None,
self.sdnq_svd_down_backup.to(devices.device) if use_svd else None,
skip_quantized_matmul=self.sdnq_dequantizer_backup.use_quantized_matmul
)
else:
weights_dtype = self.sdnq_dequantizer.weights_dtype
use_svd = bool(self.svd_up is not None)
dequantize_fp32 = bool(self.scale.dtype == torch.float32)
sdnq_dequantizer = self.sdnq_dequantizer
dequant_weight = self.sdnq_dequantizer.to(devices.device)(
model_weights.to(devices.device),
self.scale.to(devices.device),
self.zero_point.to(devices.device) if self.zero_point is not None else None,
self.svd_up.to(devices.device) if self.svd_up is not None else None,
self.svd_down.to(devices.device) if self.svd_down is not None else None,
self.svd_up.to(devices.device) if use_svd else None,
self.svd_down.to(devices.device) if use_svd else None,
skip_quantized_matmul=self.sdnq_dequantizer.use_quantized_matmul
)
@@ -192,16 +196,16 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G
del self.sdnq_dequantizer, self.scale, self.zero_point, self.svd_up, self.svd_down
self = sdnq_quantize_layer(
self,
weights_dtype=weights_dtype,
torch_dtype=devices.dtype,
group_size=shared.opts.sdnq_quantize_weights_group_size,
svd_rank=shared.opts.sdnq_svd_rank,
weights_dtype=sdnq_dequantizer.weights_dtype,
torch_dtype=sdnq_dequantizer.result_dtype,
group_size=sdnq_dequantizer.group_size,
svd_rank=sdnq_dequantizer.svd_rank,
use_quantized_matmul=sdnq_dequantizer.use_quantized_matmul,
use_quantized_matmul_conv=sdnq_dequantizer.use_quantized_matmul,
use_svd=use_svd,
dequantize_fp32=dequantize_fp32,
svd_steps=shared.opts.sdnq_svd_steps,
use_svd=shared.opts.sdnq_use_svd,
quant_conv=shared.opts.sdnq_quantize_conv_layers,
use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul,
use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv,
dequantize_fp32=shared.opts.sdnq_dequantize_fp32,
quant_conv=True, # quant_conv is True if conv layers ends up here
non_blocking=False,
quantization_device=devices.device,
return_device=device,
modules/sdnq/dequantizer.py
+56 -30
View File
@@ -137,17 +137,25 @@ class AsymmetricWeightsDequantizer(torch.nn.Module):
result_dtype: torch.dtype,
result_shape: torch.Size,
original_shape: torch.Size,
quantized_weight_shape: torch.Size,
weights_dtype: str,
use_quantized_matmul: bool = False,
**kwargs, # pylint: disable=unused-argument
group_size: int,
svd_rank: int,
use_quantized_matmul: bool,
re_quantize_for_matmul: bool,
):
super().__init__()
self.weights_dtype = weights_dtype
self.original_shape = original_shape
self.use_quantized_matmul = use_quantized_matmul
self.re_quantize_for_matmul = True
self.is_packed = False
self.is_asym = True
self.result_dtype = result_dtype
self.result_shape = result_shape
self.original_shape = original_shape
self.quantized_weight_shape = quantized_weight_shape
self.weights_dtype = weights_dtype
self.group_size = group_size
self.svd_rank = svd_rank
self.use_quantized_matmul = use_quantized_matmul
self.re_quantize_for_matmul = re_quantize_for_matmul
def pack_weight(self, weight: torch.Tensor) -> torch.Tensor:
return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"])
@@ -165,18 +173,25 @@ class SymmetricWeightsDequantizer(torch.nn.Module):
result_dtype: torch.dtype,
result_shape: torch.Size,
original_shape: torch.Size,
quantized_weight_shape: torch.Size,
weights_dtype: str,
use_quantized_matmul: bool = False,
re_quantize_for_matmul: bool = False,
**kwargs, # pylint: disable=unused-argument
group_size: int,
svd_rank: int,
use_quantized_matmul: bool,
re_quantize_for_matmul: bool,
):
super().__init__()
self.weights_dtype = weights_dtype
self.original_shape = original_shape
self.use_quantized_matmul = use_quantized_matmul
self.re_quantize_for_matmul = re_quantize_for_matmul
self.is_packed = False
self.is_asym = False
self.result_dtype = result_dtype
self.result_shape = result_shape
self.original_shape = original_shape
self.quantized_weight_shape = quantized_weight_shape
self.weights_dtype = weights_dtype
self.group_size = group_size
self.svd_rank = svd_rank
self.use_quantized_matmul = use_quantized_matmul
self.re_quantize_for_matmul = re_quantize_for_matmul
def pack_weight(self, weight: torch.Tensor) -> torch.Tensor:
return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"])
@@ -192,22 +207,28 @@ class SymmetricWeightsDequantizer(torch.nn.Module):
class PackedINTAsymmetricWeightsDequantizer(torch.nn.Module):
def __init__(
self,
quantized_weight_shape: torch.Size,
result_dtype: torch.dtype,
result_shape: torch.Size,
original_shape: torch.Size,
quantized_weight_shape: torch.Size,
weights_dtype: str,
use_quantized_matmul: bool = False,
**kwargs, # pylint: disable=unused-argument
group_size: int,
svd_rank: int,
use_quantized_matmul: bool,
re_quantize_for_matmul: bool,
):
super().__init__()
self.weights_dtype = weights_dtype
self.use_quantized_matmul = use_quantized_matmul
self.re_quantize_for_matmul = True
self.original_shape = original_shape
self.quantized_weight_shape = quantized_weight_shape
self.is_packed = True
self.is_asym = True
self.result_dtype = result_dtype
self.result_shape = result_shape
self.original_shape = original_shape
self.quantized_weight_shape = quantized_weight_shape
self.weights_dtype = weights_dtype
self.group_size = group_size
self.svd_rank = svd_rank
self.use_quantized_matmul = use_quantized_matmul
self.re_quantize_for_matmul = re_quantize_for_matmul
def pack_weight(self, weight: torch.Tensor) -> torch.Tensor:
return pack_int_asymetric(weight, self.weights_dtype)
@@ -222,23 +243,28 @@ class PackedINTAsymmetricWeightsDequantizer(torch.nn.Module):
class PackedINTSymmetricWeightsDequantizer(torch.nn.Module):
def __init__(
self,
quantized_weight_shape: torch.Size,
result_dtype: torch.dtype,
result_shape: torch.Size,
original_shape: torch.Size,
quantized_weight_shape: torch.Size,
weights_dtype: str,
use_quantized_matmul: bool = False,
re_quantize_for_matmul: bool = False,
**kwargs, # pylint: disable=unused-argument
group_size: int,
svd_rank: int,
use_quantized_matmul: bool,
re_quantize_for_matmul: bool,
):
super().__init__()
self.weights_dtype = weights_dtype
self.original_shape = original_shape
self.use_quantized_matmul = use_quantized_matmul
self.re_quantize_for_matmul = re_quantize_for_matmul
self.quantized_weight_shape = quantized_weight_shape
self.is_packed = True
self.is_asym = False
self.result_dtype = result_dtype
self.result_shape = result_shape
self.original_shape = original_shape
self.quantized_weight_shape = quantized_weight_shape
self.weights_dtype = weights_dtype
self.group_size = group_size
self.svd_rank = svd_rank
self.use_quantized_matmul = use_quantized_matmul
self.re_quantize_for_matmul = re_quantize_for_matmul
def pack_weight(self, weight: torch.Tensor) -> torch.Tensor:
return pack_int_symetric(weight, self.weights_dtype)
modules/sdnq/layers/conv/conv_int8.py
+1 -1
View File
@@ -74,7 +74,7 @@ def quantized_conv_forward_int8_matmul(self, input) -> torch.FloatTensor:
else:
weight = self.weight
scale = self.scale
quantized_weight_shape = getattr(self.sdnq_dequantizer, "quantized_weight_shape", None)
quantized_weight_shape = self.sdnq_dequantizer.quantized_weight_shape if self.sdnq_dequantizer.is_packed else None
return conv_int8_matmul(
input, weight, self.bias,
scale, self.svd_up, self.svd_down,
modules/sdnq/layers/linear/linear_int8.py
+1 -1
View File
@@ -56,7 +56,7 @@ def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torc
else:
weight = self.weight
scale = self.scale
quantized_weight_shape = getattr(self.sdnq_dequantizer, "quantized_weight_shape", None)
quantized_weight_shape = self.sdnq_dequantizer.quantized_weight_shape if self.sdnq_dequantizer.is_packed else None
return int8_matmul(input, weight, self.bias, scale, self.svd_up, self.svd_down, quantized_weight_shape, self.sdnq_dequantizer.weights_dtype)
modules/sdnq/quantizer.py
+3 -1
View File
@@ -321,11 +321,13 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz
layer.svd_up, layer.svd_down = None, None
layer.sdnq_dequantizer = dequantizer_dict[weights_dtype](
quantized_weight_shape=layer.weight.shape,
result_dtype=torch_dtype,
result_shape=result_shape,
original_shape=original_shape,
quantized_weight_shape=layer.weight.shape,
weights_dtype=weights_dtype,
group_size=group_size,
svd_rank=svd_rank,
use_quantized_matmul=use_quantized_matmul,
re_quantize_for_matmul=re_quantize_for_matmul,
)