diff --git a/modules/sdnq/common.py b/modules/sdnq/common.py index fa121416b..04fc6ee4d 100644 --- a/modules/sdnq/common.py +++ b/modules/sdnq/common.py @@ -6,7 +6,7 @@ import torch from modules import shared, devices -sdnq_version = "0.1.9" +sdnq_version = "0.2.0" sdnq_keys = {"weight", "scale", "zero_point", "svd_up", "svd_down"} torch_version = torch.__version__[:4] diff --git a/modules/sdnq/quant_utils.py b/modules/sdnq/quant_utils.py index a8789f84e..d286e46c8 100644 --- a/modules/sdnq/quant_utils.py +++ b/modules/sdnq/quant_utils.py @@ -105,7 +105,7 @@ def get_hadamard(n: int, dtype: torch.dtype | None = None, device: torch.device @devices.inference_context() -def rotate_hadamard(weight: torch.Tensor, group_size: int = 128, hadamard: torch.Tensor | None = None, is_conv: bool = False) -> torch.Tensor: +def rotate_hadamard(weight: torch.Tensor, group_size: int = 128, hadamard: torch.FloatTensor | None = None, is_conv: bool = False) -> torch.Tensor: if hadamard is None: hadamard = get_hadamard(group_size, dtype=weight.dtype, device=weight.device) else: @@ -122,7 +122,7 @@ def rotate_hadamard(weight: torch.Tensor, group_size: int = 128, hadamard: torch @devices.inference_context() -def apply_hadamard(weight: torch.Tensor, group_size: int = 128, hadamard: torch.Tensor | None = None, layer_class_name: str | None = None) -> torch.Tensor: +def apply_hadamard(weight: torch.Tensor, group_size: int = 128, hadamard: torch.FloatTensor | None = None, layer_class_name: str | None = None) -> torch.Tensor: is_conv = False use_hadamard = True if layer_class_name in conv_types or layer_class_name in conv_transpose_types: @@ -138,7 +138,7 @@ def apply_hadamard(weight: torch.Tensor, group_size: int = 128, hadamard: torch. if group_size < 4: use_hadamard = False if use_hadamard: - weight = rotate_hadamard(weight, group_size=group_size, is_conv=is_conv) + weight = rotate_hadamard(weight, group_size=group_size, hadamard=hadamard, is_conv=is_conv) return weight, use_hadamard, group_size diff --git a/modules/sdnq/quantizer.py b/modules/sdnq/quantizer.py index e2e4045e1..fd3d7005b 100644 --- a/modules/sdnq/quantizer.py +++ b/modules/sdnq/quantizer.py @@ -181,7 +181,6 @@ def sdnq_quantize_layer_weight(weight, layer_class_name=None, weights_dtype="int else: group_size = -1 - cast_scale = True transpose_weights = False re_quantize_for_matmul = re_quantize_for_matmul or num_of_groups > 1 @@ -434,6 +433,7 @@ def sdnq_post_load_quant( ): quantization_config = SDNQConfig( weights_dtype=weights_dtype, + quantized_matmul_dtype=quantized_matmul_dtype, hadamard_group_size=hadamard_group_size, group_size=group_size, svd_rank=svd_rank,