From 95a7da7e75350a0f9179f339fc06e36ece2d0b60 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 3 Oct 2025 18:54:58 +0300 Subject: [PATCH] SDNQ use non-contiguous re-quantize --- modules/sdnq/__init__.py | 1 + modules/sdnq/dequantizer.py | 23 +++++++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index 16fe6ec47..c7d832bb9 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -197,6 +197,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz re_quantize_for_matmul=re_quantize_for_matmul, ) layer.weight.data = layer.sdnq_dequantizer.pack_weight(layer.weight).to(return_device, non_blocking=non_blocking) + layer.sdnq_dequantizer = layer.sdnq_dequantizer.to(return_device, non_blocking=non_blocking) layer.forward = get_forward_func(layer_class_name, use_quantized_matmul, dtype_dict[weights_dtype]["is_integer"], use_tensorwise_fp8_matmul) layer.forward = layer.forward.__get__(layer, layer.__class__) diff --git a/modules/sdnq/dequantizer.py b/modules/sdnq/dequantizer.py index d82849177..22771b643 100644 --- a/modules/sdnq/dequantizer.py +++ b/modules/sdnq/dequantizer.py @@ -48,18 +48,25 @@ def quantize_fp8(input: torch.FloatTensor, dim: int = -1) -> Tuple[torch.Tensor, return input, scale +def re_quantize_int8(weight: torch.FloatTensor) -> Tuple[torch.CharTensor, torch.FloatTensor]: + if weight.ndim > 2: # convs + weight = weight.flatten(1,-1) + if weight.device.type == "xpu": + # return contiguous + weight, scale = quantize_int8(weight.t(), dim=0) + else: + # return non-contiguous + weight, scale = quantize_int8(weight, dim=-1) + weight, scale = weight.t_(), scale.t_() + return weight, scale + + def re_quantize_matmul_asymmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, result_shape: torch.Size) -> Tuple[torch.CharTensor, torch.FloatTensor]: - result = dequantize_asymmetric(weight, scale, zero_point, scale.dtype, result_shape) - if result.ndim > 2: # convs - result = result.flatten(1,-1) - return quantize_int8(result.t_(), dim=0) + return re_quantize_int8(dequantize_asymmetric(weight, scale, zero_point, scale.dtype, result_shape)) def re_quantize_matmul_symmetric(weight: torch.CharTensor, scale: torch.FloatTensor, result_shape: torch.Size) -> Tuple[torch.CharTensor, torch.FloatTensor]: - result = dequantize_symmetric(weight, scale, scale.dtype, result_shape) - if result.ndim > 2: # convs - result = result.flatten(1,-1) - return quantize_int8(result.t_(), dim=0) + return re_quantize_int8(dequantize_symmetric(weight, scale, scale.dtype, result_shape)) def re_quantize_matmul_packed_int_asymmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, shape: torch.Size, result_shape: torch.Size, weights_dtype: str) -> torch.FloatTensor: