Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-18 16:54:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

SDNQ use non-contiguous re-quantize

Browse Source
This commit is contained in:
Disty0
2025-10-03 18:54:58 +03:00
parent a6108dd6df
commit 95a7da7e75
2 changed files with 16 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/sdnq/__init__.py
+1
View File
@@ -197,6 +197,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz
re_quantize_for_matmul=re_quantize_for_matmul,
)
layer.weight.data = layer.sdnq_dequantizer.pack_weight(layer.weight).to(return_device, non_blocking=non_blocking)
layer.sdnq_dequantizer = layer.sdnq_dequantizer.to(return_device, non_blocking=non_blocking)
layer.forward = get_forward_func(layer_class_name, use_quantized_matmul, dtype_dict[weights_dtype]["is_integer"], use_tensorwise_fp8_matmul)
layer.forward = layer.forward.__get__(layer, layer.__class__)
modules/sdnq/dequantizer.py
+15 -8
View File
@@ -48,18 +48,25 @@ def quantize_fp8(input: torch.FloatTensor, dim: int = -1) -> Tuple[torch.Tensor,
return input, scale
def re_quantize_int8(weight: torch.FloatTensor) -> Tuple[torch.CharTensor, torch.FloatTensor]:
if weight.ndim > 2: # convs
weight = weight.flatten(1,-1)
if weight.device.type == "xpu":
# return contiguous
weight, scale = quantize_int8(weight.t(), dim=0)
else:
# return non-contiguous
weight, scale = quantize_int8(weight, dim=-1)
weight, scale = weight.t_(), scale.t_()
return weight, scale
def re_quantize_matmul_asymmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, result_shape: torch.Size) -> Tuple[torch.CharTensor, torch.FloatTensor]:
result = dequantize_asymmetric(weight, scale, zero_point, scale.dtype, result_shape)
if result.ndim > 2: # convs
result = result.flatten(1,-1)
return quantize_int8(result.t_(), dim=0)
return re_quantize_int8(dequantize_asymmetric(weight, scale, zero_point, scale.dtype, result_shape))
def re_quantize_matmul_symmetric(weight: torch.CharTensor, scale: torch.FloatTensor, result_shape: torch.Size) -> Tuple[torch.CharTensor, torch.FloatTensor]:
result = dequantize_symmetric(weight, scale, scale.dtype, result_shape)
if result.ndim > 2: # convs
result = result.flatten(1,-1)
return quantize_int8(result.t_(), dim=0)
return re_quantize_int8(dequantize_symmetric(weight, scale, scale.dtype, result_shape))
def re_quantize_matmul_packed_int_asymmetric(weight: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, shape: torch.Size, result_shape: torch.Size, weights_dtype: str) -> torch.FloatTensor: