From 22d86acda35b50dd4b881be9b4af2d47fd8adcd0 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 9 Aug 2025 01:10:05 +0300 Subject: [PATCH] Make SDNQ MatMul listen to the dequantize fp32 setting --- modules/sdnq/layers/linear/linear_fp8.py | 3 +-- modules/sdnq/layers/linear/linear_fp8_tensorwise.py | 2 +- modules/sdnq/layers/linear/linear_int8.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/sdnq/layers/linear/linear_fp8.py b/modules/sdnq/layers/linear/linear_fp8.py index 17cac32e5..3d2f4059f 100644 --- a/modules/sdnq/layers/linear/linear_fp8.py +++ b/modules/sdnq/layers/linear/linear_fp8.py @@ -8,10 +8,9 @@ from ...common import use_torch_compile # noqa: TID252 def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.Tensor, torch.FloatTensor]: - input = input.flatten(0,-2).contiguous() + input = input.flatten(0,-2).contiguous().to(dtype=torch.float32) input_scale = torch.amax(input.abs(), dim=-1, keepdims=True).div_(448) input = torch.div(input, input_scale).clamp_(-448, 448).to(dtype=torch.float8_e4m3fn) - input_scale = input_scale.to(dtype=torch.float32) return input, input_scale diff --git a/modules/sdnq/layers/linear/linear_fp8_tensorwise.py b/modules/sdnq/layers/linear/linear_fp8_tensorwise.py index 768719d18..1cdf5ced4 100644 --- a/modules/sdnq/layers/linear/linear_fp8_tensorwise.py +++ b/modules/sdnq/layers/linear/linear_fp8_tensorwise.py @@ -9,7 +9,7 @@ from ...dequantizer import dequantize_symmetric, dequantize_symmetric_with_bias def quantize_fp8_matmul_input_tensorwise(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.Tensor, torch.FloatTensor]: - input = input.flatten(0,-2).contiguous() + input = input.flatten(0,-2).contiguous().to(dtype=scale.dtype) input_scale = torch.amax(input.abs(), dim=-1, keepdims=True).div_(448) input = torch.div(input, input_scale).clamp_(-448, 448).to(dtype=torch.float8_e4m3fn) scale = torch.mul(input_scale, scale) diff --git a/modules/sdnq/layers/linear/linear_int8.py b/modules/sdnq/layers/linear/linear_int8.py index 1be08a5ae..dfc5288a0 100644 --- a/modules/sdnq/layers/linear/linear_int8.py +++ b/modules/sdnq/layers/linear/linear_int8.py @@ -10,7 +10,7 @@ from ...dequantizer import dequantize_symmetric, dequantize_symmetric_with_bias def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.CharTensor, torch.FloatTensor]: - input = input.flatten(0,-2).contiguous() + input = input.flatten(0,-2).contiguous().to(dtype=scale.dtype) input_scale = torch.amax(input.abs(), dim=-1, keepdims=True).div_(127) input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(dtype=torch.int8) scale = torch.mul(input_scale, scale)