From b9ad55857d8c17ee2cec5db05930a4835fbf9eba Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 13 May 2025 05:08:22 +0300 Subject: [PATCH] NNCF INT8 MatMul don't force FP32 with FP16 scales --- modules/model_quant_nncf.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/model_quant_nncf.py b/modules/model_quant_nncf.py index 5c830d161..206fb9cf3 100644 --- a/modules/model_quant_nncf.py +++ b/modules/model_quant_nncf.py @@ -472,9 +472,7 @@ def unpack_int4(packed_tensor: torch.Tensor, shape: torch.Size, dtype: Optional[ def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]: input_scale = torch.div(input.abs().max(), 127) input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(torch.int8).flatten(0,-2) - - scale_dtype = torch.float32 if input.dtype == torch.float16 else torch.bfloat16 - scale = torch.mul(input_scale.to(dtype=scale_dtype), scale.to(dtype=scale_dtype)) + scale = torch.mul(input_scale, scale) return input, scale