Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-18 16:54:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

NNCF INT8 MatMul don't force FP32 with FP16 scales

Browse Source
This commit is contained in:
Disty0
2025-05-13 05:08:22 +03:00
parent 129c701b3d
commit b9ad55857d
1 changed files with 1 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/model_quant_nncf.py
+1 -3
View File
@@ -472,9 +472,7 @@ def unpack_int4(packed_tensor: torch.Tensor, shape: torch.Size, dtype: Optional[
def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]:
input_scale = torch.div(input.abs().max(), 127)
input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(torch.int8).flatten(0,-2)
scale_dtype = torch.float32 if input.dtype == torch.float16 else torch.bfloat16
scale = torch.mul(input_scale.to(dtype=scale_dtype), scale.to(dtype=scale_dtype))
scale = torch.mul(input_scale, scale)
return input, scale