Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-18 16:54:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

NNCF use torch.compile directly on int8_matmul instead of sub functions

Browse Source
This commit is contained in:
Disty0
2025-05-13 04:28:37 +03:00
parent f1eefe97a4
commit 129c701b3d
1 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/model_quant_nncf.py
+7 -2
View File
@@ -667,8 +667,13 @@ if shared.opts.nncf_decompress_compile:
decompress_int4_asymmetric_compiled = torch.compile(decompress_int4_asymmetric, fullgraph=True)
decompress_int4_symmetric_compiled = torch.compile(decompress_int4_symmetric, fullgraph=True)
quantize_int8_matmul_input_compiled = torch.compile(quantize_int8_matmul_input, fullgraph=True)
unpack_int4_compiled = torch.compile(unpack_int4, fullgraph=True)
if devices.backend != "ipex": # pytorch uses the cpu device in torch._int_mm op with ipex + torch.compile
int8_matmul = torch.compile(int8_matmul, fullgraph=True)
quantize_int8_matmul_input_compiled = quantize_int8_matmul_input
unpack_int4_compiled = unpack_int4
else:
quantize_int8_matmul_input_compiled = torch.compile(quantize_int8_matmul_input, fullgraph=True)
unpack_int4_compiled = torch.compile(unpack_int4, fullgraph=True)
except Exception as e:
shared.log.warning(f"Quantization: type=nncf Decompress using torch.compile is not available: {e}")
decompress_asymmetric_compiled = decompress_asymmetric