Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 01:04:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

NNCF set the default group size to 128 for INT4

Browse Source
This commit is contained in:
Disty0
2025-05-11 08:45:27 +03:00
parent 4af570bfd4
commit 0673689d5b
2 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/model_quant_nncf.py
+6 -2
View File
@@ -62,11 +62,15 @@ def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c
reduction_axes = [i for i in range(layer.weight.ndim) if i != 1]
else:
reduction_axes = -1
if shared.opts.nncf_compress_weights_num_of_groups > 1:
if shared.opts.nncf_compress_weights_num_of_groups > 1 or num_bits == 4:
num_of_groups = shared.opts.nncf_compress_weights_num_of_groups
channel_size = layer.weight.shape[-1]
group_size = channel_size / num_of_groups
if num_of_groups == 0:
group_size = 128
num_of_groups = channel_size // group_size
else:
group_size = channel_size / num_of_groups
while channel_size % group_size != 0: # find something divisible
num_of_groups -= 1
group_size = channel_size / num_of_groups