Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-18 16:54:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

NNCF use group size instead of number of groups and set default group size for int4 to 64

Browse Source
This commit is contained in:
Disty0
2025-05-11 20:38:01 +03:00
parent 03d05b6243
commit 4eedeab9f8
2 changed files with 17 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/model_quant_nncf.py
+16 -11
View File
@@ -62,21 +62,26 @@ def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c
reduction_axes = [i for i in range(layer.weight.ndim) if i != 1]
else:
reduction_axes = -1
if shared.opts.nncf_compress_weights_num_of_groups > 1 or num_bits == 4:
num_of_groups = shared.opts.nncf_compress_weights_num_of_groups
if shared.opts.nncf_compress_weights_group_size > 0 or (num_bits == 4 and shared.opts.nncf_compress_weights_group_size != -1):
group_size = shared.opts.nncf_compress_weights_group_size
channel_size = layer.weight.shape[-1]
if num_of_groups > channel_size:
group_size = 1
num_of_groups = channel_size
elif num_of_groups == 0:
group_size = 128
if group_size == 0:
group_size = 64
num_of_groups = channel_size // group_size
if group_size >= channel_size:
group_size = channel_size
num_of_groups = 1
else:
group_size = channel_size / num_of_groups
while channel_size % group_size != 0: # find something divisible
num_of_groups -= 1
group_size = channel_size / num_of_groups
num_of_groups = channel_size // group_size
while channel_size % group_size != 0: # find something divisible
num_of_groups -= 1
if num_of_groups <= 1:
group_size = channel_size
num_of_groups = 1
break
group_size = channel_size / num_of_groups
if num_of_groups > 1:
result_shape = layer.weight.shape
modules/shared.py
+1 -2
View File
@@ -545,8 +545,7 @@ options_templates.update(options_section(('quantization', "Quantization Settings
"nncf_compress_mode": OptionInfo("post", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native and not cmd_opts.use_openvino}),
"nncf_compress_weights_mode": OptionInfo("INT8_SYM", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'] if cmd_opts.use_openvino else ['INT8', 'INT8_SYM', 'INT4', 'INT4_SYM']}),
"nncf_compress_weights_raito": OptionInfo(0, "Compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}),
"nncf_compress_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 512, "step": 1, "visible": cmd_opts.use_openvino}),
"nncf_compress_weights_num_of_groups": OptionInfo(0, "Number of quantization groups", gr.Slider, {"minimum": 0, "maximum": 128, "step": 1, "visible": native and not cmd_opts.use_openvino}),
"nncf_compress_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}),
"nncf_quantize": OptionInfo([], "OpenVINO enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "TE"], "visible": cmd_opts.use_openvino}),
"nncf_quantize_mode": OptionInfo("INT8", "OpenVINO activations mode", gr.Dropdown, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}),
"nncf_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native and not cmd_opts.use_openvino}),