diff --git a/modules/model_quant_nncf.py b/modules/model_quant_nncf.py index 09a00cdd4..cdcf054f5 100644 --- a/modules/model_quant_nncf.py +++ b/modules/model_quant_nncf.py @@ -62,21 +62,26 @@ def nncf_compress_layer(layer, num_bits, is_asym_mode, torch_dtype=None, quant_c reduction_axes = [i for i in range(layer.weight.ndim) if i != 1] else: reduction_axes = -1 - if shared.opts.nncf_compress_weights_num_of_groups > 1 or num_bits == 4: - num_of_groups = shared.opts.nncf_compress_weights_num_of_groups + if shared.opts.nncf_compress_weights_group_size > 0 or (num_bits == 4 and shared.opts.nncf_compress_weights_group_size != -1): + group_size = shared.opts.nncf_compress_weights_group_size channel_size = layer.weight.shape[-1] - if num_of_groups > channel_size: - group_size = 1 - num_of_groups = channel_size - elif num_of_groups == 0: - group_size = 128 + if group_size == 0: + group_size = 64 num_of_groups = channel_size // group_size + + if group_size >= channel_size: + group_size = channel_size + num_of_groups = 1 else: - group_size = channel_size / num_of_groups - while channel_size % group_size != 0: # find something divisible - num_of_groups -= 1 - group_size = channel_size / num_of_groups + num_of_groups = channel_size // group_size + while channel_size % group_size != 0: # find something divisible + num_of_groups -= 1 + if num_of_groups <= 1: + group_size = channel_size + num_of_groups = 1 + break + group_size = channel_size / num_of_groups if num_of_groups > 1: result_shape = layer.weight.shape diff --git a/modules/shared.py b/modules/shared.py index ee27a44b9..ca9f10235 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -545,8 +545,7 @@ options_templates.update(options_section(('quantization', "Quantization Settings "nncf_compress_mode": OptionInfo("post", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native and not cmd_opts.use_openvino}), "nncf_compress_weights_mode": OptionInfo("INT8_SYM", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'] if cmd_opts.use_openvino else ['INT8', 'INT8_SYM', 'INT4', 'INT4_SYM']}), "nncf_compress_weights_raito": OptionInfo(0, "Compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}), - "nncf_compress_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 512, "step": 1, "visible": cmd_opts.use_openvino}), - "nncf_compress_weights_num_of_groups": OptionInfo(0, "Number of quantization groups", gr.Slider, {"minimum": 0, "maximum": 128, "step": 1, "visible": native and not cmd_opts.use_openvino}), + "nncf_compress_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "nncf_quantize": OptionInfo([], "OpenVINO enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "TE"], "visible": cmd_opts.use_openvino}), "nncf_quantize_mode": OptionInfo("INT8", "OpenVINO activations mode", gr.Dropdown, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}), "nncf_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native and not cmd_opts.use_openvino}),