diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b4dc84c4..ea24e138b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - Add group size support for convolutional layers - Add quantized matmul support for for convolutional layers - Add 7-bit, 5-bit and 3-bit quantization support + - Add separate quant mode option for Text Encoders - Fix forced FP32 with tensorwise FP8 matmul - Fix PyTorch <= 2.4 compatibility with FP8 matmul - Fix VAE with conv quant diff --git a/modules/model_quant.py b/modules/model_quant.py index 6e2429e35..b90297ee6 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -113,14 +113,20 @@ def create_sdnq_config(kwargs = None, allow_sdnq: bool = True, module: str = 'Mo diffusers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["sdnq"] = SDNQConfig transformers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["sdnq"] = SDNQConfig + if weights_dtype is None: + if shared.opts.sdnq_quantize_weights_mode_te != "default" and module in {"TE", "LLM"}: + weights_dtype = shared.opts.sdnq_quantize_weights_mode_te + else: + weights_dtype = shared.opts.sdnq_quantize_weights_mode + sdnq_config = SDNQConfig( - weights_dtype=weights_dtype if weights_dtype is not None else shared.opts.sdnq_quantize_weights_mode, + weights_dtype=weights_dtype, group_size=shared.opts.sdnq_quantize_weights_group_size, quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, ) - log.debug(f'Quantization: module="{module}" type=sdnq dtype={shared.opts.sdnq_quantize_weights_mode}') + log.debug(f'Quantization: module="{module}" type=sdnq dtype={weights_dtype}') if kwargs is None: return sdnq_config else: @@ -320,9 +326,14 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc=True): if hasattr(model, "get_input_embeddings"): backup_embeddings = copy.deepcopy(model.get_input_embeddings()) + if shared.opts.sdnq_quantize_weights_mode_te != "default" and op is not None and "text_encoder" in op: + weights_dtype = shared.opts.sdnq_quantize_weights_mode_te + else: + weights_dtype = shared.opts.sdnq_quantize_weights_mode + model = apply_sdnq_to_module( model, - weights_dtype=shared.opts.sdnq_quantize_weights_mode, + weights_dtype=weights_dtype, torch_dtype=devices.dtype, group_size=shared.opts.sdnq_quantize_weights_group_size, quant_conv=shared.opts.sdnq_quantize_conv_layers, diff --git a/modules/shared.py b/modules/shared.py index 4232d3c65..2231e0484 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -519,6 +519,7 @@ options_templates.update(options_section(("quantization", "Quantization Settings "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ["pre", "post"], "visible": native}), "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "float8_e4m3fn", "int7", "int6", "int5", "uint4", "uint3", "uint2", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "uint8", "uint7", "uint6", "uint5", "int4", "int3", "int2", "uint1"], "visible": native}), + "sdnq_quantize_weights_mode_te": OptionInfo("default", "Quantization type for Text Encoders", gr.Dropdown, {"choices": ["default", "int8", "float8_e4m3fn", "int7", "int6", "int5", "uint4", "uint3", "uint2", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "uint8", "uint7", "uint6", "uint5", "int4", "int3", "int2", "uint1"], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_dequantize_compile": OptionInfo(devices.has_triton(), "Dequantize using torch.compile", gr.Checkbox, {"visible": native}), diff --git a/wiki b/wiki index 04cfb75b8..19a1dca01 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 04cfb75b8911c227109c0b0dbe64f11f71ef5619 +Subproject commit 19a1dca01821204cc114f7a955705322b6186ee0