From a78d51ea070928192646388695d390993680bbda Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 20 Aug 2025 02:07:49 +0300 Subject: [PATCH 1/4] Fix Qwen Image --- pipelines/model_qwen.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pipelines/model_qwen.py b/pipelines/model_qwen.py index 9391fb1a4..9c7984dd5 100644 --- a/pipelines/model_qwen.py +++ b/pipelines/model_qwen.py @@ -7,6 +7,7 @@ from pipelines import generic def load_qwen(checkpoint_info, diffusers_load_config={}): repo_id = sd_models.path_to_repo(checkpoint_info) sd_models.hf_auth_check(checkpoint_info) + transformer = None load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, module='Model') shared.log.debug(f'Load model: type=Qwen model="{checkpoint_info.name}" repo="{repo_id}" offload={shared.opts.diffusers_offload_mode} dtype={devices.dtype} args={load_args}') From 47ff01fd3bf853a65b239a6dc34d7fff5b73cd55 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 20 Aug 2025 03:24:19 +0300 Subject: [PATCH 2/4] SDNQ add "*" support and upcast only the first and last layer's img_mod to 6 bit with Qwen Image --- modules/sdnq/__init__.py | 13 +++++++++++-- pipelines/model_qwen.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index bca210c1b..e88bfc8d2 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -4,6 +4,7 @@ from typing import Any, Dict, List, Tuple, Optional, Union from dataclasses import dataclass from enum import Enum +import re import torch from diffusers.quantizers.base import DiffusersQuantizer from diffusers.quantizers.quantization_config import QuantizationConfigMixin @@ -270,7 +271,11 @@ class SDNQQuantizer(DiffusersQuantizer): ): if param_name.endswith(".weight"): split_param_name = param_name.split(".") - if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): + if ( + param_name not in self.modules_to_not_convert + and not any(param in split_param_name for param in self.modules_to_not_convert) + and not any("*" in param and re.match(param.replace("*", ".*"), param_name) for param in self.modules_to_not_convert) + ): layer_class_name = get_module_from_name(model, param_name)[0].__class__.__name__ if layer_class_name in allowed_types: if layer_class_name in conv_types or layer_class_name in conv_transpose_types: @@ -303,7 +308,11 @@ class SDNQQuantizer(DiffusersQuantizer): if len(self.quantization_config.modules_dtype_dict.keys()) > 0: split_param_name = param_name.split(".") for key, value in self.quantization_config.modules_dtype_dict.items(): - if param_name in value or any(param in split_param_name for param in value): + if ( + param_name in value + or any(param in split_param_name for param in value) + or any("*" in param and re.match(param.replace("*", ".*"), param_name) for param in value) + ): key = key.lower() if key in {"8bit", "8bits"}: if dtype_dict[weights_dtype]["num_bits"] != 8: diff --git a/pipelines/model_qwen.py b/pipelines/model_qwen.py index 9c7984dd5..b6e57c95f 100644 --- a/pipelines/model_qwen.py +++ b/pipelines/model_qwen.py @@ -30,7 +30,7 @@ def load_qwen(checkpoint_info, diffusers_load_config={}): # cls_name = nunchaku.pipeline.pipeline_qwenimage.NunchakuQwenImagePipeline # we dont need this if transformer is None: - transformer = generic.load_transformer(repo_id, cls_name=diffusers.QwenImageTransformer2DModel, load_config=diffusers_load_config, modules_dtype_dict={"minimum_6bit": ["img_mod", "pos_embed", "time_text_embed", "img_in", "txt_in", "norm_out"]}) + transformer = generic.load_transformer(repo_id, cls_name=diffusers.QwenImageTransformer2DModel, load_config=diffusers_load_config, modules_dtype_dict={"minimum_6bit": ["pos_embed", "time_text_embed", "img_in", "txt_in", "norm_out", "transformer_blocks.0.img_mod.1.weight", "transformer_blocks.59.img_mod.1.weight"]}) repo_te = 'Qwen/Qwen-Image' # if 'Qwen-Lightning' in repo_id or 'Qwen-Image-Edit' in repo_id else repo_id text_encoder = generic.load_text_encoder(repo_te, cls_name=transformers.Qwen2_5_VLForConditionalGeneration, load_config=diffusers_load_config) From 09467106628812eb2f9adf93989c7232f4af3577 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 20 Aug 2025 04:38:20 +0300 Subject: [PATCH 3/4] Add sdnq_modules_to_not_convert to UI settings --- CHANGELOG.md | 1 + modules/model_quant.py | 15 ++++++++++++--- modules/sdnq/__init__.py | 4 ++-- modules/shared.py | 1 + pipelines/model_qwen.py | 2 +- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58e35d05f..5cb0ba0ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ A quick service release with several important hotfixes, but also adding support default is `rust` as new `xet` is known to cause issues - support for `flux.1-kontext` lora - support for `qwen-image` lora + - add `sdnq_modules_to_not_convert` option to ui settings for - **UI** - new artwork for reference models in networks thanks @liutyi diff --git a/modules/model_quant.py b/modules/model_quant.py index 686703b9b..8b5991e8c 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -1,4 +1,5 @@ import os +import re import sys import copy import time @@ -137,6 +138,10 @@ def create_sdnq_config(kwargs = None, allow: bool = True, module: str = 'Model', if weights_dtype is None or weights_dtype == 'none': return kwargs + sdnq_modules_to_not_convert = [m.strip() for m in re.split(';|,| ', shared.opts.sdnq_modules_to_not_convert) if len(m.strip()) > 1] + if len(sdnq_modules_to_not_convert) > 0: + modules_to_not_convert.extend(sdnq_modules_to_not_convert) + quantization_device, return_device = get_sdnq_devices() sdnq_config = SDNQConfig( @@ -152,7 +157,7 @@ def create_sdnq_config(kwargs = None, allow: bool = True, module: str = 'Model', modules_to_not_convert=modules_to_not_convert, modules_dtype_dict=modules_dtype_dict, ) - log.debug(f'Quantization: module="{module}" type=sdnq dtype={weights_dtype} matmul={shared.opts.sdnq_use_quantized_matmul} group_size={shared.opts.sdnq_quantize_weights_group_size} quant_conv={shared.opts.sdnq_quantize_conv_layers} matmul_conv={shared.opts.sdnq_use_quantized_matmul_conv} dequantize_fp32={shared.opts.sdnq_dequantize_fp32} quantize_with_gpu={shared.opts.sdnq_quantize_with_gpu} quantization_device={quantization_device} return_device={return_device} device_map={shared.opts.device_map} offload_mode={shared.opts.diffusers_offload_mode} non_blocking={shared.opts.diffusers_offload_nonblocking}') + log.debug(f'Quantization: module="{module}" type=sdnq mode=pre dtype={weights_dtype} matmul={shared.opts.sdnq_use_quantized_matmul} group_size={shared.opts.sdnq_quantize_weights_group_size} quant_conv={shared.opts.sdnq_quantize_conv_layers} matmul_conv={shared.opts.sdnq_use_quantized_matmul_conv} dequantize_fp32={shared.opts.sdnq_dequantize_fp32} quantize_with_gpu={shared.opts.sdnq_quantize_with_gpu} quantization_device={quantization_device} return_device={return_device} device_map={shared.opts.device_map} offload_mode={shared.opts.diffusers_offload_mode} non_blocking={shared.opts.diffusers_offload_nonblocking} modules_to_not_convert={modules_to_not_convert} modules_dtype_dict={modules_dtype_dict}') if kwargs is None: return sdnq_config else: @@ -393,8 +398,6 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc: bool = True, weigh if weights_dtype is None or weights_dtype == 'none': return model - if debug: - log.trace(f'Quantization: type=SDNQ op={op} cls={model.__class__} dtype={weights_dtype} mode{shared.opts.diffusers_offload_mode}') quantization_device, return_device = get_sdnq_devices() @@ -410,6 +413,10 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc: bool = True, weigh else: modules_dtype_dict["minimum_6bit"].extend(["img_mod", "pos_embed", "time_text_embed", "img_in", "txt_in", "norm_out"]) + sdnq_modules_to_not_convert = [m.strip() for m in re.split(';|,| ', shared.opts.sdnq_modules_to_not_convert) if len(m.strip()) > 1] + if len(sdnq_modules_to_not_convert) > 0: + modules_to_not_convert.extend(sdnq_modules_to_not_convert) + model.eval() backup_embeddings = None if hasattr(model, "get_input_embeddings"): @@ -459,6 +466,8 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc: bool = True, weigh model = model.to(devices.cpu) if do_gc: devices.torch_gc(force=True, reason='sdnq') + + log.debug(f'Quantization: module="{op if op is not None else model.__class__}" type=sdnq mode=post dtype={weights_dtype} matmul={shared.opts.sdnq_use_quantized_matmul} group_size={shared.opts.sdnq_quantize_weights_group_size} quant_conv={shared.opts.sdnq_quantize_conv_layers} matmul_conv={shared.opts.sdnq_use_quantized_matmul_conv} dequantize_fp32={shared.opts.sdnq_dequantize_fp32} quantize_with_gpu={shared.opts.sdnq_quantize_with_gpu} quantization_device={quantization_device} return_device={return_device} device_map={shared.opts.device_map} offload_mode={shared.opts.diffusers_offload_mode} non_blocking={shared.opts.diffusers_offload_nonblocking} modules_to_not_convert={modules_to_not_convert} modules_dtype_dict={modules_dtype_dict}') return model diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index e88bfc8d2..8f260db78 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -274,7 +274,7 @@ class SDNQQuantizer(DiffusersQuantizer): if ( param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert) - and not any("*" in param and re.match(param.replace("*", ".*"), param_name) for param in self.modules_to_not_convert) + and not any("*" in param and re.match(param.replace(".*", "\\.*").replace("*", ".*"), param_name) for param in self.modules_to_not_convert) ): layer_class_name = get_module_from_name(model, param_name)[0].__class__.__name__ if layer_class_name in allowed_types: @@ -311,7 +311,7 @@ class SDNQQuantizer(DiffusersQuantizer): if ( param_name in value or any(param in split_param_name for param in value) - or any("*" in param and re.match(param.replace("*", ".*"), param_name) for param in value) + or any("*" in param and re.match(param.replace(".*", "\\.*").replace("*", ".*"), param_name) for param in value) ): key = key.lower() if key in {"8bit", "8bits"}: diff --git a/modules/shared.py b/modules/shared.py index 799a5cc04..0cffb049a 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -182,6 +182,7 @@ options_templates.update(options_section(("quantization", "Model Quantization"), "sdnq_quantize_mode": OptionInfo("auto", "Quantization mode", gr.Dropdown, {"choices": ["auto", "pre", "post"]}), "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": sdnq_quant_modes}), "sdnq_quantize_weights_mode_te": OptionInfo("Same as model", "Quantization type for Text Encoders", gr.Dropdown, {"choices": ['Same as model'] + sdnq_quant_modes}), + "sdnq_modules_to_not_convert": OptionInfo("", "Modules to not convert"), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize convolutional layers", gr.Checkbox), "sdnq_dequantize_compile": OptionInfo(devices.has_triton(), "Dequantize using torch.compile", gr.Checkbox), diff --git a/pipelines/model_qwen.py b/pipelines/model_qwen.py index b6e57c95f..5681634b5 100644 --- a/pipelines/model_qwen.py +++ b/pipelines/model_qwen.py @@ -30,7 +30,7 @@ def load_qwen(checkpoint_info, diffusers_load_config={}): # cls_name = nunchaku.pipeline.pipeline_qwenimage.NunchakuQwenImagePipeline # we dont need this if transformer is None: - transformer = generic.load_transformer(repo_id, cls_name=diffusers.QwenImageTransformer2DModel, load_config=diffusers_load_config, modules_dtype_dict={"minimum_6bit": ["pos_embed", "time_text_embed", "img_in", "txt_in", "norm_out", "transformer_blocks.0.img_mod.1.weight", "transformer_blocks.59.img_mod.1.weight"]}) + transformer = generic.load_transformer(repo_id, cls_name=diffusers.QwenImageTransformer2DModel, load_config=diffusers_load_config, modules_dtype_dict={"minimum_6bit": ["pos_embed", "time_text_embed", "img_in", "txt_in", "norm_out", "transformer_blocks.0.img_mod.1.weight"]}) repo_te = 'Qwen/Qwen-Image' # if 'Qwen-Lightning' in repo_id or 'Qwen-Image-Edit' in repo_id else repo_id text_encoder = generic.load_text_encoder(repo_te, cls_name=transformers.Qwen2_5_VLForConditionalGeneration, load_config=diffusers_load_config) From b127832c5d2798663f391ab7a8aba7bccf011b40 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 20 Aug 2025 04:49:16 +0300 Subject: [PATCH 4/4] cleanup --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cb0ba0ec..5f57556bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ A quick service release with several important hotfixes, but also adding support default is `rust` as new `xet` is known to cause issues - support for `flux.1-kontext` lora - support for `qwen-image` lora - - add `sdnq_modules_to_not_convert` option to ui settings for + - add `sdnq_modules_to_not_convert` option to ui settings - **UI** - new artwork for reference models in networks thanks @liutyi