From 2ba64abcde9cfabdfac05bdc21947a47f6c279b5 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 14 Jun 2025 00:54:18 +0300 Subject: [PATCH] Cleanup --- modules/lora/lora_apply.py | 2 ++ modules/model_quant.py | 4 ++++ modules/sdnq/__init__.py | 20 +++++++++++++++----- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index 62e9cbdfa..205a64e96 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -155,6 +155,8 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, + quantize_with_gpu=shared.opts.sdnq_quantize_with_gpu, + dequantize_fp32=shared.opts.sdnq_dequantize_fp32, param_name=getattr(self, 'network_layer_name', None), ) self = self.to(device) diff --git a/modules/model_quant.py b/modules/model_quant.py index b90297ee6..29ae5dd7f 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -125,6 +125,8 @@ def create_sdnq_config(kwargs = None, allow_sdnq: bool = True, module: str = 'Mo quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, + quantize_with_gpu=shared.opts.sdnq_quantize_with_gpu, + dequantize_fp32=shared.opts.sdnq_dequantize_fp32, ) log.debug(f'Quantization: module="{module}" type=sdnq dtype={weights_dtype}') if kwargs is None: @@ -339,6 +341,8 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc=True): quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, + quantize_with_gpu=shared.opts.sdnq_quantize_with_gpu, + dequantize_fp32=shared.opts.sdnq_dequantize_fp32, param_name=op, ) model.quantization_method = 'SDNQ' diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index f7607fcd1..fff1a3126 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -14,7 +14,7 @@ from .dequantizer import dequantizer_dict from .forward import get_forward_func -def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, param_name=None, pre_mode=False): +def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, quantize_with_gpu=True, dequantize_fp32=False, param_name=None, pre_mode=False): layer_class_name = layer.__class__.__name__ if layer_class_name in allowed_types: is_conv_type = False @@ -111,20 +111,20 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz elif pre_mode: if shared.opts.device_map == "gpu": return_device = devices.device - elif shared.opts.sdnq_quantize_with_gpu: + elif quantize_with_gpu: return_device = devices.cpu else: return_device = layer.weight.device else: return_device = layer.weight.device if not pre_mode: - if shared.opts.sdnq_quantize_with_gpu: + if quantize_with_gpu: layer.weight.data = layer.weight.to(devices.device).to(dtype=torch.float32) else: layer.weight.data = layer.weight.to(dtype=torch.float32) layer.weight.data, scale, zero_point = quantize_weight(layer.weight, reduction_axes, weights_dtype) - if not shared.opts.sdnq_dequantize_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"] and not use_tensorwise_fp8_matmul): + if not dequantize_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"] and not use_tensorwise_fp8_matmul): scale = scale.to(torch_dtype) if zero_point is not None: zero_point = zero_point.to(torch_dtype) @@ -158,7 +158,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz return layer -def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, param_name=None): # pylint: disable=unused-argument +def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, quantize_with_gpu=True, dequantize_fp32=False, param_name=None): # pylint: disable=unused-argument has_children = list(model.children()) if not has_children: return model @@ -172,6 +172,8 @@ def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_si quant_conv=quant_conv, use_quantized_matmul=use_quantized_matmul, use_quantized_matmul_conv=use_quantized_matmul_conv, + quantize_with_gpu=quantize_with_gpu, + dequantize_fp32=dequantize_fp32, param_name=module_param_name, ) module = apply_sdnq_to_module( @@ -182,6 +184,8 @@ def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_si quant_conv=quant_conv, use_quantized_matmul=use_quantized_matmul, use_quantized_matmul_conv=use_quantized_matmul_conv, + quantize_with_gpu=quantize_with_gpu, + dequantize_fp32=dequantize_fp32, param_name=module_param_name, ) return model @@ -295,6 +299,8 @@ class SDNQQuantizer(DiffusersQuantizer): quant_conv=self.quantization_config.quant_conv, use_quantized_matmul=self.quantization_config.use_quantized_matmul, use_quantized_matmul_conv=self.quantization_config.use_quantized_matmul_conv, + quantize_with_gpu=self.quantization_config.quantize_with_gpu, + dequantize_fp32=self.quantization_config.dequantize_fp32, param_name=param_name, pre_mode=True, ) @@ -389,6 +395,8 @@ class SDNQConfig(QuantizationConfigMixin): quant_conv: bool = False, use_quantized_matmul: bool = False, use_quantized_matmul_conv: bool = False, + quantize_with_gpu: bool = True, + dequantize_fp32: bool = False, modules_to_not_convert: Optional[List[str]] = None, **kwargs, # pylint: disable=unused-argument ): @@ -398,6 +406,8 @@ class SDNQConfig(QuantizationConfigMixin): self.quant_conv = quant_conv self.use_quantized_matmul = use_quantized_matmul self.use_quantized_matmul_conv = use_quantized_matmul_conv + self.quantize_with_gpu = quantize_with_gpu, + self.dequantize_fp32 = dequantize_fp32, self.modules_to_not_convert = modules_to_not_convert self.post_init() self.is_integer = dtype_dict[self.weights_dtype]["is_integer"]