diff --git a/modules/sdnq/quantizer.py b/modules/sdnq/quantizer.py index 78b4770b9..47f015e3e 100644 --- a/modules/sdnq/quantizer.py +++ b/modules/sdnq/quantizer.py @@ -225,7 +225,7 @@ def add_module_skip_keys(model, modules_to_not_convert: list[str] = None, module @devices.inference_context() -def sdnq_quantize_layer_weight(weight, layer_class_name=None, weights_dtype="int8", quantized_matmul_dtype=None, torch_dtype=None, group_size=0, svd_rank=32, svd_steps=8, use_svd=False, use_quantized_matmul=False, use_stochastic_rounding=False, dequantize_fp32=False, using_pre_calculated_svd=False, skip_sr=False, param_name=None): # pylint: disable=unused-argument +def sdnq_quantize_layer_weight(weight, layer_class_name=None, weights_dtype="int8", quantized_matmul_dtype=None, torch_dtype=None, group_size=0, svd_rank=32, svd_steps=8, use_svd=False, use_quantized_matmul=False, use_stochastic_rounding=False, dequantize_fp32=True, using_pre_calculated_svd=False, skip_sr=False, param_name=None): # pylint: disable=unused-argument num_of_groups = 1 is_conv_type = False is_conv_transpose_type = False @@ -413,7 +413,7 @@ def sdnq_quantize_layer_weight(weight, layer_class_name=None, weights_dtype="int @devices.inference_context() -def sdnq_quantize_layer_weight_dynamic(weight, layer_class_name=None, weights_dtype="int2", quantized_matmul_dtype=None, torch_dtype=None, group_size=0, svd_rank=32, svd_steps=8, dynamic_loss_threshold=1e-2, use_svd=False, use_quantized_matmul=False, use_dynamic_quantization=False, use_stochastic_rounding=False, dequantize_fp32=False, param_name=None): # pylint: disable=unused-argument +def sdnq_quantize_layer_weight_dynamic(weight, layer_class_name=None, weights_dtype="int2", quantized_matmul_dtype=None, torch_dtype=None, group_size=0, svd_rank=32, svd_steps=8, dynamic_loss_threshold=1e-2, use_svd=False, use_quantized_matmul=False, use_dynamic_quantization=False, use_stochastic_rounding=False, dequantize_fp32=True, param_name=None): # pylint: disable=unused-argument if torch_dtype is None: torch_dtype = weight.dtype if weight.dtype != torch.float64: @@ -465,7 +465,7 @@ def sdnq_quantize_layer_weight_dynamic(weight, layer_class_name=None, weights_dt @devices.inference_context() -def sdnq_quantize_layer(layer, weights_dtype="int8", quantized_matmul_dtype=None, torch_dtype=None, group_size=0, svd_rank=32, svd_steps=8, dynamic_loss_threshold=1e-2, use_svd=False, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, use_dynamic_quantization=False, use_stochastic_rounding=False, dequantize_fp32=False, non_blocking=False, modules_to_not_convert=None, modules_dtype_dict=None, quantization_device=None, return_device=None, param_name=None): # pylint: disable=unused-argument +def sdnq_quantize_layer(layer, weights_dtype="int8", quantized_matmul_dtype=None, torch_dtype=None, group_size=0, svd_rank=32, svd_steps=8, dynamic_loss_threshold=1e-2, use_svd=False, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, use_dynamic_quantization=False, use_stochastic_rounding=False, dequantize_fp32=True, non_blocking=False, modules_to_not_convert=None, modules_dtype_dict=None, quantization_device=None, return_device=None, param_name=None): # pylint: disable=unused-argument layer_class_name = layer.__class__.__name__ if layer_class_name in conv_transpose_types or layer_class_name in conv_types: if not quant_conv: @@ -549,7 +549,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", quantized_matmul_dtype=None @devices.inference_context() -def apply_sdnq_to_module(model, weights_dtype="int8", quantized_matmul_dtype=None, torch_dtype=None, group_size=0, svd_rank=32, svd_steps=8, dynamic_loss_threshold=1e-2, use_svd=False, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, use_dynamic_quantization=False, use_stochastic_rounding=False, dequantize_fp32=False, non_blocking=False, modules_to_not_convert: list[str] = None, modules_dtype_dict: dict[str, list[str]] = None, modules_quant_config: dict[str, dict] = None, quantization_device=None, return_device=None, full_param_name=""): # pylint: disable=unused-argument +def apply_sdnq_to_module(model, weights_dtype="int8", quantized_matmul_dtype=None, torch_dtype=None, group_size=0, svd_rank=32, svd_steps=8, dynamic_loss_threshold=1e-2, use_svd=False, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, use_dynamic_quantization=False, use_stochastic_rounding=False, dequantize_fp32=True, non_blocking=False, modules_to_not_convert: list[str] = None, modules_dtype_dict: dict[str, list[str]] = None, modules_quant_config: dict[str, dict] = None, quantization_device=None, return_device=None, full_param_name=""): # pylint: disable=unused-argument has_children = list(model.children()) if not has_children: return model, modules_to_not_convert, modules_dtype_dict @@ -642,7 +642,7 @@ def sdnq_post_load_quant( use_quantized_matmul_conv: bool = False, use_dynamic_quantization: bool = False, use_stochastic_rounding: bool = False, - dequantize_fp32: bool = False, + dequantize_fp32: bool = True, non_blocking: bool = False, add_skip_keys:bool = True, quantization_device: torch.device = None, @@ -1070,7 +1070,7 @@ class SDNQConfig(QuantizationConfigMixin): use_static_quantization: bool = True, use_dynamic_quantization: bool = False, use_stochastic_rounding: bool = False, - dequantize_fp32: bool = False, + dequantize_fp32: bool = True, non_blocking: bool = False, add_skip_keys: bool = True, quantization_device: torch.device = None, diff --git a/modules/ui_definitions.py b/modules/ui_definitions.py index e9ae1c015..2c55bbcf4 100644 --- a/modules/ui_definitions.py +++ b/modules/ui_definitions.py @@ -150,7 +150,7 @@ def create_settings(cmd_opts): "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1}), "sdnq_svd_rank": OptionInfo(32, "SVD rank size", gr.Slider, {"minimum": 1, "maximum": 512, "step": 1}), "sdnq_svd_steps": OptionInfo(8, "SVD steps", gr.Slider, {"minimum": 1, "maximum": 128, "step": 1}), - "sdnq_dynamic_loss_threshold": OptionInfo(1e-2, "Dynamic loss threshold", gr.Slider, {"minimum": 1e-4, "maximum": 1e-1, "step": 1e-4}), + "sdnq_dynamic_loss_threshold": OptionInfo(1e-2, "Dynamic loss threshold", gr.Slider, {"minimum": 1e-8, "maximum": 1, "step": 1e-4}), "sdnq_use_svd": OptionInfo(False, "Use SVD quantization", gr.Checkbox), "sdnq_use_dynamic_quantization": OptionInfo(False, "Use Dynamic quantization", gr.Checkbox), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize convolutional layers", gr.Checkbox), @@ -158,7 +158,7 @@ def create_settings(cmd_opts): "sdnq_use_quantized_matmul": OptionInfo(False, "Use quantized MatMul", gr.Checkbox), "sdnq_use_quantized_matmul_conv": OptionInfo(False, "Use quantized MatMul with conv", gr.Checkbox), "sdnq_quantize_with_gpu": OptionInfo(True, "Quantize using GPU", gr.Checkbox), - "sdnq_dequantize_fp32": OptionInfo(False, "Dequantize using full precision", gr.Checkbox), + "sdnq_dequantize_fp32": OptionInfo(True, "Dequantize using full precision", gr.Checkbox), "sdnq_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights in post mode", gr.Checkbox), "nunchaku_sep": OptionInfo("

Nunchaku Engine

", "", gr.HTML),