From c7fb5b1690fe685edbfdeacccc283986edc06e57 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 5 Jun 2025 13:24:02 +0300 Subject: [PATCH 01/63] SDNQ fix VAE quant --- modules/processing_vae.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 5a375adc2..2d3f8bb7a 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -125,10 +125,6 @@ def full_vae_decode(latents, model): model.vae.orig_dtype = model.vae.dtype model.vae = model.vae.to(dtype=torch.float32) latents = latents.to(devices.device) - if getattr(model.vae, "post_quant_conv", None) is not None: - latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) - else: - latents = latents.to(model.vae.dtype) # normalize latents latents_mean = model.vae.config.get("latents_mean", None) @@ -144,6 +140,11 @@ def full_vae_decode(latents, model): if shift_factor: latents = latents + shift_factor + if getattr(model.vae, "post_quant_conv", None) is not None and "VAE" not in shared.opts.sdnq_quantize_weights: + latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) + else: + latents = latents.to(model.vae.dtype) + log_debug(f'VAE config: {model.vae.config}') try: with devices.inference_context(): From e25890bb1d0d339a087c5ccea0ebd80442af79f4 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 5 Jun 2025 13:36:49 +0300 Subject: [PATCH 02/63] SDNQ INT8 matmul support for Conv2d --- modules/model_quant_sdnq.py | 103 +++++++++++++++++++++++++++++++++--- modules/shared.py | 3 +- 2 files changed, 99 insertions(+), 7 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 1281b05e4..dbf0cfa4f 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -57,21 +57,30 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if layer_class_name in conv_types: if not quant_conv: return layer + if dtype_dict[weights_dtype]["num_bits"] < 4: + weights_dtype = "uint4" is_conv_type = True reduction_axes = 1 output_channel_size, channel_size = layer.weight.shape[:2] + group_channel_size = channel_size // layer.groups use_quantized_matmul = False - if dtype_dict[weights_dtype]["num_bits"] < 4: - weights_dtype = "uint4" + if shared.opts.sdnq_use_quantized_matmul_conv: + use_quantized_matmul = dtype_dict[weights_dtype]["is_integer"] and weights_dtype in quantized_matmul_dtypes and group_channel_size >= 32 and output_channel_size >= 32 + #if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: + # use_quantized_matmul = output_channel_size % 16 == 0 and group_channel_size % 16 == 0 + # use_tensorwise_fp8_matmul = torch_version < 2.5 or devices.backend in {"cpu", "openvino"} or (devices.backend == "cuda" and sys.platform == "win32" and torch_version <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) + if use_quantized_matmul: + result_shape = layer.weight.shape + layer.weight.data = layer.weight.reshape(output_channel_size, -1) elif layer_class_name in conv_transpose_types: if not quant_conv: return layer + if dtype_dict[weights_dtype]["num_bits"] < 4: + weights_dtype = "uint4" is_conv_transpose_type = True reduction_axes = 0 channel_size, output_channel_size = layer.weight.shape[:2] use_quantized_matmul = False - if dtype_dict[weights_dtype]["num_bits"] < 4: - weights_dtype = "uint4" else: is_linear_type = True reduction_axes = -1 @@ -193,7 +202,10 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz else: layer.forward = quantized_linear_forward elif is_conv_type: - layer.forward = quantized_conv_forward + if use_quantized_matmul: + layer.forward = quantized_conv2d_forward_int8_matmul + else: + layer.forward = quantized_conv_forward elif is_conv_transpose_type: if layer_class_name.endswith("1d"): layer.forward = quantized_conv_transpose_1d_forward @@ -393,7 +405,7 @@ def quantize_fp8_matmul_input_tensorwise(input: torch.FloatTensor, scale: torch. return input, scale -def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]: +def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor, flatten: bool = True) -> Tuple[torch.ByteTensor, torch.FloatTensor]: input = input.flatten(0,-2).contiguous() input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 127) input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(torch.int8) @@ -454,6 +466,54 @@ def int8_matmul( return result +def conv2d_int8_matmul( + input: torch.FloatTensor, + weight: torch.ByteTensor, + bias: torch.FloatTensor, + scale: torch.FloatTensor, + result_shape: torch.Size, + compressed_weight_shape: torch.Size, + weights_dtype: str, + reversed_padding_repeated_twice: List[int], + padding_mode: str, groups: int, + stride_h: int, stride_w: int, + padding_h: int, padding_w: int, + dilation_h: int, dilation_w: int, +) -> torch.FloatTensor: + return_dtype = input.dtype + batch_size, _, H_in, W_in = input.shape + C_out, _, K_h, K_w = result_shape + W_out = (W_in + 2 * padding_w - dilation_w * (K_w - 1) - 1) // stride_w + 1 + H_out = (H_in + 2 * padding_h - dilation_h * (K_h - 1) - 1) // stride_h + 1 + mm_output_shape = (batch_size, H_out, W_out, C_out) + + if compressed_weight_shape is not None: + weight = unpack_int_symetric(weight, compressed_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) + if padding_mode != "zeros": + input = torch.nn.functional.pad(input, reversed_padding_repeated_twice, mode=padding_mode) + padding_h = padding_w = 0 + + input, scale = quantize_int8_matmul_input( + torch.nn.functional.unfold( + input, kernel_size=(K_h, K_w), padding=(padding_h, padding_w), stride=(stride_h, stride_w), dilation=(dilation_h, dilation_w) + ).transpose(1,2), + scale, + ) + + if groups == 1: + result = decompress_symmetric(torch._int_mm(input, weight), scale, return_dtype, mm_output_shape) + else: + weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) + input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) + result = [] + for i in range(groups): + result.append(torch._int_mm(input[i], weight[i])) + result = decompress_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) + if bias is not None: + result.add_(bias) + return result.permute(0,3,1,2) + + def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: if torch.numel(input) / input.shape[-1] < 32: return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) @@ -472,6 +532,36 @@ def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torc return int8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale, getattr(self.sdnq_decompressor, "compressed_weight_shape", None), self.sdnq_decompressor.weights_dtype) +def quantized_conv2d_forward_int8_matmul(self, input) -> torch.FloatTensor: + if isinstance(self.stride, int): + stride_h = stride_w = self.stride + else: + stride_h, stride_w = self.stride + + if isinstance(self.padding, int): + padding_h = padding_w = self.padding + else: + padding_h, padding_w = self.padding + + if isinstance(self.dilation, int): + dilation_h = dilation_w = self.dilation + else: + dilation_h, dilation_w = self.dilation + + return conv2d_int8_matmul( + input, self.weight, self.bias, + self.sdnq_decompressor.scale, + self.sdnq_decompressor.result_shape, + getattr(self.sdnq_decompressor, "compressed_weight_shape", None), + self.sdnq_decompressor.weights_dtype, + self._reversed_padding_repeated_twice, + self.padding_mode, self.groups, + stride_h, stride_w, + padding_h, padding_w, + dilation_h, dilation_w, + ) + + def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTensor: return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight), self.bias) @@ -840,6 +930,7 @@ if shared.opts.sdnq_decompress_compile: fp8_matmul = torch.compile(fp8_matmul, fullgraph=True) fp8_matmul_tensorwise = torch.compile(fp8_matmul_tensorwise, fullgraph=True) int8_matmul = torch.compile(int8_matmul, fullgraph=True) + conv2d_int8_matmul = torch.compile(conv2d_int8_matmul, fullgraph=True) except Exception as e: shared.log.warning(f"Quantization: type=sdnq Decompress using torch.compile is not available: {e}") decompress_asymmetric_compiled = decompress_asymmetric diff --git a/modules/shared.py b/modules/shared.py index a1c089c48..6fdd80ad3 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -521,10 +521,11 @@ options_templates.update(options_section(("quantization", "Quantization Settings "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "int6", "uint4", "float8_e4m3fn", "uint8", "uint6", "int4", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "int2", "uint2", "uint1"], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), - "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), "sdnq_decompress_compile": OptionInfo(devices.has_triton(), "Decompress using torch.compile", gr.Checkbox, {"visible": native}), "sdnq_use_quantized_matmul": OptionInfo(False, "Use Quantized MatMul", gr.Checkbox, {"visible": native}), + "sdnq_use_quantized_matmul_conv": OptionInfo(False, "Use Quantized MatMul with convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_quantize_with_gpu": OptionInfo(True, "Quantize with the GPU", gr.Checkbox, {"visible": native}), + "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), "sdnq_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights in post mode", gr.Checkbox, {"visible": native}), "bnb_quantization_sep": OptionInfo("

BitsAndBytes

", "", gr.HTML), From ad2a4ad616f43e318da3687f3ec2f421371814a7 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 5 Jun 2025 13:38:04 +0300 Subject: [PATCH 03/63] Cleanup --- modules/model_quant_sdnq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index dbf0cfa4f..b084df453 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -405,7 +405,7 @@ def quantize_fp8_matmul_input_tensorwise(input: torch.FloatTensor, scale: torch. return input, scale -def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor, flatten: bool = True) -> Tuple[torch.ByteTensor, torch.FloatTensor]: +def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]: input = input.flatten(0,-2).contiguous() input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 127) input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(torch.int8) From 9b55ffe449d14b3aed4fe9e272e4e83c962f5d3d Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 5 Jun 2025 14:17:59 +0300 Subject: [PATCH 04/63] SDNQ fix VAE x2 --- modules/processing_vae.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 2d3f8bb7a..38e2574cd 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -140,8 +140,14 @@ def full_vae_decode(latents, model): if shift_factor: latents = latents + shift_factor - if getattr(model.vae, "post_quant_conv", None) is not None and "VAE" not in shared.opts.sdnq_quantize_weights: - latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) + if getattr(model.vae, "post_quant_conv", None) is not None: + if hasattr(model.vae.post_quant_conv, "bias"): + latents = latents.to(model.vae.post_quant_conv.bias.dtype) + else: + if "VAE" in shared.opts.sdnq_quantize_weights: + latents = latents.to(devices.dtype_vae) + else: + latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) else: latents = latents.to(model.vae.dtype) From 1a005173381c939ceb196becdd48befc9cf17cca Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 5 Jun 2025 14:32:26 +0300 Subject: [PATCH 05/63] SDNQ FP8 matmul support for Conv2d --- modules/model_quant_sdnq.py | 194 +++++++++++++++++++++++++++++------- 1 file changed, 159 insertions(+), 35 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index b084df453..9c780cdf5 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -29,6 +29,7 @@ dtype_dict = { "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, } +use_tensorwise_fp8_matmul = torch_version < 2.5 or devices.backend in {"cpu", "openvino"} or (devices.backend == "cuda" and sys.platform == "win32" and torch_version <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) quantized_matmul_dtypes = ("int8", "int6", "int4", "int2", "float8_e4m3fn", "float8_e5m2") if devices.backend in {"cpu", "openvino"}: quantized_matmul_dtypes += ("float8_e4m3fnuz", "float8_e5m2fnuz") @@ -49,7 +50,6 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz is_conv_type = False is_conv_transpose_type = False is_linear_type = False - use_tensorwise_fp8_matmul = False result_shape = None if torch_dtype is None: torch_dtype = devices.dtype @@ -65,10 +65,9 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz group_channel_size = channel_size // layer.groups use_quantized_matmul = False if shared.opts.sdnq_use_quantized_matmul_conv: - use_quantized_matmul = dtype_dict[weights_dtype]["is_integer"] and weights_dtype in quantized_matmul_dtypes and group_channel_size >= 32 and output_channel_size >= 32 - #if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: - # use_quantized_matmul = output_channel_size % 16 == 0 and group_channel_size % 16 == 0 - # use_tensorwise_fp8_matmul = torch_version < 2.5 or devices.backend in {"cpu", "openvino"} or (devices.backend == "cuda" and sys.platform == "win32" and torch_version <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) + use_quantized_matmul = weights_dtype in quantized_matmul_dtypes and group_channel_size >= 32 and output_channel_size >= 32 + if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: + use_quantized_matmul = output_channel_size % 16 == 0 and group_channel_size % 16 == 0 if use_quantized_matmul: result_shape = layer.weight.shape layer.weight.data = layer.weight.reshape(output_channel_size, -1) @@ -89,7 +88,6 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz use_quantized_matmul = weights_dtype in quantized_matmul_dtypes and channel_size >= 32 and output_channel_size >= 32 if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: use_quantized_matmul = output_channel_size % 16 == 0 and channel_size % 16 == 0 - use_tensorwise_fp8_matmul = torch_version < 2.5 or devices.backend in {"cpu", "openvino"} or (devices.backend == "cuda" and sys.platform == "win32" and torch_version <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) if group_size == 0: if is_linear_type: @@ -203,7 +201,13 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz layer.forward = quantized_linear_forward elif is_conv_type: if use_quantized_matmul: - layer.forward = quantized_conv2d_forward_int8_matmul + if dtype_dict[weights_dtype]["is_integer"]: + layer.forward = quantized_conv2d_forward_int8_matmul + else: + if use_tensorwise_fp8_matmul: + layer.forward = quantized_conv2d_forward_fp8_matmul_tensorwise + else: + layer.forward = quantized_conv2d_forward_fp8_matmul else: layer.forward = quantized_conv_forward elif is_conv_transpose_type: @@ -466,6 +470,89 @@ def int8_matmul( return result +def conv2d_fp8_matmul( + input: torch.FloatTensor, + weight: torch.ByteTensor, + bias: torch.FloatTensor, + scale: torch.FloatTensor, + result_shape: torch.Size, + weights_dtype: str, + reversed_padding_repeated_twice: List[int], + padding_mode: str, groups: int, + stride_h: int, stride_w: int, + padding_h: int, padding_w: int, + dilation_h: int, dilation_w: int, +) -> torch.FloatTensor: + return_dtype = input.dtype + mm_output_shape, K_h, K_w = get_conv2d_shapes(input.shape, result_shape, stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w) + if padding_mode != "zeros": + input = torch.nn.functional.pad(input, reversed_padding_repeated_twice, mode=padding_mode) + padding_h = padding_w = 0 + + input, input_scale = quantize_fp8_matmul_input( + torch.nn.functional.unfold( + input, kernel_size=(K_h, K_w), padding=(padding_h, padding_w), stride=(stride_h, stride_w), dilation=(dilation_h, dilation_w) + ).transpose(1,2), + ) + + if groups == 1: + result = torch._scaled_mm(input, weight, scale_a=input_scale, scale_b=scale, bias=bias, out_dtype=return_dtype).reshape(mm_output_shape).permute(0,3,1,2) + else: + scale = scale.reshape(groups, 1, scale.shape[1] // groups) + input_scale = input_scale.reshape(groups, input_scale.shape[0] // groups, 1) + weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) + input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) + result = [] + for i in range(groups): + result.append(torch._scaled_mm(input[i], weight[i], scale_a=input_scale[i], scale_b=scale[i], bias=None, out_dtype=return_dtype)) + result = torch.cat(result, dim=-1).reshape(mm_output_shape) + if bias is not None: + result.add_(bias) + result = result.permute(0,3,1,2) + return result + + +def conv2d_fp8_matmul_tensorwise( + input: torch.FloatTensor, + weight: torch.ByteTensor, + bias: torch.FloatTensor, + scale: torch.FloatTensor, + result_shape: torch.Size, + weights_dtype: str, + reversed_padding_repeated_twice: List[int], + padding_mode: str, groups: int, + stride_h: int, stride_w: int, + padding_h: int, padding_w: int, + dilation_h: int, dilation_w: int, +) -> torch.FloatTensor: + return_dtype = input.dtype + mm_output_shape, K_h, K_w = get_conv2d_shapes(input.shape, result_shape, stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w) + if padding_mode != "zeros": + input = torch.nn.functional.pad(input, reversed_padding_repeated_twice, mode=padding_mode) + padding_h = padding_w = 0 + + input, scale = quantize_fp8_matmul_input_tensorwise( + torch.nn.functional.unfold( + input, kernel_size=(K_h, K_w), padding=(padding_h, padding_w), stride=(stride_h, stride_w), dilation=(dilation_h, dilation_w) + ).transpose(1,2), + scale, + ) + + dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) + if groups == 1: + result = decompress_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, mm_output_shape) + else: + weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) + input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) + result = [] + for i in range(groups): + result.append(torch._scaled_mm(input[i], weight[i], scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype)) + result = decompress_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) + if bias is not None: + result.add_(bias) + return result.permute(0,3,1,2) + + def conv2d_int8_matmul( input: torch.FloatTensor, weight: torch.ByteTensor, @@ -481,17 +568,12 @@ def conv2d_int8_matmul( dilation_h: int, dilation_w: int, ) -> torch.FloatTensor: return_dtype = input.dtype - batch_size, _, H_in, W_in = input.shape - C_out, _, K_h, K_w = result_shape - W_out = (W_in + 2 * padding_w - dilation_w * (K_w - 1) - 1) // stride_w + 1 - H_out = (H_in + 2 * padding_h - dilation_h * (K_h - 1) - 1) // stride_h + 1 - mm_output_shape = (batch_size, H_out, W_out, C_out) - - if compressed_weight_shape is not None: - weight = unpack_int_symetric(weight, compressed_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) + mm_output_shape, K_h, K_w = get_conv2d_shapes(input.shape, result_shape, stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w) if padding_mode != "zeros": input = torch.nn.functional.pad(input, reversed_padding_repeated_twice, mode=padding_mode) padding_h = padding_w = 0 + if compressed_weight_shape is not None: + weight = unpack_int_symetric(weight, compressed_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) input, scale = quantize_int8_matmul_input( torch.nn.functional.unfold( @@ -532,22 +614,66 @@ def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torc return int8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale, getattr(self.sdnq_decompressor, "compressed_weight_shape", None), self.sdnq_decompressor.weights_dtype) +def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTensor: + return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight), self.bias) + + +def get_conv2d_args(stride, padding, dilation): + if isinstance(stride, int): + stride_h = stride_w = stride + else: + stride_h, stride_w = stride + if isinstance(padding, int): + padding_h = padding_w = padding + else: + padding_h, padding_w = padding + if isinstance(dilation, int): + dilation_h = dilation_w = dilation + else: + dilation_h, dilation_w = dilation + return stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w + + +def get_conv2d_shapes(input_shape, result_shape, stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w): + batch_size, _, H_in, W_in = input_shape + C_out, _, K_h, K_w = result_shape + W_out = (W_in + 2 * padding_w - dilation_w * (K_w - 1) - 1) // stride_w + 1 + H_out = (H_in + 2 * padding_h - dilation_h * (K_h - 1) - 1) // stride_h + 1 + return (batch_size, H_out, W_out, C_out), K_h, K_w + + +def quantized_conv2d_forward_fp8_matmul(self, input) -> torch.FloatTensor: + stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w = get_conv2d_args(self.stride, self.padding, self.dilation) + return conv2d_fp8_matmul( + input, self.weight, self.bias, + self.sdnq_decompressor.scale, + self.sdnq_decompressor.result_shape, + self.sdnq_decompressor.weights_dtype, + self._reversed_padding_repeated_twice, + self.padding_mode, self.groups, + stride_h, stride_w, + padding_h, padding_w, + dilation_h, dilation_w, + ) + + +def quantized_conv2d_forward_fp8_matmul_tensorwise(self, input) -> torch.FloatTensor: + stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w = get_conv2d_args(self.stride, self.padding, self.dilation) + return conv2d_fp8_matmul_tensorwise( + input, self.weight, self.bias, + self.sdnq_decompressor.scale, + self.sdnq_decompressor.result_shape, + self.sdnq_decompressor.weights_dtype, + self._reversed_padding_repeated_twice, + self.padding_mode, self.groups, + stride_h, stride_w, + padding_h, padding_w, + dilation_h, dilation_w, + ) + + def quantized_conv2d_forward_int8_matmul(self, input) -> torch.FloatTensor: - if isinstance(self.stride, int): - stride_h = stride_w = self.stride - else: - stride_h, stride_w = self.stride - - if isinstance(self.padding, int): - padding_h = padding_w = self.padding - else: - padding_h, padding_w = self.padding - - if isinstance(self.dilation, int): - dilation_h = dilation_w = self.dilation - else: - dilation_h, dilation_w = self.dilation - + stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w = get_conv2d_args(self.stride, self.padding, self.dilation) return conv2d_int8_matmul( input, self.weight, self.bias, self.sdnq_decompressor.scale, @@ -562,10 +688,6 @@ def quantized_conv2d_forward_int8_matmul(self, input) -> torch.FloatTensor: ) -def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTensor: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight), self.bias) - - def quantized_conv_forward(self, input) -> torch.FloatTensor: return self._conv_forward(input, self.sdnq_decompressor(self.weight), self.bias) @@ -927,10 +1049,12 @@ if shared.opts.sdnq_decompress_compile: decompress_symmetric_compiled = torch.compile(decompress_symmetric, fullgraph=True) decompress_packed_int_asymmetric_compiled = torch.compile(decompress_packed_int_asymmetric, fullgraph=True) decompress_packed_int_symmetric_compiled = torch.compile(decompress_packed_int_symmetric, fullgraph=True) + int8_matmul = torch.compile(int8_matmul, fullgraph=True) fp8_matmul = torch.compile(fp8_matmul, fullgraph=True) fp8_matmul_tensorwise = torch.compile(fp8_matmul_tensorwise, fullgraph=True) - int8_matmul = torch.compile(int8_matmul, fullgraph=True) conv2d_int8_matmul = torch.compile(conv2d_int8_matmul, fullgraph=True) + conv2d_fp8_matmul = torch.compile(conv2d_fp8_matmul, fullgraph=True) + conv2d_fp8_matmul_tensorwise = torch.compile(conv2d_fp8_matmul_tensorwise, fullgraph=True) except Exception as e: shared.log.warning(f"Quantization: type=sdnq Decompress using torch.compile is not available: {e}") decompress_asymmetric_compiled = decompress_asymmetric From 8c03f781977fc32063c267f533205f8377434d95 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 5 Jun 2025 14:37:00 +0300 Subject: [PATCH 06/63] Fix bias is None --- modules/processing_vae.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 38e2574cd..15831a5a4 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -141,7 +141,7 @@ def full_vae_decode(latents, model): latents = latents + shift_factor if getattr(model.vae, "post_quant_conv", None) is not None: - if hasattr(model.vae.post_quant_conv, "bias"): + if getattr(model.vae.post_quant_conv, "bias", None) is not None: latents = latents.to(model.vae.post_quant_conv.bias.dtype) else: if "VAE" in shared.opts.sdnq_quantize_weights: From 413cf54cb63a6eb8be014f3e5465055b575a357b Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 5 Jun 2025 18:11:36 +0300 Subject: [PATCH 07/63] Update changelog --- CHANGELOG.md | 9 ++++++++- modules/processing_vae.py | 7 +++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 968bb8e1c..880f16846 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Change Log for SD.Next +## Update for 2025-06-05 + +- **SDNQ Quantization** + - Add group size support for convolutional layers + - Add quantized matmul support for for convolutional layers + - Fix VAE with conv quant + + ## Update for 2025-06-02 ### Highlights for 2025-06-02 @@ -31,7 +39,6 @@ Take a look at [Docs](https://github.com/vladmandic/sdnext/wiki/Docs), [Hints](h - `INT4` -> `uint4` - Add `float8_e4m3fn`, `float8_e5m2`, `float8_e4m3fnuz`, `float8_e5m2fnuz`, `int6`, `uint6`, `int2`, `uint2` and `uint1` support - Add quantized matmul support for `float8_e4m3fn` and `float8_e5m2` - - Add group size support for convolutional layers - Set the default quant mode to `pre` - Use per token input quant with int8 and fp8 quantized matmul - Implement better layer hijacks diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 15831a5a4..b8e5dc4ae 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -143,11 +143,10 @@ def full_vae_decode(latents, model): if getattr(model.vae, "post_quant_conv", None) is not None: if getattr(model.vae.post_quant_conv, "bias", None) is not None: latents = latents.to(model.vae.post_quant_conv.bias.dtype) + elif "VAE" in shared.opts.sdnq_quantize_weights: + latents = latents.to(devices.dtype_vae) else: - if "VAE" in shared.opts.sdnq_quantize_weights: - latents = latents.to(devices.dtype_vae) - else: - latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) + latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) else: latents = latents.to(model.vae.dtype) From 778ca0436b941a3c2c88d528d5a3f8707f208ba7 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 5 Jun 2025 19:26:22 +0300 Subject: [PATCH 08/63] Update wiki --- wiki | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki b/wiki index 693b0dafa..ed3ead9cb 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 693b0dafa0dea17fe8057201f397e80a9b46b0c0 +Subproject commit ed3ead9cba0b3dc7b6d7162d63444193c897781a From 6bcd335f3759ef2f482aee00a8de5781f8c53a78 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 5 Jun 2025 20:17:46 +0300 Subject: [PATCH 09/63] Update changelog and wiki --- CHANGELOG.md | 2 ++ wiki | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 880f16846..663195bed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ - **SDNQ Quantization** - Add group size support for convolutional layers - Add quantized matmul support for for convolutional layers + - Fix forced FP32 with tensorwise FP8 matmul + - Fix PyTorch <= 2.4 compatibility with FP8 matmul - Fix VAE with conv quant diff --git a/wiki b/wiki index ed3ead9cb..c798cbab8 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit ed3ead9cba0b3dc7b6d7162d63444193c897781a +Subproject commit c798cbab895573c8c94da2e9ee3e03b7fc40678f From 976f0ba61f4a3959b5a9d2919b1f2fd869eb1f27 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 5 Jun 2025 20:59:58 +0300 Subject: [PATCH 10/63] Cleanup --- modules/lora/lora_apply.py | 11 ++++++++++- modules/model_quant.py | 2 ++ modules/model_quant_sdnq.py | 11 ++++++++--- wiki | 2 +- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index 7edc9f461..a5bfd8194 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -147,7 +147,16 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G new_weight = dequant_weight.to(devices.device, dtype=torch.float32) + lora_weights.to(devices.device, dtype=torch.float32) self.weight = torch.nn.Parameter(new_weight, requires_grad=False) self.sdnq_decompressor = None - self = sdnq_quantize_layer(self, sdnq_decompressor.weights_dtype, torch_dtype=devices.dtype, group_size=shared.opts.sdnq_quantize_weights_group_size, quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, param_name=getattr(self, 'network_layer_name', None)) + self = sdnq_quantize_layer( + self, + sdnq_decompressor.weights_dtype, + torch_dtype=devices.dtype, + group_size=shared.opts.sdnq_quantize_weights_group_size, + quant_conv=shared.opts.sdnq_quantize_conv_layers, + use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, + use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, + param_name=getattr(self, 'network_layer_name', None), + ) self = self.to(device) weight = None del dequant_weight diff --git a/modules/model_quant.py b/modules/model_quant.py index 96579c4eb..099c175bc 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -118,6 +118,7 @@ def create_sdnq_config(kwargs = None, allow_sdnq: bool = True, module: str = 'Mo group_size=shared.opts.sdnq_quantize_weights_group_size, quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, + use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, ) log.debug(f'Quantization: module="{module}" type=sdnq dtype={shared.opts.sdnq_quantize_weights_mode}') if kwargs is None: @@ -326,6 +327,7 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc=True): group_size=shared.opts.sdnq_quantize_weights_group_size, quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, + use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, param_name=op, ) model.quantization_method = 'SDNQ' diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 9c780cdf5..2f741ca91 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -44,7 +44,7 @@ class QuantizationMethod(str, Enum): SDNQ = "sdnq" -def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, param_name=None, pre_mode=False): # pylint: disable=unused-argument +def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, param_name=None, pre_mode=False): layer_class_name = layer.__class__.__name__ if layer_class_name in allowed_types: is_conv_type = False @@ -64,7 +64,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz output_channel_size, channel_size = layer.weight.shape[:2] group_channel_size = channel_size // layer.groups use_quantized_matmul = False - if shared.opts.sdnq_use_quantized_matmul_conv: + if use_quantized_matmul_conv: use_quantized_matmul = weights_dtype in quantized_matmul_dtypes and group_channel_size >= 32 and output_channel_size >= 32 if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: use_quantized_matmul = output_channel_size % 16 == 0 and group_channel_size % 16 == 0 @@ -222,7 +222,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz return layer -def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, param_name=None): +def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, param_name=None): has_children = list(model.children()) if not has_children: return model @@ -235,6 +235,7 @@ def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_si group_size=group_size, quant_conv=quant_conv, use_quantized_matmul=use_quantized_matmul, + use_quantized_matmul_conv=use_quantized_matmul_conv, param_name=module_param_name, ) module = apply_sdnq_to_module( @@ -244,6 +245,7 @@ def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_si group_size=group_size, quant_conv=quant_conv, use_quantized_matmul=use_quantized_matmul, + use_quantized_matmul_conv=use_quantized_matmul_conv, param_name=module_param_name, ) return model @@ -907,6 +909,7 @@ class SDNQQuantizer(DiffusersQuantizer): group_size=self.quantization_config.group_size, quant_conv=self.quantization_config.quant_conv, use_quantized_matmul=self.quantization_config.use_quantized_matmul, + use_quantized_matmul_conv=self.quantization_config.use_quantized_matmul_conv, param_name=param_name, pre_mode=True, ) @@ -1001,6 +1004,7 @@ class SDNQConfig(QuantizationConfigMixin): group_size: int = 0, quant_conv: bool = False, use_quantized_matmul: bool = False, + use_quantized_matmul_conv: bool = False, modules_to_not_convert: Optional[List[str]] = None, **kwargs, # pylint: disable=unused-argument ): @@ -1009,6 +1013,7 @@ class SDNQConfig(QuantizationConfigMixin): self.group_size = group_size self.quant_conv = quant_conv self.use_quantized_matmul = use_quantized_matmul + self.use_quantized_matmul_conv = use_quantized_matmul_conv self.modules_to_not_convert = modules_to_not_convert self.post_init() self.is_integer = dtype_dict[self.weights_dtype]["is_integer"] diff --git a/wiki b/wiki index c798cbab8..d6fce6bde 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit c798cbab895573c8c94da2e9ee3e03b7fc40678f +Subproject commit d6fce6bde637f71a5703dcd6dff348d32fd2992a From 06fcc3cf85ac3e7992b9574a4f67767314927712 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 6 Jun 2025 00:19:54 +0300 Subject: [PATCH 11/63] SDNQ add quantized matmul support for Conv1d and Conv3d too --- modules/model_quant_sdnq.py | 224 ++++++++++++++++++++---------------- wiki | 2 +- 2 files changed, 128 insertions(+), 98 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 2f741ca91..b6d61c9bf 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -202,12 +202,12 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz elif is_conv_type: if use_quantized_matmul: if dtype_dict[weights_dtype]["is_integer"]: - layer.forward = quantized_conv2d_forward_int8_matmul + layer.forward = quantized_conv_forward_int8_matmul else: if use_tensorwise_fp8_matmul: - layer.forward = quantized_conv2d_forward_fp8_matmul_tensorwise + layer.forward = quantized_conv_forward_fp8_matmul_tensorwise else: - layer.forward = quantized_conv2d_forward_fp8_matmul + layer.forward = quantized_conv_forward_fp8_matmul else: layer.forward = quantized_conv_forward elif is_conv_transpose_type: @@ -472,7 +472,56 @@ def int8_matmul( return result -def conv2d_fp8_matmul( +def process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation): + if conv_type == 1: + batch_size, _, L_in = input.shape + C_out, _, K_l = result_shape + L_out = (L_in + 2 * padding[1] - dilation[1] * (K_l - 1) - 1) // stride[1] + 1 + mm_output_shape = (batch_size, L_out, C_out) + kernel_size = (1, K_l) + if conv_type == 2: + batch_size, _, H_in, W_in = input.shape + C_out, _, K_h, K_w = result_shape + H_out = (H_in + 2 * padding[0] - dilation[0] * (K_h - 1) - 1) // stride[0] + 1 + W_out = (W_in + 2 * padding[1] - dilation[1] * (K_w - 1) - 1) // stride[1] + 1 + mm_output_shape = (batch_size, H_out, W_out, C_out) + kernel_size = (K_h, K_w) + elif conv_type == 3: + batch_size, _, D_in, H_in, W_in = input.shape + C_out, _, K_d, K_h, K_w = result_shape + D_out = (D_in + 2 * padding[0] - dilation[0] * (K_d - 1) - 1) // stride[0] + 1 + H_out = (H_in + 2 * padding[1] - dilation[1] * (K_h - 1) - 1) // stride[1] + 1 + W_out = (W_in + 2 * padding[2] - dilation[2] * (K_w - 1) - 1) // stride[2] + 1 + mm_output_shape = (batch_size, D_out, H_out, W_out, C_out) + kernel_size = (K_d, K_h, K_w) + + if padding_mode != "zeros": + input = torch.nn.functional.pad(input, reversed_padding_repeated_twice, mode=padding_mode) + padding = (0,) * (conv_type if conv_type != 1 else 2) + elif conv_type == 3: + input = torch.nn.functional.pad(input, reversed_padding_repeated_twice) + + if conv_type == 1: + input = input.unsqueeze(2) + + if conv_type == 3: + K_D_eff = kernel_size[0] + (kernel_size[0] - 1) * (dilation[0] - 1) + K_H_eff = kernel_size[1] + (kernel_size[1] - 1) * (dilation[0] - 1) + K_W_eff = kernel_size[2] + (kernel_size[2] - 1) * (dilation[0] - 1) + input = input.unfold(2, K_D_eff, stride[0]).unfold(3, K_H_eff, stride[1]).unfold(4, K_W_eff, stride[2]) + if dilation[0] > 1: + input = input[..., ::dilation[0], :, :] + if dilation[1] > 1: + input = input[..., ::dilation[1], :] + if dilation[2] > 1: + input = input[..., ::dilation[2]] + input = input.permute(0, 2, 3, 4, 1, 5, 6, 7).reshape(mm_output_shape[0], mm_output_shape[1] * mm_output_shape[2] * mm_output_shape[3], -1) + else: + input = torch.nn.functional.unfold(input, kernel_size=kernel_size, padding=padding, stride=stride, dilation=dilation).transpose(1,2) + return input, mm_output_shape + + +def conv_fp8_matmul( input: torch.FloatTensor, weight: torch.ByteTensor, bias: torch.FloatTensor, @@ -480,25 +529,16 @@ def conv2d_fp8_matmul( result_shape: torch.Size, weights_dtype: str, reversed_padding_repeated_twice: List[int], - padding_mode: str, groups: int, - stride_h: int, stride_w: int, - padding_h: int, padding_w: int, - dilation_h: int, dilation_w: int, + padding_mode: str, conv_type: int, + groups: int, stride: List[int], + padding: List[int], dilation: List[int], ) -> torch.FloatTensor: return_dtype = input.dtype - mm_output_shape, K_h, K_w = get_conv2d_shapes(input.shape, result_shape, stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w) - if padding_mode != "zeros": - input = torch.nn.functional.pad(input, reversed_padding_repeated_twice, mode=padding_mode) - padding_h = padding_w = 0 - - input, input_scale = quantize_fp8_matmul_input( - torch.nn.functional.unfold( - input, kernel_size=(K_h, K_w), padding=(padding_h, padding_w), stride=(stride_h, stride_w), dilation=(dilation_h, dilation_w) - ).transpose(1,2), - ) + input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) + input, input_scale = quantize_fp8_matmul_input(input) if groups == 1: - result = torch._scaled_mm(input, weight, scale_a=input_scale, scale_b=scale, bias=bias, out_dtype=return_dtype).reshape(mm_output_shape).permute(0,3,1,2) + result = torch._scaled_mm(input, weight, scale_a=input_scale, scale_b=scale, bias=bias, out_dtype=return_dtype).reshape(mm_output_shape) else: scale = scale.reshape(groups, 1, scale.shape[1] // groups) input_scale = input_scale.reshape(groups, input_scale.shape[0] // groups, 1) @@ -510,11 +550,17 @@ def conv2d_fp8_matmul( result = torch.cat(result, dim=-1).reshape(mm_output_shape) if bias is not None: result.add_(bias) + + if conv_type == 1: + result = result.transpose(1,2) + elif conv_type == 2: result = result.permute(0,3,1,2) + elif conv_type == 3: + result = result.permute(0,4,1,2,3) return result -def conv2d_fp8_matmul_tensorwise( +def conv_fp8_matmul_tensorwise( input: torch.FloatTensor, weight: torch.ByteTensor, bias: torch.FloatTensor, @@ -522,25 +568,15 @@ def conv2d_fp8_matmul_tensorwise( result_shape: torch.Size, weights_dtype: str, reversed_padding_repeated_twice: List[int], - padding_mode: str, groups: int, - stride_h: int, stride_w: int, - padding_h: int, padding_w: int, - dilation_h: int, dilation_w: int, + padding_mode: str, conv_type: int, + groups: int, stride: List[int], + padding: List[int], dilation: List[int], ) -> torch.FloatTensor: return_dtype = input.dtype - mm_output_shape, K_h, K_w = get_conv2d_shapes(input.shape, result_shape, stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w) - if padding_mode != "zeros": - input = torch.nn.functional.pad(input, reversed_padding_repeated_twice, mode=padding_mode) - padding_h = padding_w = 0 - - input, scale = quantize_fp8_matmul_input_tensorwise( - torch.nn.functional.unfold( - input, kernel_size=(K_h, K_w), padding=(padding_h, padding_w), stride=(stride_h, stride_w), dilation=(dilation_h, dilation_w) - ).transpose(1,2), - scale, - ) - + input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) + input, scale = quantize_fp8_matmul_input_tensorwise(input, scale) dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) + if groups == 1: result = decompress_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, mm_output_shape) else: @@ -552,10 +588,17 @@ def conv2d_fp8_matmul_tensorwise( result = decompress_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) if bias is not None: result.add_(bias) - return result.permute(0,3,1,2) + + if conv_type == 1: + result = result.transpose(1,2) + elif conv_type == 2: + result = result.permute(0,3,1,2) + elif conv_type == 3: + result = result.permute(0,4,1,2,3) + return result -def conv2d_int8_matmul( +def conv_int8_matmul( input: torch.FloatTensor, weight: torch.ByteTensor, bias: torch.FloatTensor, @@ -564,26 +607,16 @@ def conv2d_int8_matmul( compressed_weight_shape: torch.Size, weights_dtype: str, reversed_padding_repeated_twice: List[int], - padding_mode: str, groups: int, - stride_h: int, stride_w: int, - padding_h: int, padding_w: int, - dilation_h: int, dilation_w: int, + padding_mode: str, conv_type: int, + groups: int, stride: List[int], + padding: List[int], dilation: List[int], ) -> torch.FloatTensor: return_dtype = input.dtype - mm_output_shape, K_h, K_w = get_conv2d_shapes(input.shape, result_shape, stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w) - if padding_mode != "zeros": - input = torch.nn.functional.pad(input, reversed_padding_repeated_twice, mode=padding_mode) - padding_h = padding_w = 0 + input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) + input, scale = quantize_int8_matmul_input(input, scale) if compressed_weight_shape is not None: weight = unpack_int_symetric(weight, compressed_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) - input, scale = quantize_int8_matmul_input( - torch.nn.functional.unfold( - input, kernel_size=(K_h, K_w), padding=(padding_h, padding_w), stride=(stride_h, stride_w), dilation=(dilation_h, dilation_w) - ).transpose(1,2), - scale, - ) - if groups == 1: result = decompress_symmetric(torch._int_mm(input, weight), scale, return_dtype, mm_output_shape) else: @@ -595,7 +628,14 @@ def conv2d_int8_matmul( result = decompress_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) if bias is not None: result.add_(bias) - return result.permute(0,3,1,2) + + if conv_type == 1: + result = result.transpose(1,2) + elif conv_type == 2: + result = result.permute(0,3,1,2) + elif conv_type == 3: + result = result.permute(0,4,1,2,3) + return result def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: @@ -620,73 +660,63 @@ def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTenso return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight), self.bias) -def get_conv2d_args(stride, padding, dilation): +def get_conv_args(input_ndim, stride, padding, dilation): + if input_ndim == 3: + conv_type = 1 + elif input_ndim == 4: + conv_type = 2 + elif input_ndim == 5: + conv_type = 3 if isinstance(stride, int): - stride_h = stride_w = stride - else: - stride_h, stride_w = stride + stride = (stride,) * conv_type if isinstance(padding, int): - padding_h = padding_w = padding - else: - padding_h, padding_w = padding + padding = (padding,) * conv_type if isinstance(dilation, int): - dilation_h = dilation_w = dilation - else: - dilation_h, dilation_w = dilation - return stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w + dilation = (dilation,) * conv_type + if conv_type == 1: + stride = (1, stride[0]) + padding = (0, padding[0]) + dilation = (1, dilation[0]) + return conv_type, stride, padding, dilation -def get_conv2d_shapes(input_shape, result_shape, stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w): - batch_size, _, H_in, W_in = input_shape - C_out, _, K_h, K_w = result_shape - W_out = (W_in + 2 * padding_w - dilation_w * (K_w - 1) - 1) // stride_w + 1 - H_out = (H_in + 2 * padding_h - dilation_h * (K_h - 1) - 1) // stride_h + 1 - return (batch_size, H_out, W_out, C_out), K_h, K_w - - -def quantized_conv2d_forward_fp8_matmul(self, input) -> torch.FloatTensor: - stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w = get_conv2d_args(self.stride, self.padding, self.dilation) - return conv2d_fp8_matmul( +def quantized_conv_forward_fp8_matmul(self, input) -> torch.FloatTensor: + conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) + return conv_fp8_matmul( input, self.weight, self.bias, self.sdnq_decompressor.scale, self.sdnq_decompressor.result_shape, self.sdnq_decompressor.weights_dtype, self._reversed_padding_repeated_twice, - self.padding_mode, self.groups, - stride_h, stride_w, - padding_h, padding_w, - dilation_h, dilation_w, + self.padding_mode, conv_type, + self.groups, stride, padding, dilation, ) -def quantized_conv2d_forward_fp8_matmul_tensorwise(self, input) -> torch.FloatTensor: - stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w = get_conv2d_args(self.stride, self.padding, self.dilation) - return conv2d_fp8_matmul_tensorwise( +def quantized_conv_forward_fp8_matmul_tensorwise(self, input) -> torch.FloatTensor: + conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) + return conv_fp8_matmul_tensorwise( input, self.weight, self.bias, self.sdnq_decompressor.scale, self.sdnq_decompressor.result_shape, self.sdnq_decompressor.weights_dtype, self._reversed_padding_repeated_twice, - self.padding_mode, self.groups, - stride_h, stride_w, - padding_h, padding_w, - dilation_h, dilation_w, + self.padding_mode, conv_type, + self.groups, stride, padding, dilation, ) -def quantized_conv2d_forward_int8_matmul(self, input) -> torch.FloatTensor: - stride_h, stride_w, padding_h, padding_w, dilation_h, dilation_w = get_conv2d_args(self.stride, self.padding, self.dilation) - return conv2d_int8_matmul( +def quantized_conv_forward_int8_matmul(self, input) -> torch.FloatTensor: + conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) + return conv_int8_matmul( input, self.weight, self.bias, self.sdnq_decompressor.scale, self.sdnq_decompressor.result_shape, getattr(self.sdnq_decompressor, "compressed_weight_shape", None), self.sdnq_decompressor.weights_dtype, self._reversed_padding_repeated_twice, - self.padding_mode, self.groups, - stride_h, stride_w, - padding_h, padding_w, - dilation_h, dilation_w, + self.padding_mode, conv_type, + self.groups, stride, padding, dilation, ) @@ -1057,9 +1087,9 @@ if shared.opts.sdnq_decompress_compile: int8_matmul = torch.compile(int8_matmul, fullgraph=True) fp8_matmul = torch.compile(fp8_matmul, fullgraph=True) fp8_matmul_tensorwise = torch.compile(fp8_matmul_tensorwise, fullgraph=True) - conv2d_int8_matmul = torch.compile(conv2d_int8_matmul, fullgraph=True) - conv2d_fp8_matmul = torch.compile(conv2d_fp8_matmul, fullgraph=True) - conv2d_fp8_matmul_tensorwise = torch.compile(conv2d_fp8_matmul_tensorwise, fullgraph=True) + conv_int8_matmul = torch.compile(conv_int8_matmul, fullgraph=True) + conv_fp8_matmul = torch.compile(conv_fp8_matmul, fullgraph=True) + conv_fp8_matmul_tensorwise = torch.compile(conv_fp8_matmul_tensorwise, fullgraph=True) except Exception as e: shared.log.warning(f"Quantization: type=sdnq Decompress using torch.compile is not available: {e}") decompress_asymmetric_compiled = decompress_asymmetric diff --git a/wiki b/wiki index d6fce6bde..066397640 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit d6fce6bde637f71a5703dcd6dff348d32fd2992a +Subproject commit 0663976403988e9c8c5ff5a6e4c9da6f56e9b65a From 9a54efda9bd3d2bca5a398f179955609ec839c60 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 6 Jun 2025 01:55:35 +0300 Subject: [PATCH 12/63] Cleanup --- modules/model_quant_sdnq.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index b6d61c9bf..a17f8b3c1 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -505,9 +505,9 @@ def process_conv_input(conv_type, input, reversed_padding_repeated_twice, paddin input = input.unsqueeze(2) if conv_type == 3: - K_D_eff = kernel_size[0] + (kernel_size[0] - 1) * (dilation[0] - 1) - K_H_eff = kernel_size[1] + (kernel_size[1] - 1) * (dilation[0] - 1) - K_W_eff = kernel_size[2] + (kernel_size[2] - 1) * (dilation[0] - 1) + K_D_eff = K_d + (K_d - 1) * (dilation[0] - 1) + K_H_eff = K_h + (K_h - 1) * (dilation[0] - 1) + K_W_eff = K_w + (K_w - 1) * (dilation[0] - 1) input = input.unfold(2, K_D_eff, stride[0]).unfold(3, K_H_eff, stride[1]).unfold(4, K_W_eff, stride[2]) if dilation[0] > 1: input = input[..., ::dilation[0], :, :] @@ -515,7 +515,7 @@ def process_conv_input(conv_type, input, reversed_padding_repeated_twice, paddin input = input[..., ::dilation[1], :] if dilation[2] > 1: input = input[..., ::dilation[2]] - input = input.permute(0, 2, 3, 4, 1, 5, 6, 7).reshape(mm_output_shape[0], mm_output_shape[1] * mm_output_shape[2] * mm_output_shape[3], -1) + input = input.permute(0, 2, 3, 4, 1, 5, 6, 7).reshape(batch_size, D_out * H_out * W_out, -1) else: input = torch.nn.functional.unfold(input, kernel_size=kernel_size, padding=padding, stride=stride, dilation=dilation).transpose(1,2) return input, mm_output_shape @@ -681,6 +681,8 @@ def get_conv_args(input_ndim, stride, padding, dilation): def quantized_conv_forward_fp8_matmul(self, input) -> torch.FloatTensor: + if torch.numel(input) / input.shape[2] < 32: + return self._conv_forward(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) return conv_fp8_matmul( input, self.weight, self.bias, @@ -694,6 +696,8 @@ def quantized_conv_forward_fp8_matmul(self, input) -> torch.FloatTensor: def quantized_conv_forward_fp8_matmul_tensorwise(self, input) -> torch.FloatTensor: + if torch.numel(input) / input.shape[2] < 32: + return self._conv_forward(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) return conv_fp8_matmul_tensorwise( input, self.weight, self.bias, @@ -707,6 +711,8 @@ def quantized_conv_forward_fp8_matmul_tensorwise(self, input) -> torch.FloatTens def quantized_conv_forward_int8_matmul(self, input) -> torch.FloatTensor: + if torch.numel(input) / input.shape[2] < 32: + return self._conv_forward(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) return conv_int8_matmul( input, self.weight, self.bias, From b5d6b575004f750900552dde07f002dffb5fe808 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 6 Jun 2025 11:37:10 +0300 Subject: [PATCH 13/63] Update PyTorch to 2.7.1 --- CHANGELOG.md | 5 ++++- installer.py | 16 ++++++++-------- wiki | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 663195bed..c12e8192d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Change Log for SD.Next -## Update for 2025-06-05 +## Update for 2025-06-06 + +- **Torch**: + - set default to `torch==2.7.1` - **SDNQ Quantization** - Add group size support for convolutional layers diff --git a/installer.py b/installer.py index cecf6d0c6..d683c83ef 100644 --- a/installer.py +++ b/installer.py @@ -582,7 +582,7 @@ def install_cuda(): cmd = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu128 --extra-index-url https://download.pytorch.org/whl/nightly/cu126') else: # cmd = os.environ.get('TORCH_COMMAND', 'torch==2.6.0+cu126 torchvision==0.21.0+cu126 --index-url https://download.pytorch.org/whl/cu126') - cmd = os.environ.get('TORCH_COMMAND', 'torch==2.7.0+cu128 torchvision==0.22.0+cu128 --index-url https://download.pytorch.org/whl/cu128') + cmd = os.environ.get('TORCH_COMMAND', 'torch==2.7.1+cu128 torchvision==0.22.1+cu128 --index-url https://download.pytorch.org/whl/cu128') return cmd @@ -655,7 +655,7 @@ def install_rocm_zluda(): if error is None: try: zluda_installer.load() - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.0 torchvision --index-url https://download.pytorch.org/whl/cu118') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1 torchvision --index-url https://download.pytorch.org/whl/cu118') except Exception as e: error = e log.warning(f'Failed to load ZLUDA: {e}') @@ -675,10 +675,10 @@ def install_rocm_zluda(): torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm6.2.4') else: if rocm.version is None or float(rocm.version) >= 6.3: # assume the latest if version check fails - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.0+rocm6.3 torchvision==0.22.0+rocm6.3 --index-url https://download.pytorch.org/whl/rocm6.3') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1+rocm6.3 torchvision==0.22.1+rocm6.3 --index-url https://download.pytorch.org/whl/rocm6.3') elif rocm.version == "6.2": - # use rocm 6.2.4 instead of 6.2 as torch==2.7.0+rocm6.2 doesn't exists - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.0+rocm6.2.4 torchvision==0.22.0+rocm6.2.4 --index-url https://download.pytorch.org/whl/rocm6.2.4') + # use rocm 6.2.4 instead of 6.2 as torch==2.7.1+rocm6.2 doesn't exists + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1+rocm6.2.4 torchvision==0.22.1+rocm6.2.4 --index-url https://download.pytorch.org/whl/rocm6.2.4') elif rocm.version == "6.1": torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.6.0+rocm6.1 torchvision==0.21.0+rocm6.1 --index-url https://download.pytorch.org/whl/rocm6.1') elif rocm.version == "6.0": @@ -736,7 +736,7 @@ def install_ipex(torch_command): if args.use_nightly: torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/xpu') else: - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.0+xpu torchvision==0.22.0+xpu --index-url https://download.pytorch.org/whl/xpu') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1+xpu torchvision==0.22.1+xpu --index-url https://download.pytorch.org/whl/xpu') ts('ipex', t_start) return torch_command @@ -747,9 +747,9 @@ def install_openvino(torch_command): check_python(supported_minors=[9, 10, 11, 12], reason='OpenVINO backend requires a Python version between 3.9 and 3.12') log.info('OpenVINO: selected') if sys.platform == 'darwin': - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.0 torchvision==0.22.0') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1 torchvision==0.22.1') else: - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.0+cpu torchvision==0.22.0+cpu --index-url https://download.pytorch.org/whl/cpu') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1+cpu torchvision==0.22.1+cpu --index-url https://download.pytorch.org/whl/cpu') install(os.environ.get('OPENVINO_COMMAND', 'openvino==2025.1.0'), 'openvino') install(os.environ.get('NNCF_COMMAND', 'nncf==2.16.0'), 'nncf') diff --git a/wiki b/wiki index 066397640..336343b02 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 0663976403988e9c8c5ff5a6e4c9da6f56e9b65a +Subproject commit 336343b02341c849c54c9070ef8bd64e7462376f From 2ccc76ab91c3758fc65fe826113d4cce52ded4a8 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 6 Jun 2025 15:27:30 +0300 Subject: [PATCH 14/63] Increase medvram mode to 12 GB and update wiki --- modules/shared.py | 2 +- wiki | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index 6fdd80ad3..ffe783280 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -353,7 +353,7 @@ def get_default_modes(): default_offload_mode = "sequential" default_diffusers_offload_min_gpu_memory = 0 log.info(f"Device detect: memory={gpu_memory:.1f} default=sequential optimization=lowvram") - elif gpu_memory <= 8: + elif gpu_memory <= 12: cmd_opts.medvram = True # VAE Tiling and other stuff default_offload_mode = "balanced" default_diffusers_offload_min_gpu_memory = 0 diff --git a/wiki b/wiki index 336343b02..048798a26 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 336343b02341c849c54c9070ef8bd64e7462376f +Subproject commit 048798a26a7f86957f9bb5cb67846fded630bad5 From 7679028c1a516a5dcec8d31bd3ec0d0363c0d31a Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 6 Jun 2025 15:33:51 +0300 Subject: [PATCH 15/63] Override CPU to use FP32 by default --- CHANGELOG.md | 18 +++++++++++------- modules/devices.py | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c12e8192d..2af39866e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,15 +2,19 @@ ## Update for 2025-06-06 -- **Torch**: - - set default to `torch==2.7.1` +- **Changes** + - Increase the medvram mode threshold from 8GB to 12GB + - Set CPU backend to use FP32 by default + +- **Torch** + - set default to `torch==2.7.1` - **SDNQ Quantization** - - Add group size support for convolutional layers - - Add quantized matmul support for for convolutional layers - - Fix forced FP32 with tensorwise FP8 matmul - - Fix PyTorch <= 2.4 compatibility with FP8 matmul - - Fix VAE with conv quant + - Add group size support for convolutional layers + - Add quantized matmul support for for convolutional layers + - Fix forced FP32 with tensorwise FP8 matmul + - Fix PyTorch <= 2.4 compatibility with FP8 matmul + - Fix VAE with conv quant ## Update for 2025-06-02 diff --git a/modules/devices.py b/modules/devices.py index 8bf2def3f..2e8dc73ce 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -331,7 +331,7 @@ def test_fp16(): if fp16_ok is not None: return fp16_ok if opts.cuda_dtype != 'FP16': # don't override if the user sets it - if sys.platform == "darwin" or backend == 'openvino': # override + if sys.platform == "darwin" or backend in {'openvino', 'cpu'}: # override fp16_ok = False return fp16_ok elif backend == 'rocm': @@ -362,7 +362,7 @@ def test_bf16(): if bf16_ok is not None: return bf16_ok if opts.cuda_dtype != 'BF16': # don't override if the user sets it - if sys.platform == "darwin" or backend == 'openvino' or backend == 'directml': # override + if sys.platform == "darwin" or backend in {'openvino', 'directml', 'cpu'}: # override bf16_ok = False return bf16_ok elif backend == 'rocm' or backend == 'zluda': From c039ba90f6b9a05b62311a3fb145052e82608848 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 6 Jun 2025 18:02:49 +0300 Subject: [PATCH 16/63] Fix Meissonic by adding multiple generator support --- CHANGELOG.md | 2 ++ modules/meissonic/scheduler.py | 30 ++++++++++++++++++++---------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2af39866e..83a94e3a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ - Fix PyTorch <= 2.4 compatibility with FP8 matmul - Fix VAE with conv quant +- **Fixes** + - Meissonic with multiple generators ## Update for 2025-06-02 diff --git a/modules/meissonic/scheduler.py b/modules/meissonic/scheduler.py index 3d2fe4276..757469b4d 100644 --- a/modules/meissonic/scheduler.py +++ b/modules/meissonic/scheduler.py @@ -23,8 +23,12 @@ from diffusers.schedulers.scheduling_utils import SchedulerMixin def gumbel_noise(t, generator=None): - device = generator.device if generator is not None else t.device - noise = torch.zeros_like(t, device=device).uniform_(0, 1, generator=generator).to(t.device) + noise = [] + noise_shape = t.shape[1:] + for i in range(len(generator)): + device = generator[i].device if generator[i] is not None else t.device + noise.append(torch.zeros(noise_shape, device=device, dtype=t.dtype).uniform_(0, 1, generator=generator[i]).to(t.device)) + noise = torch.stack(noise, dim=0) return -torch.log((-torch.log(noise.clamp(1e-20))).clamp(1e-20)) @@ -100,14 +104,20 @@ class Scheduler(SchedulerMixin, ConfigMixin): unknown_map = sample == self.config.mask_token_id probs = model_output.softmax(dim=-1) - device = probs.device - probs_ = probs.to(generator.device) if generator is not None else probs # handles when generator is on CPU - if probs_.device.type == "cpu" and probs_.dtype != torch.float32: - probs_ = probs_.float() # multinomial is not implemented for cpu half precision - probs_ = probs_.reshape(-1, probs.size(-1)) - pred_original_sample = torch.multinomial(probs_, 1, generator=generator).to(device=device) - pred_original_sample = pred_original_sample[:, 0].view(*probs.shape[:-1]) + probs_view_shape = probs.shape[1:-1] + if not isinstance(generator, list): + generator = [generator] * probs.size(0) + elif isinstance(generator, list) and len(generator) == 1 and len(generator) != probs.size(0): + generator = generator * probs.size(0) + + pred_original_sample = [] + for i in range(len(generator)): + probs_ = probs[i].to(generator[i].device) if generator[i] is not None else probs[i] # handles when generator is on CPU + if probs_.device.type == "cpu" and probs_.dtype != torch.float32: + probs_ = probs_.float() # multinomial is not implemented for cpu half precision + pred_original_sample.append(torch.multinomial(probs_, 1, generator=generator[i]).to(device=device).view(*probs_view_shape)) + pred_original_sample = torch.stack(pred_original_sample, dim=0) pred_original_sample = torch.where(unknown_map, pred_original_sample, sample) if timestep == 0: @@ -163,7 +173,7 @@ class Scheduler(SchedulerMixin, ConfigMixin): mask_indices = ( torch.rand( - sample.shape, device=generator.device if generator is not None else sample.device, generator=generator + sample.shape, device=generator[0].device if generator[0] is not None else sample.device, generator=generator ).to(sample.device) < mask_ratio ) From 089e437708a02fa20df272cdc5271232745b2893 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 6 Jun 2025 18:53:57 +0300 Subject: [PATCH 17/63] Don't set attention processors with models outside of SD 1.5 and SDXL --- CHANGELOG.md | 1 + modules/sd_models.py | 22 +++------------------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83a94e3a3..80ed4f77a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - **Fixes** - Meissonic with multiple generators + - Kandinsky V2.2 invalid attention processor ## Update for 2025-06-02 diff --git a/modules/sd_models.py b/modules/sd_models.py index 22ba21a49..9faa12992 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -902,25 +902,9 @@ def set_diffusers_attention(pipe, quiet:bool=False): def set_attn(pipe, attention): if attention is None: return - if not hasattr(pipe, "_internal_dict"): - return - modules = [getattr(pipe, n, None) for n in pipe._internal_dict.keys()] # pylint: disable=protected-access - modules = [m for m in modules if isinstance(m, torch.nn.Module) and hasattr(m, "set_attn_processor")] - for module in modules: - if module.__class__.__name__ in ['SD3Transformer2DModel']: - module.set_attn_processor(p.JointAttnProcessor2_0()) - elif module.__class__.__name__ in ['FluxTransformer2DModel']: - module.set_attn_processor(p.FluxAttnProcessor2_0()) - elif module.__class__.__name__ in ['HunyuanDiT2DModel']: - module.set_attn_processor(p.HunyuanAttnProcessor2_0()) - elif module.__class__.__name__ in ['AuraFlowTransformer2DModel']: - module.set_attn_processor(p.AuraFlowAttnProcessor2_0()) - elif 'KandinskyCombinedPipeline' in pipe.__class__.__name__: - pass - elif 'Transformer' in module.__class__.__name__: - pass # unknown transformer so probably dont want to force attention processor - else: - module.set_attn_processor(attention) + # other models uses their own attention processor + if pipe.__class__.__name__.startswith("StableDiffusion") and hasattr(pipe, "unet"): + pipe.unet.set_attn_processor(attention) # if hasattr(pipe, 'pipe'): # set_diffusers_attention(pipe.pipe) From 56246711915552a1f7bf31e3d99a5508a2b360e3 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 6 Jun 2025 19:25:26 +0300 Subject: [PATCH 18/63] Fix PixArt Sigma Small and Large --- CHANGELOG.md | 1 + modules/model_pixart.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80ed4f77a..4b0b1d531 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - **Fixes** - Meissonic with multiple generators - Kandinsky V2.2 invalid attention processor + - PixArt Sigma Small and Large loading ## Update for 2025-06-02 diff --git a/modules/model_pixart.py b/modules/model_pixart.py index 58d3dbfa7..6f6d6cf1c 100644 --- a/modules/model_pixart.py +++ b/modules/model_pixart.py @@ -1,11 +1,19 @@ import transformers import diffusers +from huggingface_hub import file_exists def load_pixart(checkpoint_info, diffusers_load_config={}): from modules import shared, devices, modelloader, sd_models, model_quant modelloader.hf_login() repo_id = sd_models.path_to_repo(checkpoint_info.name) + repo_id_tenc = repo_id + repo_id_pipe = repo_id + + if not file_exists(repo_id_tenc, "text_encoder/config.json"): + repo_id_tenc = "PixArt-alpha/PixArt-Sigma-XL-2-1024-MS" + if not file_exists(repo_id_pipe, "model_index.json"): + repo_id_pipe = "PixArt-alpha/PixArt-Sigma-XL-2-1024-MS" load_args, quant_args = model_quant.get_dit_args(diffusers_load_config, module='Transformer') transformer = diffusers.PixArtTransformer2DModel.from_pretrained( @@ -17,7 +25,7 @@ def load_pixart(checkpoint_info, diffusers_load_config={}): ) load_args, quant_args = model_quant.get_dit_args(diffusers_load_config, module='TE', device_map=True) text_encoder = transformers.T5EncoderModel.from_pretrained( - repo_id, + repo_id_tenc, subfolder="text_encoder", cache_dir=shared.opts.hfcache_dir, **load_args, @@ -26,7 +34,7 @@ def load_pixart(checkpoint_info, diffusers_load_config={}): load_args, _quant_args = model_quant.get_dit_args(diffusers_load_config, allow_quant=False) pipe = diffusers.PixArtSigmaPipeline.from_pretrained( - 'PixArt-alpha/PixArt-Sigma-XL-2-1024-MS', + repo_id_pipe, cache_dir=shared.opts.diffusers_dir, transformer=transformer, text_encoder=text_encoder, From 2f7aff525035b4febbe6611aa9361decfc6a1681 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 6 Jun 2025 19:43:00 +0300 Subject: [PATCH 19/63] Fix TAESD previews with PixArt --- CHANGELOG.md | 1 + modules/modeldata.py | 4 ++++ modules/sd_vae_taesd.py | 10 +++++++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b0b1d531..12793fb72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Meissonic with multiple generators - Kandinsky V2.2 invalid attention processor - PixArt Sigma Small and Large loading + - TAESD previews with PixArt ## Update for 2025-06-02 diff --git a/modules/modeldata.py b/modules/modeldata.py index b7be0866b..688cff23d 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -56,6 +56,10 @@ def get_model_type(pipe): model_type = 'mochivideo' elif "Allegro" in name: model_type = 'allegrovideo' + elif "PixArtSigma" in name: + model_type = 'pixartsigma' + elif "PixArtAlpha" in name: + model_type = 'pixartalpha' else: model_type = name return model_type diff --git a/modules/sd_vae_taesd.py b/modules/sd_vae_taesd.py index da12422f1..7f6bf191b 100644 --- a/modules/sd_vae_taesd.py +++ b/modules/sd_vae_taesd.py @@ -36,7 +36,7 @@ prev_cls = '' prev_type = '' prev_model = '' lock = threading.Lock() -supported = ['sd', 'sdxl', 'f1', 'h1', 'hunyuanvideo', 'wanvideo', 'mochivideo'] +supported = ['sd', 'sdxl', 'f1', 'h1', 'hunyuanvideo', 'wanvideo', 'mochivideo', 'pixartsigma', 'pixartalpha'] def warn_once(msg, variant=None): @@ -55,9 +55,13 @@ def get_model(model_type = 'decoder', variant = None): cls = shared.sd_model_type if cls == 'ldm': # original backend cls = 'sd' - if cls == 'h1': # hidream uses flux vae + elif cls == 'h1': # hidream uses flux vae cls = 'f1' - if cls not in supported: + elif cls == 'pixartsigma': + cls = 'sdxl' + elif cls == 'pixartalpha': + cls = 'sd' + elif cls not in supported: warn_once(f'cls={shared.sd_model.__class__.__name__} type={cls} unsuppported', variant=variant) variant = variant or shared.opts.taesd_variant folder = os.path.join(paths.models_path, "TAESD") From 8e08ef0edc8e61627801523b901ff72252f37e57 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 7 Jun 2025 01:25:08 +0300 Subject: [PATCH 20/63] Fix VAE Tiling with non-default tile sizes --- CHANGELOG.md | 1 + modules/sd_models.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12793fb72..996813e50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Kandinsky V2.2 invalid attention processor - PixArt Sigma Small and Large loading - TAESD previews with PixArt + - VAE Tiling with non-default tile sizes ## Update for 2025-06-02 diff --git a/modules/sd_models.py b/modules/sd_models.py index 9faa12992..ce35d0aba 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -107,7 +107,7 @@ def set_vae_options(sd_model, vae=None, op:str='model', quiet:bool=False): if hasattr(sd_model, 'vae') and hasattr(sd_model.vae, 'config') and hasattr(sd_model.vae.config, 'sample_size') and isinstance(sd_model.vae.config.sample_size, int): if shared.opts.diffusers_vae_tile_size > 0: sd_model.vae.tile_sample_min_size = int(shared.opts.diffusers_vae_tile_size) - sd_model.vae.tile_latent_min_size = int(sd_model.vae.config.sample_size / (2 ** (len(sd_model.vae.config.block_out_channels) - 1))) + sd_model.vae.tile_latent_min_size = int(shared.opts.diffusers_vae_tile_size / (2 ** (len(sd_model.vae.config.block_out_channels) - 1))) if shared.opts.diffusers_vae_tile_overlap != 0.25: sd_model.vae.tile_overlap_factor = float(shared.opts.diffusers_vae_tile_overlap) shared.log.quiet(quiet, f'Setting {op}: component=VAE tiling=True tile={sd_model.vae.tile_sample_min_size} overlap={sd_model.vae.tile_overlap_factor}') From 92d23796263ce6651768aa92c4fa21b8b59fdfce Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 9 Jun 2025 20:18:03 +0300 Subject: [PATCH 21/63] Relax Python version check with Zluda --- installer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/installer.py b/installer.py index d683c83ef..f380e459b 100644 --- a/installer.py +++ b/installer.py @@ -633,7 +633,7 @@ def install_rocm_zluda(): log.info(msg) if sys.platform == "win32": # TODO install: enable ROCm for windows when available - check_python(supported_minors=[10, 11], reason='ZLUDA backend requires Python 3.10 or 3.11') + #check_python(supported_minors=[9, 10, 11, 12], reason='ZLUDA backend requires a Python version between 3.9 and 3.12') if args.device_id is not None: if os.environ.get('HIP_VISIBLE_DEVICES', None) is not None: @@ -663,7 +663,7 @@ def install_rocm_zluda(): log.info('Using CPU-only torch') torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') else: - check_python(supported_minors=[9, 10, 11, 12], reason='ROCm backend requires a Python version between 3.9 and 3.12') + #check_python(supported_minors=[9, 10, 11, 12], reason='ROCm backend requires a Python version between 3.9 and 3.12') if os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", None) is None: os.environ.setdefault('TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL', '1') @@ -711,7 +711,7 @@ def install_rocm_zluda(): def install_ipex(torch_command): t_start = time.time() - check_python(supported_minors=[9, 10, 11, 12], reason='IPEX backend requires a Python version between 3.9 and 3.12') + #check_python(supported_minors=[9, 10, 11, 12], reason='IPEX backend requires a Python version between 3.9 and 3.12') args.use_ipex = True # pylint: disable=attribute-defined-outside-init log.info('IPEX: Intel OneAPI toolkit detected') @@ -744,7 +744,7 @@ def install_ipex(torch_command): def install_openvino(torch_command): t_start = time.time() - check_python(supported_minors=[9, 10, 11, 12], reason='OpenVINO backend requires a Python version between 3.9 and 3.12') + #check_python(supported_minors=[9, 10, 11, 12], reason='OpenVINO backend requires a Python version between 3.9 and 3.12') log.info('OpenVINO: selected') if sys.platform == 'darwin': torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1 torchvision==0.22.1') From bd2d9d167788be730b8ddd8386b5580a85b7d44d Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 9 Jun 2025 22:58:08 +0300 Subject: [PATCH 22/63] Python 3.13 support --- installer.py | 16 +++++++++++++--- modules/postprocess/gfpgan_model.py | 4 ++-- modules/rocm.py | 2 +- repositories/codeformer/basicsr/__init__.py | 3 ++- repositories/codeformer/basicsr/losses/losses.py | 2 +- requirements.txt | 2 +- 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/installer.py b/installer.py index f380e459b..2d0a944f5 100644 --- a/installer.py +++ b/installer.py @@ -513,7 +513,7 @@ def get_platform(): def check_python(supported_minors=[], experimental_minors=[], reason=None): if supported_minors is None or len(supported_minors) == 0: supported_minors = [9, 10, 11, 12] - experimental_minors = [] + experimental_minors = [13] t_start = time.time() if args.quick: return @@ -1155,8 +1155,8 @@ def ensure_base_requirements(): def install_optional(): t_start = time.time() log.info('Installing optional requirements...') - install('basicsr') - install('gfpgan') + install('git+https://github.com/Disty0/BasicSR@2b6a12c28e0c81bfb13b7e984144f0b0f5461484', 'basicsr') + install('git+https://github.com/Disty0/GFPGAN@09b1190eabbc77e5f15c61fa7c38a2064b403e20', 'gfpgan') install('clean-fid') install('pillow-jxl-plugin==1.3.3', ignore=True) install('optimum-quanto==0.2.7', ignore=True) @@ -1188,6 +1188,16 @@ def install_requirements(): pr.enable() if args.skip_requirements and not args.requirements: return + if int(sys.version_info.minor) >= 13: + install("audioop-lts") + # gcc 15 patch + backup_cmake_policy = os.environ.get("CMAKE_POLICY_VERSION_MINIMUM", None) + backup_cxxflags = os.environ.get("CXXFLAGS", None) + os.environ.setdefault("CMAKE_POLICY_VERSION_MINIMUM", "3.5") + os.environ.setdefault("CXXFLAGS", "-include cstdint") + install("git+https://github.com/google/sentencepiece#subdirectory=python", "sentencepiece") + os.environ.setdefault("CMAKE_POLICY_VERSION_MINIMUM", backup_cmake_policy) + os.environ.setdefault("CXXFLAGS", backup_cxxflags) if not installed('diffusers', quiet=True): # diffusers are not installed, so run initial installation global quick_allowed # pylint: disable=global-statement quick_allowed = False diff --git a/modules/postprocess/gfpgan_model.py b/modules/postprocess/gfpgan_model.py index ad0aa8221..8eb998921 100644 --- a/modules/postprocess/gfpgan_model.py +++ b/modules/postprocess/gfpgan_model.py @@ -72,8 +72,8 @@ def setup_model(dirname): except Exception: pass try: - install('basicsr', quiet=True) - install('gfpgan', quiet=True) + install('git+https://github.com/Disty0/BasicSR@2b6a12c28e0c81bfb13b7e984144f0b0f5461484', 'basicsr') + install('git+https://github.com/Disty0/GFPGAN@09b1190eabbc77e5f15c61fa7c38a2064b403e20', 'gfpgan') import gfpgan import facexlib import modules.detailer diff --git a/modules/rocm.py b/modules/rocm.py index f16809291..cc3268860 100644 --- a/modules/rocm.py +++ b/modules/rocm.py @@ -206,7 +206,7 @@ else: if agent.gfx_version >= 0x1100 and os.environ.get("FLASH_ATTENTION_USE_TRITON_ROCM", "false").lower() != "true": # use the navi_rotary_fix fork because the original doesn't support rotary_emb for transformers # original: "git+https://github.com/ROCm/flash-attention@howiejay/navi_support" - default = "https://github.com/Disty0/flash-attention@navi_rotary_fix" + default = "git+https://github.com/Disty0/flash-attention@navi_rotary_fix" return os.environ.get("FLASH_ATTENTION_PACKAGE", default) is_wsl: bool = os.environ.get('WSL_DISTRO_NAME', 'unknown' if spawn('wslpath -w /') else None) is not None diff --git a/repositories/codeformer/basicsr/__init__.py b/repositories/codeformer/basicsr/__init__.py index c7ffcccd7..24be6f0af 100644 --- a/repositories/codeformer/basicsr/__init__.py +++ b/repositories/codeformer/basicsr/__init__.py @@ -8,4 +8,5 @@ from .models import * from .ops import * from .train import * from .utils import * -from .version import __gitsha__, __version__ +__gitsha__ = '366a46c91d51923c56e09963dbc358bc61315408' +__version__ = '1.3.2' diff --git a/repositories/codeformer/basicsr/losses/losses.py b/repositories/codeformer/basicsr/losses/losses.py index 1bcf272cf..71331aa01 100644 --- a/repositories/codeformer/basicsr/losses/losses.py +++ b/repositories/codeformer/basicsr/losses/losses.py @@ -1,5 +1,4 @@ import math -import lpips import torch from torch import autograd as autograd from torch import nn as nn @@ -260,6 +259,7 @@ class LPIPSLoss(nn.Module): use_input_norm=True, range_norm=False,): super(LPIPSLoss, self).__init__() + import lpips self.perceptual = lpips.LPIPS(net="vgg", spatial=False).eval() self.loss_weight = loss_weight self.use_input_norm = use_input_norm diff --git a/requirements.txt b/requirements.txt index 5c464a7a1..1de823e6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,6 +48,7 @@ gradio==3.43.2 huggingface_hub==0.31.2 numexpr==2.10.2 numpy==1.26.4 +pandas==2.3.0 numba==0.61.2 protobuf==4.25.3 pytorch_lightning==1.9.4 @@ -63,7 +64,6 @@ typing-extensions==4.12.2 # additional blendmodes scipy -pandas torchdiffeq dctorch scikit-image From 92dbf3941b1e5cdf21701d02a8df587f748cf36b Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 9 Jun 2025 23:06:39 +0300 Subject: [PATCH 23/63] Update changelog --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 996813e50..048f5a5b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,14 @@ # Change Log for SD.Next -## Update for 2025-06-06 +## Update for 2025-06-09 + +- **Feature** + - Support Python 3.13 - **Changes** - Increase the medvram mode threshold from 8GB to 12GB - Set CPU backend to use FP32 by default + - Relax Python version checks for Zluda - **Torch** - set default to `torch==2.7.1` @@ -16,7 +20,7 @@ - Fix PyTorch <= 2.4 compatibility with FP8 matmul - Fix VAE with conv quant -- **Fixes** +- **Fixes** - Meissonic with multiple generators - Kandinsky V2.2 invalid attention processor - PixArt Sigma Small and Large loading From 58b646e7f228a71f76752d69d9f1a48900601fad Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 10 Jun 2025 01:48:51 +0300 Subject: [PATCH 24/63] SDNQ add 5-bit and 3-bit quantization support --- CHANGELOG.md | 1 + installer.py | 6 +- modules/model_quant_sdnq.py | 151 ++++++++++++++++++++++++++++++++---- modules/shared.py | 2 +- wiki | 2 +- 5 files changed, 142 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 048f5a5b8..63b86dbd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - **SDNQ Quantization** - Add group size support for convolutional layers - Add quantized matmul support for for convolutional layers + - Add 5-bit and 3-bit quantization support - Fix forced FP32 with tensorwise FP8 matmul - Fix PyTorch <= 2.4 compatibility with FP8 matmul - Fix VAE with conv quant diff --git a/installer.py b/installer.py index 2d0a944f5..f92082dbe 100644 --- a/installer.py +++ b/installer.py @@ -663,7 +663,7 @@ def install_rocm_zluda(): log.info('Using CPU-only torch') torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') else: - #check_python(supported_minors=[9, 10, 11, 12], reason='ROCm backend requires a Python version between 3.9 and 3.12') + #check_python(supported_minors=[9, 10, 11, 12, 13], reason='ROCm backend requires a Python version between 3.9 and 3.12') if os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", None) is None: os.environ.setdefault('TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL', '1') @@ -711,7 +711,7 @@ def install_rocm_zluda(): def install_ipex(torch_command): t_start = time.time() - #check_python(supported_minors=[9, 10, 11, 12], reason='IPEX backend requires a Python version between 3.9 and 3.12') + #check_python(supported_minors=[9, 10, 11, 12, 13], reason='IPEX backend requires a Python version between 3.9 and 3.12') args.use_ipex = True # pylint: disable=attribute-defined-outside-init log.info('IPEX: Intel OneAPI toolkit detected') @@ -744,7 +744,7 @@ def install_ipex(torch_command): def install_openvino(torch_command): t_start = time.time() - #check_python(supported_minors=[9, 10, 11, 12], reason='OpenVINO backend requires a Python version between 3.9 and 3.12') + #check_python(supported_minors=[9, 10, 11, 12, 13], reason='OpenVINO backend requires a Python version between 3.9 and 3.12') log.info('OpenVINO: selected') if sys.platform == 'darwin': torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1 torchvision==0.22.1') diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index a17f8b3c1..06e50ed29 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -15,12 +15,16 @@ torch_version = float(torch.__version__[:3]) dtype_dict = { "int8": {"min": -128, "max": 127, "num_bits": 8, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.int8, "is_unsigned": False, "is_integer": True}, - "uint8": {"min": 0, "max": 255, "num_bits": 8, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "int6": {"min": -32, "max": 31, "num_bits": 6, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, - "uint6": {"min": 0, "max": 63, "num_bits": 6, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "int5": {"min": -16, "max": 15, "num_bits": 5, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "int4": {"min": -8, "max": 7, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, - "uint4": {"min": 0, "max": 15, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "int3": {"min": -4, "max": 3, "num_bits": 3, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "int2": {"min": -2, "max": 1, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, + "uint8": {"min": 0, "max": 255, "num_bits": 8, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint6": {"min": 0, "max": 63, "num_bits": 6, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint5": {"min": 0, "max": 31, "num_bits": 5, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint4": {"min": 0, "max": 15, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint3": {"min": 0, "max": 7, "num_bits": 3, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "uint2": {"min": 0, "max": 3, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "uint1": {"min": 0, "max": 1, "num_bits": 1, "target_dtype": torch.bool, "torch_dtype": torch.bool, "storage_dtype": torch.bool, "is_unsigned": True, "is_integer": True}, "float8_e4m3fn": {"min": -448, "max": 448, "num_bits": 8, "target_dtype": torch.float8_e4m3fn, "torch_dtype": torch.float8_e4m3fn, "storage_dtype": torch.float8_e4m3fn, "is_unsigned": False, "is_integer": False}, @@ -28,9 +32,10 @@ dtype_dict = { "float8_e4m3fnuz": {"min": -240, "max": 240, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e4m3fnuz, "storage_dtype": torch.float8_e4m3fnuz, "is_unsigned": False, "is_integer": False}, "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, } +dtype_dict["bool"] = dtype_dict["uint1"] use_tensorwise_fp8_matmul = torch_version < 2.5 or devices.backend in {"cpu", "openvino"} or (devices.backend == "cuda" and sys.platform == "win32" and torch_version <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) -quantized_matmul_dtypes = ("int8", "int6", "int4", "int2", "float8_e4m3fn", "float8_e5m2") +quantized_matmul_dtypes = ("int8", "int6", "int5", "int4", "int3", "int2", "float8_e4m3fn", "float8_e5m2") if devices.backend in {"cpu", "openvino"}: quantized_matmul_dtypes += ("float8_e4m3fnuz", "float8_e5m2fnuz") @@ -91,11 +96,9 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if group_size == 0: if is_linear_type: - if dtype_dict[weights_dtype]["num_bits"] < 6: - group_size = 2 ** (2 + dtype_dict[weights_dtype]["num_bits"]) + group_size = 2 ** (2 + dtype_dict[weights_dtype]["num_bits"]) else: - if dtype_dict[weights_dtype]["num_bits"] < 8: - group_size = 2 ** (1 + dtype_dict[weights_dtype]["num_bits"]) + group_size = 2 ** (1 + dtype_dict[weights_dtype]["num_bits"]) if not use_quantized_matmul and group_size > 0: if group_size >= channel_size: @@ -337,6 +340,65 @@ def pack_uint6(tensor: torch.Tensor) -> torch.Tensor: return packed_tensor +def pack_uint5(tensor: torch.Tensor) -> torch.Tensor: + if tensor.dtype != torch.uint8: + raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") + packed_tensor = tensor.contiguous().reshape(-1, 8) + packed_tensor = torch.stack( + ( + torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 5], 5)), + torch.bitwise_or(packed_tensor[:, 1], torch.bitwise_left_shift(packed_tensor[:, 6], 5)), + torch.bitwise_or(packed_tensor[:, 2], torch.bitwise_left_shift(packed_tensor[:, 7], 5)), + torch.bitwise_or( + packed_tensor[:, 3], + torch.bitwise_or( + torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 5], 2), 96), + torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 3), 128), + ), + ), + torch.bitwise_or( + packed_tensor[:, 4], + torch.bitwise_or( + torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 6], 2), 96), + torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 4), 128), + ), + ), + ), + dim=-1 + ) + return packed_tensor + + +def unpack_uint5(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: + result = torch.stack( + ( + torch.bitwise_and(packed_tensor[:, 0], 31), + torch.bitwise_and(packed_tensor[:, 1], 31), + torch.bitwise_and(packed_tensor[:, 2], 31), + torch.bitwise_and(packed_tensor[:, 3], 31), + torch.bitwise_and(packed_tensor[:, 4], 31), + torch.bitwise_or( + torch.bitwise_right_shift(packed_tensor[:, 0], 5), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 3], 2), 24), + ), + torch.bitwise_or( + torch.bitwise_right_shift(packed_tensor[:, 1], 5), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 4], 2), 24), + ), + torch.bitwise_or( + torch.bitwise_right_shift(packed_tensor[:, 2], 5), + torch.bitwise_or( + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 3], 3), 16), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 4], 4), 8), + ), + ), + ), + dim=-1 + ).reshape(shape) + return result + + + def pack_uint4(tensor: torch.Tensor) -> torch.Tensor: if tensor.dtype != torch.uint8: raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") @@ -345,6 +407,33 @@ def pack_uint4(tensor: torch.Tensor) -> torch.Tensor: return packed_tensor +def pack_uint3(tensor: torch.Tensor) -> torch.Tensor: + if tensor.dtype != torch.uint8: + raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") + packed_tensor = tensor.contiguous().reshape(-1, 8) + packed_tensor = torch.stack( + ( + torch.bitwise_or( + torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 1], 3)), + torch.bitwise_left_shift(packed_tensor[:, 6], 6), + ), + torch.bitwise_or( + torch.bitwise_or(packed_tensor[:, 2], torch.bitwise_left_shift(packed_tensor[:, 3], 3)), + torch.bitwise_left_shift(packed_tensor[:, 7], 6), + ), + torch.bitwise_or( + torch.bitwise_or(packed_tensor[:, 4], torch.bitwise_left_shift(packed_tensor[:, 5], 3)), + torch.bitwise_or( + torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 6], 4), 64), + torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 5), 128), + ) + ), + ), + dim=-1 + ) + return packed_tensor + + def pack_uint2(tensor: torch.Tensor) -> torch.Tensor: if tensor.dtype != torch.uint8: raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") @@ -380,6 +469,29 @@ def unpack_uint4(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor return result +def unpack_uint3(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: + result = torch.stack( + ( + torch.bitwise_and(packed_tensor[:, 0], 7), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 0], 3), 7), + torch.bitwise_and(packed_tensor[:, 1], 7), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 1], 3), 7), + torch.bitwise_and(packed_tensor[:, 2], 7), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 2], 3), 7), + torch.bitwise_or( + torch.bitwise_right_shift(packed_tensor[:, 0], 6), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 2], 4), 4), + ), + torch.bitwise_or( + torch.bitwise_right_shift(packed_tensor[:, 1], 6), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 2], 5), 4), + ), + ), + dim=-1 + ).reshape(shape) + return result + + def unpack_uint2(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: result = torch.stack( ( @@ -849,14 +961,19 @@ class PackedINTSymmetricWeightsDecompressor(torch.nn.Module): decompressor_dict = { "int8": SymmetricWeightsDecompressor, - "uint8": AsymmetricWeightsDecompressor, "int6": PackedINTSymmetricWeightsDecompressor, - "uint6": PackedINTAsymmetricWeightsDecompressor, + "int5": PackedINTSymmetricWeightsDecompressor, "int4": PackedINTSymmetricWeightsDecompressor, - "uint4": PackedINTAsymmetricWeightsDecompressor, + "int3": PackedINTSymmetricWeightsDecompressor, "int2": PackedINTSymmetricWeightsDecompressor, + "uint8": AsymmetricWeightsDecompressor, + "uint6": PackedINTAsymmetricWeightsDecompressor, + "uint5": PackedINTAsymmetricWeightsDecompressor, + "uint4": PackedINTAsymmetricWeightsDecompressor, + "uint3": PackedINTAsymmetricWeightsDecompressor, "uint2": PackedINTAsymmetricWeightsDecompressor, "uint1": AsymmetricWeightsDecompressor, + "bool": AsymmetricWeightsDecompressor, "float8_e4m3fn": SymmetricWeightsDecompressor, "float8_e4m3fnuz": SymmetricWeightsDecompressor, "float8_e5m2": SymmetricWeightsDecompressor, @@ -866,10 +983,14 @@ decompressor_dict = { packed_int_function_dict = { "int6": {"pack": pack_uint6, "unpack": unpack_uint6}, - "uint6": {"pack": pack_uint6, "unpack": unpack_uint6}, + "int5": {"pack": pack_uint5, "unpack": unpack_uint5}, "int4": {"pack": pack_uint4, "unpack": unpack_uint4}, - "uint4": {"pack": pack_uint4, "unpack": unpack_uint4}, + "int3": {"pack": pack_uint3, "unpack": unpack_uint3}, "int2": {"pack": pack_uint2, "unpack": unpack_uint2}, + "uint6": {"pack": pack_uint6, "unpack": unpack_uint6}, + "uint5": {"pack": pack_uint5, "unpack": unpack_uint5}, + "uint4": {"pack": pack_uint4, "unpack": unpack_uint4}, + "uint3": {"pack": pack_uint3, "unpack": unpack_uint3}, "uint2": {"pack": pack_uint2, "unpack": unpack_uint2}, } @@ -1028,7 +1149,7 @@ class SDNQConfig(QuantizationConfigMixin): Args: weights_dtype (`str`, *optional*, defaults to `"int8"`): The target dtype for the weights after quantization. Supported values are: - ("int8", "uint8", "int6", "uint6", "int4", "uint4", "uint2", "uint1", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz") + ("int8", "int6", "int5", "int4", "int3", "int2", "uint8", "uint6", "uint5", "uint4", "uint3", "uint2", "uint1", "bool", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz") modules_to_not_convert (`list`, *optional*, default to `None`): The list of modules to not quantize, useful for quantizing models that explicitly require to have some modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers). @@ -1058,7 +1179,7 @@ class SDNQConfig(QuantizationConfigMixin): r""" Safety checker that arguments are correct """ - accepted_weights = ["int8", "uint8", "int6", "uint6", "int4", "uint4", "uint2", "uint1", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz"] + accepted_weights = ["int8", "int6", "int5", "int4", "int3", "int2", "uint8", "uint6", "uint5", "uint4", "uint3", "uint2", "uint1", "bool", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz"] if self.weights_dtype not in accepted_weights: raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}") diff --git a/modules/shared.py b/modules/shared.py index ffe783280..c05039363 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -518,7 +518,7 @@ options_templates.update(options_section(("quantization", "Quantization Settings "sdnq_quantize_sep": OptionInfo("

SDNQ: SD.Next Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ["pre", "post"], "visible": native}), - "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "int6", "uint4", "float8_e4m3fn", "uint8", "uint6", "int4", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "int2", "uint2", "uint1"], "visible": native}), + "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "float8_e4m3fn", "int6", "int5", "uint4", "uint3", "uint2", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "uint8", "uint6", "uint5", "int4", "int3", "int2", "uint1"], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_decompress_compile": OptionInfo(devices.has_triton(), "Decompress using torch.compile", gr.Checkbox, {"visible": native}), diff --git a/wiki b/wiki index 048798a26..f01d19390 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 048798a26a7f86957f9bb5cb67846fded630bad5 +Subproject commit f01d19390603551f4713015f7842cae21e70f6f0 From 5eed9135e3b0a69aaf1cb895fddff6de49034739 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 10 Jun 2025 03:18:25 +0300 Subject: [PATCH 25/63] Split SDNQ into multiple files and linting --- modules/lora/lora_apply.py | 2 +- modules/model_quant.py | 6 +- modules/model_quant_sdnq.py | 1230 ---------------------------------- modules/sdnq/__init__.py | 428 ++++++++++++ modules/sdnq/common.py | 41 ++ modules/sdnq/decompressor.py | 170 +++++ modules/sdnq/forward.py | 402 +++++++++++ modules/sdnq/packed_int.py | 212 ++++++ 8 files changed, 1257 insertions(+), 1234 deletions(-) delete mode 100644 modules/model_quant_sdnq.py create mode 100644 modules/sdnq/__init__.py create mode 100644 modules/sdnq/common.py create mode 100644 modules/sdnq/decompressor.py create mode 100644 modules/sdnq/forward.py create mode 100644 modules/sdnq/packed_int.py diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index a5bfd8194..b265fcce2 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -138,7 +138,7 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G shared.log.error(f'Network load: type=LoRA quant=bnb cls={self.__class__.__name__} type={self.quant_type} blocksize={self.blocksize} state={vars(self.quant_state)} weight={self.weight} bias={lora_weights} {e}') elif not bias and hasattr(self, "sdnq_decompressor"): try: - from modules.model_quant_sdnq import sdnq_quantize_layer + from modules.sdnq import sdnq_quantize_layer if hasattr(self, "sdnq_decompressor_backup"): sdnq_decompressor = self.sdnq_decompressor_backup.to(devices.device) else: diff --git a/modules/model_quant.py b/modules/model_quant.py index 099c175bc..6e2429e35 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -107,7 +107,7 @@ def create_sdnq_config(kwargs = None, allow_sdnq: bool = True, module: str = 'Mo from modules import shared if len(shared.opts.sdnq_quantize_weights) > 0 and (shared.opts.sdnq_quantize_mode == 'pre') and allow_sdnq: if 'Model' in shared.opts.sdnq_quantize_weights or (module is not None and module in shared.opts.sdnq_quantize_weights) or module == 'any': - from modules.model_quant_sdnq import SDNQQuantizer, SDNQConfig + from modules.sdnq import SDNQQuantizer, SDNQConfig diffusers.quantizers.auto.AUTO_QUANTIZER_MAPPING["sdnq"] = SDNQQuantizer transformers.quantizers.auto.AUTO_QUANTIZER_MAPPING["sdnq"] = SDNQQuantizer diffusers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["sdnq"] = SDNQConfig @@ -303,13 +303,13 @@ def apply_layerwise(sd_model, quiet:bool=False): def sdnq_quantize_model(model, op=None, sd_model=None, do_gc=True): global quant_last_model_name, quant_last_model_device # pylint: disable=global-statement from modules import devices, shared - from modules.model_quant_sdnq import apply_sdnq_to_module + from modules.sdnq import apply_sdnq_to_module model.eval() if model.__class__.__name__ in {"T5EncoderModel", "UMT5EncoderModel"}: import torch - from modules.model_quant_sdnq import SDNQ_T5DenseGatedActDense # T5DenseGatedActDense uses fp32 + from modules.sdnq import SDNQ_T5DenseGatedActDense # T5DenseGatedActDense uses fp32 for i in range(len(model.encoder.block)): model.encoder.block[i].layer[1].DenseReluDense = SDNQ_T5DenseGatedActDense( model.encoder.block[i].layer[1].DenseReluDense, diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py deleted file mode 100644 index 06e50ed29..000000000 --- a/modules/model_quant_sdnq.py +++ /dev/null @@ -1,1230 +0,0 @@ -# pylint: disable=redefined-builtin,no-member,protected-access - -from typing import Any, Dict, List, Tuple, Optional, Union -from dataclasses import dataclass -from enum import Enum -import sys -import torch -from diffusers.quantizers.base import DiffusersQuantizer -from diffusers.quantizers.quantization_config import QuantizationConfigMixin -from diffusers.utils import get_module_from_name -from accelerate.utils import CustomDtype -from modules import devices, shared - -torch_version = float(torch.__version__[:3]) - -dtype_dict = { - "int8": {"min": -128, "max": 127, "num_bits": 8, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.int8, "is_unsigned": False, "is_integer": True}, - "int6": {"min": -32, "max": 31, "num_bits": 6, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, - "int5": {"min": -16, "max": 15, "num_bits": 5, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, - "int4": {"min": -8, "max": 7, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, - "int3": {"min": -4, "max": 3, "num_bits": 3, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, - "int2": {"min": -2, "max": 1, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, - "uint8": {"min": 0, "max": 255, "num_bits": 8, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, - "uint6": {"min": 0, "max": 63, "num_bits": 6, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, - "uint5": {"min": 0, "max": 31, "num_bits": 5, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, - "uint4": {"min": 0, "max": 15, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, - "uint3": {"min": 0, "max": 7, "num_bits": 3, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, - "uint2": {"min": 0, "max": 3, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, - "uint1": {"min": 0, "max": 1, "num_bits": 1, "target_dtype": torch.bool, "torch_dtype": torch.bool, "storage_dtype": torch.bool, "is_unsigned": True, "is_integer": True}, - "float8_e4m3fn": {"min": -448, "max": 448, "num_bits": 8, "target_dtype": torch.float8_e4m3fn, "torch_dtype": torch.float8_e4m3fn, "storage_dtype": torch.float8_e4m3fn, "is_unsigned": False, "is_integer": False}, - "float8_e5m2": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": torch.float8_e5m2, "torch_dtype": torch.float8_e5m2, "storage_dtype": torch.float8_e5m2, "is_unsigned": False, "is_integer": False}, - "float8_e4m3fnuz": {"min": -240, "max": 240, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e4m3fnuz, "storage_dtype": torch.float8_e4m3fnuz, "is_unsigned": False, "is_integer": False}, - "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, -} -dtype_dict["bool"] = dtype_dict["uint1"] - -use_tensorwise_fp8_matmul = torch_version < 2.5 or devices.backend in {"cpu", "openvino"} or (devices.backend == "cuda" and sys.platform == "win32" and torch_version <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) -quantized_matmul_dtypes = ("int8", "int6", "int5", "int4", "int3", "int2", "float8_e4m3fn", "float8_e5m2") -if devices.backend in {"cpu", "openvino"}: - quantized_matmul_dtypes += ("float8_e4m3fnuz", "float8_e5m2fnuz") - -linear_types = ("Linear",) -conv_types = ("Conv1d", "Conv2d", "Conv3d") -conv_transpose_types = ("ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d") -allowed_types = linear_types + conv_types + conv_transpose_types - - -class QuantizationMethod(str, Enum): - SDNQ = "sdnq" - - -def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, param_name=None, pre_mode=False): - layer_class_name = layer.__class__.__name__ - if layer_class_name in allowed_types: - is_conv_type = False - is_conv_transpose_type = False - is_linear_type = False - result_shape = None - if torch_dtype is None: - torch_dtype = devices.dtype - - if layer_class_name in conv_types: - if not quant_conv: - return layer - if dtype_dict[weights_dtype]["num_bits"] < 4: - weights_dtype = "uint4" - is_conv_type = True - reduction_axes = 1 - output_channel_size, channel_size = layer.weight.shape[:2] - group_channel_size = channel_size // layer.groups - use_quantized_matmul = False - if use_quantized_matmul_conv: - use_quantized_matmul = weights_dtype in quantized_matmul_dtypes and group_channel_size >= 32 and output_channel_size >= 32 - if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: - use_quantized_matmul = output_channel_size % 16 == 0 and group_channel_size % 16 == 0 - if use_quantized_matmul: - result_shape = layer.weight.shape - layer.weight.data = layer.weight.reshape(output_channel_size, -1) - elif layer_class_name in conv_transpose_types: - if not quant_conv: - return layer - if dtype_dict[weights_dtype]["num_bits"] < 4: - weights_dtype = "uint4" - is_conv_transpose_type = True - reduction_axes = 0 - channel_size, output_channel_size = layer.weight.shape[:2] - use_quantized_matmul = False - else: - is_linear_type = True - reduction_axes = -1 - output_channel_size, channel_size = layer.weight.shape - if use_quantized_matmul: - use_quantized_matmul = weights_dtype in quantized_matmul_dtypes and channel_size >= 32 and output_channel_size >= 32 - if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: - use_quantized_matmul = output_channel_size % 16 == 0 and channel_size % 16 == 0 - - if group_size == 0: - if is_linear_type: - group_size = 2 ** (2 + dtype_dict[weights_dtype]["num_bits"]) - else: - group_size = 2 ** (1 + dtype_dict[weights_dtype]["num_bits"]) - - if not use_quantized_matmul and group_size > 0: - if group_size >= channel_size: - group_size = channel_size - num_of_groups = 1 - else: - num_of_groups = channel_size // group_size - while channel_size % group_size != 0: # find something divisible - num_of_groups -= 1 - if num_of_groups <= 1: - group_size = channel_size - num_of_groups = 1 - break - group_size = channel_size / num_of_groups - group_size = int(group_size) - num_of_groups = int(num_of_groups) - - if num_of_groups > 1: - result_shape = layer.weight.shape - new_shape = list(result_shape) - if is_conv_type: - # output_channel_size, channel_size, X, X - # output_channel_size, num_of_groups, group_size, X, X - new_shape[1] = group_size - new_shape.insert(1, num_of_groups) - reduction_axes = 2 - elif is_conv_transpose_type: - #channel_size, output_channel_size, X, X - #num_of_groups, group_size, output_channel_size, X, X - new_shape[0] = group_size - new_shape.insert(0, num_of_groups) - reduction_axes = 1 - elif is_linear_type: - # output_channel_size, channel_size - # output_channel_size, num_of_groups, group_size - last_dim_index = layer.weight.ndim - new_shape[last_dim_index - 1 : last_dim_index] = (num_of_groups, group_size) - layer.weight.data = layer.weight.reshape(new_shape) - - layer.weight.requires_grad = False - if shared.opts.diffusers_offload_mode in {"none", "model"}: - return_device = devices.device - elif pre_mode: - if shared.opts.device_map == "gpu": - return_device = devices.device - elif shared.opts.sdnq_quantize_with_gpu: - return_device = devices.cpu - else: - return_device = layer.weight.device - else: - return_device = layer.weight.device - if not pre_mode: - if shared.opts.sdnq_quantize_with_gpu: - layer.weight.data = layer.weight.to(devices.device).to(dtype=torch.float32) - else: - layer.weight.data = layer.weight.to(dtype=torch.float32) - - if dtype_dict[weights_dtype]["is_unsigned"]: - scale, zero_point = get_scale_asymmetric(layer.weight, reduction_axes, weights_dtype) - else: - scale = get_scale_symmetric(layer.weight, reduction_axes, weights_dtype) - zero_point = None - layer.weight.data = quantize_weight(layer.weight, scale, zero_point, weights_dtype) - - if not shared.opts.sdnq_decompress_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"] and not use_tensorwise_fp8_matmul): - scale = scale.to(torch_dtype) - if zero_point is not None: - zero_point = zero_point.to(torch_dtype) - - if use_quantized_matmul: - scale = scale.transpose(0,1) - if dtype_dict[weights_dtype]["num_bits"] == 8: - layer.weight.data = layer.weight.transpose(0,1) - if not dtype_dict[weights_dtype]["is_integer"]: - stride = layer.weight.stride() - if stride[0] > stride[1] and stride[1] == 1: - layer.weight.data = layer.weight.t().contiguous().t() - if not use_tensorwise_fp8_matmul: - scale = scale.to(torch.float32) - - layer.sdnq_decompressor = decompressor_dict[weights_dtype]( - scale=scale, - zero_point=zero_point, - compressed_weight_shape=layer.weight.shape, - result_dtype=torch_dtype, - result_shape=result_shape, - weights_dtype=weights_dtype, - use_quantized_matmul=use_quantized_matmul, - ) - layer.weight.data = layer.sdnq_decompressor.pack_weight(layer.weight).to(return_device) - layer.sdnq_decompressor = layer.sdnq_decompressor.to(return_device) - - if is_linear_type: - if use_quantized_matmul: - if dtype_dict[weights_dtype]["is_integer"]: - layer.forward = quantized_linear_forward_int8_matmul - else: - if use_tensorwise_fp8_matmul: - layer.forward = quantized_linear_forward_fp8_matmul_tensorwise - else: - layer.forward = quantized_linear_forward_fp8_matmul - else: - layer.forward = quantized_linear_forward - elif is_conv_type: - if use_quantized_matmul: - if dtype_dict[weights_dtype]["is_integer"]: - layer.forward = quantized_conv_forward_int8_matmul - else: - if use_tensorwise_fp8_matmul: - layer.forward = quantized_conv_forward_fp8_matmul_tensorwise - else: - layer.forward = quantized_conv_forward_fp8_matmul - else: - layer.forward = quantized_conv_forward - elif is_conv_transpose_type: - if layer_class_name.endswith("1d"): - layer.forward = quantized_conv_transpose_1d_forward - elif layer_class_name.endswith("2d"): - layer.forward = quantized_conv_transpose_2d_forward - elif layer_class_name.endswith("3d"): - layer.forward = quantized_conv_transpose_3d_forward - layer.forward = layer.forward.__get__(layer, layer.__class__) - devices.torch_gc(force=False, reason=f"SDNQ param_name: {param_name}") - return layer - - -def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, param_name=None): - has_children = list(model.children()) - if not has_children: - return model - for module_param_name, module in model.named_children(): - if hasattr(module, "weight") and module.weight is not None: - module = sdnq_quantize_layer( - module, - weights_dtype=weights_dtype, - torch_dtype=torch_dtype, - group_size=group_size, - quant_conv=quant_conv, - use_quantized_matmul=use_quantized_matmul, - use_quantized_matmul_conv=use_quantized_matmul_conv, - param_name=module_param_name, - ) - module = apply_sdnq_to_module( - module, - weights_dtype=weights_dtype, - torch_dtype=torch_dtype, - group_size=group_size, - quant_conv=quant_conv, - use_quantized_matmul=use_quantized_matmul, - use_quantized_matmul_conv=use_quantized_matmul_conv, - param_name=module_param_name, - ) - return model - - -def get_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> Tuple[torch.FloatTensor, torch.FloatTensor]: - zero_point = torch.amin(weight, dim=reduction_axes, keepdims=True) - scale = torch.amax(weight, dim=reduction_axes, keepdims=True).sub_(zero_point).div_(dtype_dict[weights_dtype]["max"] - dtype_dict[weights_dtype]["min"]) - eps = torch.finfo(scale.dtype).eps # prevent divison by 0 - scale = torch.where(torch.abs(scale) < eps, eps, scale) - if dtype_dict[weights_dtype]["min"] != 0: - zero_point.sub_(torch.mul(scale, dtype_dict[weights_dtype]["min"])) - return scale, zero_point - - -def get_scale_symmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> torch.FloatTensor: - abs_min_values = torch.amin(weight, dim=reduction_axes, keepdims=True).abs_() - max_values = torch.amax(weight, dim=reduction_axes, keepdims=True) - scale = torch.where(abs_min_values >= max_values, abs_min_values, -max_values).div_(dtype_dict[weights_dtype]["max"]) - eps = torch.finfo(scale.dtype).eps # prevent divison by 0 - scale = torch.where(torch.abs(scale) < eps, eps, scale) - return scale - - -def quantize_weight(weight: torch.FloatTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, weights_dtype: str) -> torch.ByteTensor: - if zero_point is not None: - compressed_weight = torch.sub(weight, zero_point).div_(scale) - else: - compressed_weight = torch.div(weight, scale) - if dtype_dict[weights_dtype]["is_integer"]: - compressed_weight.round_() - compressed_weight = compressed_weight.clamp_(dtype_dict[weights_dtype]["min"], dtype_dict[weights_dtype]["max"]).to(dtype_dict[weights_dtype]["torch_dtype"]) - return compressed_weight - - -def decompress_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: torch.Tensor, dtype: torch.dtype, result_shape: torch.Size) -> torch.Tensor: - result = torch.addcmul(zero_point, input.to(dtype=scale.dtype), scale).to(dtype=dtype) - if result_shape is not None: - result = result.reshape(result_shape) - return result - - -def decompress_symmetric(input: torch.Tensor, scale: torch.Tensor, dtype: torch.dtype, result_shape: torch.Size, skip_quantized_matmul: bool = False) -> torch.Tensor: - if skip_quantized_matmul: - result = input.transpose(0,1).to(dtype=scale.dtype).mul_(scale.transpose(0,1)).to(dtype=dtype) - else: - result = input.to(dtype=scale.dtype).mul_(scale).to(dtype=dtype) - if result_shape is not None: - result = result.reshape(result_shape) - return result - - -def decompress_packed_int_asymmetric(input: torch.Tensor, scale: torch.Tensor, zero_point: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str) -> torch.Tensor: - return decompress_asymmetric(packed_int_function_dict[weights_dtype]["unpack"](input, shape), scale, zero_point, dtype, result_shape) - - -def decompress_packed_int_symmetric(input: torch.Tensor, scale: torch.Tensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str, skip_quantized_matmul: bool = False) -> torch.Tensor: - if skip_quantized_matmul: - return decompress_symmetric(unpack_int_symetric(input, shape, weights_dtype, dtype=scale.dtype), scale.transpose(0,1), dtype, result_shape) - else: - return decompress_symmetric(unpack_int_symetric(input, shape, weights_dtype, dtype=scale.dtype), scale, dtype, result_shape) - - -def pack_int_symetric(tensor: torch.Tensor, weights_dtype: str) -> torch.Tensor: - return packed_int_function_dict[weights_dtype]["pack"](tensor.to(dtype=dtype_dict[weights_dtype]["torch_dtype"]).sub_(dtype_dict[weights_dtype]["min"]).to(dtype=dtype_dict[weights_dtype]["storage_dtype"])) - - -def unpack_int_symetric(packed_tensor: torch.Tensor, shape: torch.Size, weights_dtype: str, dtype: Optional[torch.dtype] = None, transpose: Optional[bool] = False) -> torch.Tensor: - if dtype is None: - dtype = dtype_dict[weights_dtype]["torch_dtype"] - result = packed_int_function_dict[weights_dtype]["unpack"](packed_tensor, shape).to(dtype=dtype).add_(dtype_dict[weights_dtype]["min"]) - if transpose: - result = result.transpose(0,1) - return result - - -def pack_uint6(tensor: torch.Tensor) -> torch.Tensor: - if tensor.dtype != torch.uint8: - raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") - packed_tensor = tensor.contiguous().reshape(-1, 4) - packed_tensor = torch.stack( - ( - torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 3], 2), 192)), - torch.bitwise_or(packed_tensor[:, 1], torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 3], 4), 192)), - torch.bitwise_or(packed_tensor[:, 2], torch.bitwise_left_shift(packed_tensor[:, 3], 6)), - ), - dim=-1 - ) - return packed_tensor - - -def pack_uint5(tensor: torch.Tensor) -> torch.Tensor: - if tensor.dtype != torch.uint8: - raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") - packed_tensor = tensor.contiguous().reshape(-1, 8) - packed_tensor = torch.stack( - ( - torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 5], 5)), - torch.bitwise_or(packed_tensor[:, 1], torch.bitwise_left_shift(packed_tensor[:, 6], 5)), - torch.bitwise_or(packed_tensor[:, 2], torch.bitwise_left_shift(packed_tensor[:, 7], 5)), - torch.bitwise_or( - packed_tensor[:, 3], - torch.bitwise_or( - torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 5], 2), 96), - torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 3), 128), - ), - ), - torch.bitwise_or( - packed_tensor[:, 4], - torch.bitwise_or( - torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 6], 2), 96), - torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 4), 128), - ), - ), - ), - dim=-1 - ) - return packed_tensor - - -def unpack_uint5(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: - result = torch.stack( - ( - torch.bitwise_and(packed_tensor[:, 0], 31), - torch.bitwise_and(packed_tensor[:, 1], 31), - torch.bitwise_and(packed_tensor[:, 2], 31), - torch.bitwise_and(packed_tensor[:, 3], 31), - torch.bitwise_and(packed_tensor[:, 4], 31), - torch.bitwise_or( - torch.bitwise_right_shift(packed_tensor[:, 0], 5), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 3], 2), 24), - ), - torch.bitwise_or( - torch.bitwise_right_shift(packed_tensor[:, 1], 5), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 4], 2), 24), - ), - torch.bitwise_or( - torch.bitwise_right_shift(packed_tensor[:, 2], 5), - torch.bitwise_or( - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 3], 3), 16), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 4], 4), 8), - ), - ), - ), - dim=-1 - ).reshape(shape) - return result - - - -def pack_uint4(tensor: torch.Tensor) -> torch.Tensor: - if tensor.dtype != torch.uint8: - raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") - packed_tensor = tensor.contiguous().reshape(-1, 2) - packed_tensor = torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 1], 4)) - return packed_tensor - - -def pack_uint3(tensor: torch.Tensor) -> torch.Tensor: - if tensor.dtype != torch.uint8: - raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") - packed_tensor = tensor.contiguous().reshape(-1, 8) - packed_tensor = torch.stack( - ( - torch.bitwise_or( - torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 1], 3)), - torch.bitwise_left_shift(packed_tensor[:, 6], 6), - ), - torch.bitwise_or( - torch.bitwise_or(packed_tensor[:, 2], torch.bitwise_left_shift(packed_tensor[:, 3], 3)), - torch.bitwise_left_shift(packed_tensor[:, 7], 6), - ), - torch.bitwise_or( - torch.bitwise_or(packed_tensor[:, 4], torch.bitwise_left_shift(packed_tensor[:, 5], 3)), - torch.bitwise_or( - torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 6], 4), 64), - torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 5), 128), - ) - ), - ), - dim=-1 - ) - return packed_tensor - - -def pack_uint2(tensor: torch.Tensor) -> torch.Tensor: - if tensor.dtype != torch.uint8: - raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") - packed_tensor = tensor.contiguous().reshape(-1, 4) - packed_tensor = torch.bitwise_or( - torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 1], 2)), - torch.bitwise_or(torch.bitwise_left_shift(packed_tensor[:, 2], 4), torch.bitwise_left_shift(packed_tensor[:, 3], 6)), - ) - return packed_tensor - - -def unpack_uint6(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: - result = torch.stack( - ( - torch.bitwise_and(packed_tensor[:, 0], 63), - torch.bitwise_and(packed_tensor[:, 1], 63), - torch.bitwise_and(packed_tensor[:, 2], 63), - torch.bitwise_or( - torch.bitwise_or( - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 0], 2), 48), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 1], 4), 12), - ), - torch.bitwise_right_shift(packed_tensor[:, 2], 6) - ) - ), - dim=-1 - ).reshape(shape) - return result - - -def unpack_uint4(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: - result = torch.stack((torch.bitwise_and(packed_tensor, 15), torch.bitwise_right_shift(packed_tensor, 4)), dim=-1).reshape(shape) - return result - - -def unpack_uint3(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: - result = torch.stack( - ( - torch.bitwise_and(packed_tensor[:, 0], 7), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 0], 3), 7), - torch.bitwise_and(packed_tensor[:, 1], 7), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 1], 3), 7), - torch.bitwise_and(packed_tensor[:, 2], 7), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 2], 3), 7), - torch.bitwise_or( - torch.bitwise_right_shift(packed_tensor[:, 0], 6), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 2], 4), 4), - ), - torch.bitwise_or( - torch.bitwise_right_shift(packed_tensor[:, 1], 6), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 2], 5), 4), - ), - ), - dim=-1 - ).reshape(shape) - return result - - -def unpack_uint2(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: - result = torch.stack( - ( - torch.bitwise_and(packed_tensor, 3), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 2), 3), - torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 4), 3), - torch.bitwise_right_shift(packed_tensor, 6), - ), - dim=-1 - ).reshape(shape) - return result - - -def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.FloatTensor, torch.FloatTensor]: - input = input.flatten(0,-2).contiguous() - input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 448) - input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn) - input_scale = input_scale.to(torch.float32) - return input, input_scale - - -def quantize_fp8_matmul_input_tensorwise(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]: - input = input.flatten(0,-2).contiguous() - input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 448) - input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn) - scale = torch.mul(input_scale, scale) - if scale.dtype == torch.float16: # fp16 will overflow - scale = scale.to(dtype=torch.float32) - return input, scale - - -def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.ByteTensor, torch.FloatTensor]: - input = input.flatten(0,-2).contiguous() - input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 127) - input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(torch.int8) - scale = torch.mul(input_scale, scale) - if scale.dtype == torch.float16: # fp16 will overflow - scale = scale.to(dtype=torch.float32) - return input, scale - - -def fp8_matmul( - input: torch.FloatTensor, - weight: torch.Tensor, - bias: torch.FloatTensor, - scale: torch.FloatTensor, -) -> torch.FloatTensor: - return_dtype = input.dtype - output_shape = list(input.shape) - output_shape[-1] = weight.shape[-1] - input, input_scale = quantize_fp8_matmul_input(input) - return torch._scaled_mm(input, weight, scale_a=input_scale, scale_b=scale, bias=bias, out_dtype=return_dtype).reshape(output_shape) - - -# sm89 doesn't support row wise scale in Windows -def fp8_matmul_tensorwise( - input: torch.FloatTensor, - weight: torch.Tensor, - bias: torch.FloatTensor, - scale: torch.FloatTensor, -) -> torch.FloatTensor: - return_dtype = input.dtype - output_shape = list(input.shape) - output_shape[-1] = weight.shape[-1] - dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) - input, scale = quantize_fp8_matmul_input_tensorwise(input, scale) - result = decompress_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, output_shape) - if bias is not None: - result.add_(bias) - return result - - -def int8_matmul( - input: torch.FloatTensor, - weight: torch.Tensor, - bias: torch.FloatTensor, - scale: torch.FloatTensor, - compressed_weight_shape: torch.Size, - weights_dtype: str, -) -> torch.FloatTensor: - if compressed_weight_shape is not None: - weight = unpack_int_symetric(weight, compressed_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) - return_dtype = input.dtype - output_shape = list(input.shape) - output_shape[-1] = weight.shape[-1] - input, scale = quantize_int8_matmul_input(input, scale) - result = decompress_symmetric(torch._int_mm(input, weight), scale, return_dtype, output_shape) - if bias is not None: - result.add_(bias) - return result - - -def process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation): - if conv_type == 1: - batch_size, _, L_in = input.shape - C_out, _, K_l = result_shape - L_out = (L_in + 2 * padding[1] - dilation[1] * (K_l - 1) - 1) // stride[1] + 1 - mm_output_shape = (batch_size, L_out, C_out) - kernel_size = (1, K_l) - if conv_type == 2: - batch_size, _, H_in, W_in = input.shape - C_out, _, K_h, K_w = result_shape - H_out = (H_in + 2 * padding[0] - dilation[0] * (K_h - 1) - 1) // stride[0] + 1 - W_out = (W_in + 2 * padding[1] - dilation[1] * (K_w - 1) - 1) // stride[1] + 1 - mm_output_shape = (batch_size, H_out, W_out, C_out) - kernel_size = (K_h, K_w) - elif conv_type == 3: - batch_size, _, D_in, H_in, W_in = input.shape - C_out, _, K_d, K_h, K_w = result_shape - D_out = (D_in + 2 * padding[0] - dilation[0] * (K_d - 1) - 1) // stride[0] + 1 - H_out = (H_in + 2 * padding[1] - dilation[1] * (K_h - 1) - 1) // stride[1] + 1 - W_out = (W_in + 2 * padding[2] - dilation[2] * (K_w - 1) - 1) // stride[2] + 1 - mm_output_shape = (batch_size, D_out, H_out, W_out, C_out) - kernel_size = (K_d, K_h, K_w) - - if padding_mode != "zeros": - input = torch.nn.functional.pad(input, reversed_padding_repeated_twice, mode=padding_mode) - padding = (0,) * (conv_type if conv_type != 1 else 2) - elif conv_type == 3: - input = torch.nn.functional.pad(input, reversed_padding_repeated_twice) - - if conv_type == 1: - input = input.unsqueeze(2) - - if conv_type == 3: - K_D_eff = K_d + (K_d - 1) * (dilation[0] - 1) - K_H_eff = K_h + (K_h - 1) * (dilation[0] - 1) - K_W_eff = K_w + (K_w - 1) * (dilation[0] - 1) - input = input.unfold(2, K_D_eff, stride[0]).unfold(3, K_H_eff, stride[1]).unfold(4, K_W_eff, stride[2]) - if dilation[0] > 1: - input = input[..., ::dilation[0], :, :] - if dilation[1] > 1: - input = input[..., ::dilation[1], :] - if dilation[2] > 1: - input = input[..., ::dilation[2]] - input = input.permute(0, 2, 3, 4, 1, 5, 6, 7).reshape(batch_size, D_out * H_out * W_out, -1) - else: - input = torch.nn.functional.unfold(input, kernel_size=kernel_size, padding=padding, stride=stride, dilation=dilation).transpose(1,2) - return input, mm_output_shape - - -def conv_fp8_matmul( - input: torch.FloatTensor, - weight: torch.ByteTensor, - bias: torch.FloatTensor, - scale: torch.FloatTensor, - result_shape: torch.Size, - weights_dtype: str, - reversed_padding_repeated_twice: List[int], - padding_mode: str, conv_type: int, - groups: int, stride: List[int], - padding: List[int], dilation: List[int], -) -> torch.FloatTensor: - return_dtype = input.dtype - input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) - input, input_scale = quantize_fp8_matmul_input(input) - - if groups == 1: - result = torch._scaled_mm(input, weight, scale_a=input_scale, scale_b=scale, bias=bias, out_dtype=return_dtype).reshape(mm_output_shape) - else: - scale = scale.reshape(groups, 1, scale.shape[1] // groups) - input_scale = input_scale.reshape(groups, input_scale.shape[0] // groups, 1) - weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) - input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) - result = [] - for i in range(groups): - result.append(torch._scaled_mm(input[i], weight[i], scale_a=input_scale[i], scale_b=scale[i], bias=None, out_dtype=return_dtype)) - result = torch.cat(result, dim=-1).reshape(mm_output_shape) - if bias is not None: - result.add_(bias) - - if conv_type == 1: - result = result.transpose(1,2) - elif conv_type == 2: - result = result.permute(0,3,1,2) - elif conv_type == 3: - result = result.permute(0,4,1,2,3) - return result - - -def conv_fp8_matmul_tensorwise( - input: torch.FloatTensor, - weight: torch.ByteTensor, - bias: torch.FloatTensor, - scale: torch.FloatTensor, - result_shape: torch.Size, - weights_dtype: str, - reversed_padding_repeated_twice: List[int], - padding_mode: str, conv_type: int, - groups: int, stride: List[int], - padding: List[int], dilation: List[int], -) -> torch.FloatTensor: - return_dtype = input.dtype - input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) - input, scale = quantize_fp8_matmul_input_tensorwise(input, scale) - dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) - - if groups == 1: - result = decompress_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, mm_output_shape) - else: - weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) - input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) - result = [] - for i in range(groups): - result.append(torch._scaled_mm(input[i], weight[i], scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype)) - result = decompress_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) - if bias is not None: - result.add_(bias) - - if conv_type == 1: - result = result.transpose(1,2) - elif conv_type == 2: - result = result.permute(0,3,1,2) - elif conv_type == 3: - result = result.permute(0,4,1,2,3) - return result - - -def conv_int8_matmul( - input: torch.FloatTensor, - weight: torch.ByteTensor, - bias: torch.FloatTensor, - scale: torch.FloatTensor, - result_shape: torch.Size, - compressed_weight_shape: torch.Size, - weights_dtype: str, - reversed_padding_repeated_twice: List[int], - padding_mode: str, conv_type: int, - groups: int, stride: List[int], - padding: List[int], dilation: List[int], -) -> torch.FloatTensor: - return_dtype = input.dtype - input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) - input, scale = quantize_int8_matmul_input(input, scale) - if compressed_weight_shape is not None: - weight = unpack_int_symetric(weight, compressed_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) - - if groups == 1: - result = decompress_symmetric(torch._int_mm(input, weight), scale, return_dtype, mm_output_shape) - else: - weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) - input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) - result = [] - for i in range(groups): - result.append(torch._int_mm(input[i], weight[i])) - result = decompress_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) - if bias is not None: - result.add_(bias) - - if conv_type == 1: - result = result.transpose(1,2) - elif conv_type == 2: - result = result.permute(0,3,1,2) - elif conv_type == 3: - result = result.permute(0,4,1,2,3) - return result - - -def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: - if torch.numel(input) / input.shape[-1] < 32: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) - return fp8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale) - - -def quantized_linear_forward_fp8_matmul_tensorwise(self, input: torch.FloatTensor) -> torch.FloatTensor: - if torch.numel(input) / input.shape[-1] < 32: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) - return fp8_matmul_tensorwise(input, self.weight, self.bias, self.sdnq_decompressor.scale) - - -def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: - if torch.numel(input) / input.shape[-1] < 32: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) - return int8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale, getattr(self.sdnq_decompressor, "compressed_weight_shape", None), self.sdnq_decompressor.weights_dtype) - - -def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTensor: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight), self.bias) - - -def get_conv_args(input_ndim, stride, padding, dilation): - if input_ndim == 3: - conv_type = 1 - elif input_ndim == 4: - conv_type = 2 - elif input_ndim == 5: - conv_type = 3 - if isinstance(stride, int): - stride = (stride,) * conv_type - if isinstance(padding, int): - padding = (padding,) * conv_type - if isinstance(dilation, int): - dilation = (dilation,) * conv_type - if conv_type == 1: - stride = (1, stride[0]) - padding = (0, padding[0]) - dilation = (1, dilation[0]) - return conv_type, stride, padding, dilation - - -def quantized_conv_forward_fp8_matmul(self, input) -> torch.FloatTensor: - if torch.numel(input) / input.shape[2] < 32: - return self._conv_forward(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) - conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) - return conv_fp8_matmul( - input, self.weight, self.bias, - self.sdnq_decompressor.scale, - self.sdnq_decompressor.result_shape, - self.sdnq_decompressor.weights_dtype, - self._reversed_padding_repeated_twice, - self.padding_mode, conv_type, - self.groups, stride, padding, dilation, - ) - - -def quantized_conv_forward_fp8_matmul_tensorwise(self, input) -> torch.FloatTensor: - if torch.numel(input) / input.shape[2] < 32: - return self._conv_forward(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) - conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) - return conv_fp8_matmul_tensorwise( - input, self.weight, self.bias, - self.sdnq_decompressor.scale, - self.sdnq_decompressor.result_shape, - self.sdnq_decompressor.weights_dtype, - self._reversed_padding_repeated_twice, - self.padding_mode, conv_type, - self.groups, stride, padding, dilation, - ) - - -def quantized_conv_forward_int8_matmul(self, input) -> torch.FloatTensor: - if torch.numel(input) / input.shape[2] < 32: - return self._conv_forward(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) - conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) - return conv_int8_matmul( - input, self.weight, self.bias, - self.sdnq_decompressor.scale, - self.sdnq_decompressor.result_shape, - getattr(self.sdnq_decompressor, "compressed_weight_shape", None), - self.sdnq_decompressor.weights_dtype, - self._reversed_padding_repeated_twice, - self.padding_mode, conv_type, - self.groups, stride, padding, dilation, - ) - - -def quantized_conv_forward(self, input) -> torch.FloatTensor: - return self._conv_forward(input, self.sdnq_decompressor(self.weight), self.bias) - - -def quantized_conv_transpose_1d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: - output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 1, self.dilation) - return torch.nn.functional.conv_transpose1d(input, self.sdnq_decompressor(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) - - -def quantized_conv_transpose_2d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: - output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 2, self.dilation) - return torch.nn.functional.conv_transpose2d(input, self.sdnq_decompressor(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) - - -def quantized_conv_transpose_3d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: - output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 3, self.dilation) - return torch.nn.functional.conv_transpose3d(input, self.sdnq_decompressor(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) - - -class AsymmetricWeightsDecompressor(torch.nn.Module): - def __init__( - self, - scale: torch.Tensor, - zero_point: torch.Tensor, - result_dtype: torch.dtype, - result_shape: torch.Size, - weights_dtype: str, - **kwargs, - ): - super().__init__() - self.weights_dtype = weights_dtype - self.use_quantized_matmul = False - self.result_dtype = result_dtype - self.result_shape = result_shape - self.register_buffer("scale", scale) - self.register_buffer("zero_point", zero_point) - - def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: - return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) - - def forward(self, weight, **kwargs): - return decompress_asymmetric_compiled(weight, self.scale, self.zero_point, self.result_dtype, self.result_shape) - - -class SymmetricWeightsDecompressor(torch.nn.Module): - def __init__( - self, - scale: torch.Tensor, - result_dtype: torch.dtype, - result_shape: torch.Size, - weights_dtype: str, - use_quantized_matmul: bool = False, - **kwargs, - ): - super().__init__() - self.weights_dtype = weights_dtype - self.use_quantized_matmul = use_quantized_matmul - self.result_dtype = result_dtype - self.result_shape = result_shape - self.register_buffer("scale", scale) - - def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: - return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) - - def forward(self, weight, skip_quantized_matmul=False, **kwargs): - return decompress_symmetric_compiled(weight, self.scale, self.result_dtype, self.result_shape, skip_quantized_matmul=skip_quantized_matmul) - - -class PackedINTAsymmetricWeightsDecompressor(torch.nn.Module): - def __init__( - self, - scale: torch.Tensor, - zero_point: torch.Tensor, - compressed_weight_shape: torch.Size, - result_dtype: torch.dtype, - result_shape: torch.Size, - weights_dtype: str, - **kwargs, - ): - super().__init__() - self.weights_dtype = weights_dtype - self.use_quantized_matmul = False - self.compressed_weight_shape = compressed_weight_shape - self.result_dtype = result_dtype - self.result_shape = result_shape - self.register_buffer("scale", scale) - self.register_buffer("zero_point", zero_point) - - def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: - return packed_int_function_dict[self.weights_dtype]["pack"](weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"])) - - def forward(self, weight, **kwargs): - return decompress_packed_int_asymmetric_compiled(weight, self.scale, self.zero_point, self.compressed_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype) - - -class PackedINTSymmetricWeightsDecompressor(torch.nn.Module): - def __init__( - self, - scale: torch.Tensor, - compressed_weight_shape: torch.Size, - result_dtype: torch.dtype, - result_shape: torch.Size, - weights_dtype: str, - use_quantized_matmul: bool = False, - **kwargs, - ): - super().__init__() - self.weights_dtype = weights_dtype - self.use_quantized_matmul = use_quantized_matmul - self.compressed_weight_shape = compressed_weight_shape - self.result_dtype = result_dtype - self.result_shape = result_shape - self.register_buffer("scale", scale) - - def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: - return pack_int_symetric(weight, self.weights_dtype) - - def forward(self, weight, skip_quantized_matmul=False, **kwargs): - return decompress_packed_int_symmetric_compiled(weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype, skip_quantized_matmul=skip_quantized_matmul) - - -decompressor_dict = { - "int8": SymmetricWeightsDecompressor, - "int6": PackedINTSymmetricWeightsDecompressor, - "int5": PackedINTSymmetricWeightsDecompressor, - "int4": PackedINTSymmetricWeightsDecompressor, - "int3": PackedINTSymmetricWeightsDecompressor, - "int2": PackedINTSymmetricWeightsDecompressor, - "uint8": AsymmetricWeightsDecompressor, - "uint6": PackedINTAsymmetricWeightsDecompressor, - "uint5": PackedINTAsymmetricWeightsDecompressor, - "uint4": PackedINTAsymmetricWeightsDecompressor, - "uint3": PackedINTAsymmetricWeightsDecompressor, - "uint2": PackedINTAsymmetricWeightsDecompressor, - "uint1": AsymmetricWeightsDecompressor, - "bool": AsymmetricWeightsDecompressor, - "float8_e4m3fn": SymmetricWeightsDecompressor, - "float8_e4m3fnuz": SymmetricWeightsDecompressor, - "float8_e5m2": SymmetricWeightsDecompressor, - "float8_e5m2fnuz": SymmetricWeightsDecompressor, -} - - -packed_int_function_dict = { - "int6": {"pack": pack_uint6, "unpack": unpack_uint6}, - "int5": {"pack": pack_uint5, "unpack": unpack_uint5}, - "int4": {"pack": pack_uint4, "unpack": unpack_uint4}, - "int3": {"pack": pack_uint3, "unpack": unpack_uint3}, - "int2": {"pack": pack_uint2, "unpack": unpack_uint2}, - "uint6": {"pack": pack_uint6, "unpack": unpack_uint6}, - "uint5": {"pack": pack_uint5, "unpack": unpack_uint5}, - "uint4": {"pack": pack_uint4, "unpack": unpack_uint4}, - "uint3": {"pack": pack_uint3, "unpack": unpack_uint3}, - "uint2": {"pack": pack_uint2, "unpack": unpack_uint2}, -} - - -class SDNQQuantizer(DiffusersQuantizer): - r""" - Diffusers Quantizer for SDNQ - """ - - requires_parameters_quantization = True - use_keep_in_fp32_modules = True - requires_calibration = False - required_packages = None - torch_dtype = None - - def __init__(self, quantization_config, **kwargs): # pylint: disable=useless-parent-delegation - super().__init__(quantization_config, **kwargs) - - def check_if_quantized_param( - self, - model, - param_value: "torch.Tensor", - param_name: str, - state_dict: Dict[str, Any], - **kwargs, - ): - if param_name.endswith(".weight"): - split_param_name = param_name.split(".") - if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): - layer_class_name = get_module_from_name(model, param_name)[0].__class__.__name__ - if layer_class_name in allowed_types: - if layer_class_name in conv_types or layer_class_name in conv_transpose_types: - if self.quantization_config.quant_conv: - return True - else: - return True - param_value.data = param_value.clone() # safetensors is unable to release the cpu memory without this - return False - - def check_quantized_param(self, *args, **kwargs) -> bool: - """ - needed for transformers compatibilty, returns self.check_if_quantized_param - """ - return self.check_if_quantized_param(*args, **kwargs) - - def create_quantized_param( # pylint: disable=arguments-differ - self, - model, - param_value: torch.FloatTensor, - param_name: str, - target_device: torch.device, - state_dict: Dict[str, Any], # pylint: disable=unused-argument - unexpected_keys: List[str], # pylint: disable=unused-argument - **kwargs, - ): - # load the model params to target_device first - layer, _ = get_module_from_name(model, param_name) - if shared.opts.sdnq_quantize_with_gpu: - if param_value.dtype == torch.float32 and devices.same_device(param_value.device, devices.device): - param_value = param_value.clone() - else: - param_value = param_value.to(devices.device).to(dtype=torch.float32) - else: - if param_value.dtype == torch.float32 and devices.same_device(param_value.device, target_device): - param_value = param_value.clone() - else: - param_value = param_value.to(target_device).to(dtype=torch.float32) - layer.weight = torch.nn.Parameter(param_value, requires_grad=False) - layer = sdnq_quantize_layer( - layer, - weights_dtype=self.quantization_config.weights_dtype, - torch_dtype=self.torch_dtype, - group_size=self.quantization_config.group_size, - quant_conv=self.quantization_config.quant_conv, - use_quantized_matmul=self.quantization_config.use_quantized_matmul, - use_quantized_matmul_conv=self.quantization_config.use_quantized_matmul_conv, - param_name=param_name, - pre_mode=True, - ) - - def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: - max_memory = {key: val * 0.80 for key, val in max_memory.items()} - return max_memory - - def adjust_target_dtype(self, target_dtype: torch.dtype) -> torch.dtype: # pylint: disable=unused-argument,arguments-renamed - return dtype_dict[self.quantization_config.weights_dtype]["target_dtype"] - - def update_torch_dtype(self, torch_dtype: torch.dtype = None) -> torch.dtype: - if torch_dtype is None: - torch_dtype = devices.dtype - self.torch_dtype = torch_dtype - return torch_dtype - - def _process_model_before_weight_loading( # pylint: disable=arguments-differ - self, - model, - device_map, # pylint: disable=unused-argument - keep_in_fp32_modules: List[str] = [], - **kwargs, - ): - model.config.quantization_config = self.quantization_config - self.modules_to_not_convert = self.quantization_config.modules_to_not_convert - if not isinstance(self.modules_to_not_convert, list): - self.modules_to_not_convert = [self.modules_to_not_convert] - if keep_in_fp32_modules is not None: - self.modules_to_not_convert.extend(keep_in_fp32_modules) - - def _process_model_after_weight_loading(self, model, **kwargs): - if shared.opts.diffusers_offload_mode != "none": - model = model.to(devices.cpu) - devices.torch_gc(force=True) - return model - - def get_cuda_warm_up_factor(self): - return 32 // dtype_dict[self.quantization_config.weights_dtype]["num_bits"] - - def update_tp_plan(self, config): - """ - needed for transformers compatibilty, no-op function - """ - return config - - def update_unexpected_keys(self, model, unexpected_keys: List[str], prefix: str) -> List[str]: # pylint: disable=unused-argument - """ - needed for transformers compatibilty, no-op function - """ - return unexpected_keys - - def update_missing_keys_after_loading(self, model, missing_keys: List[str], prefix: str) -> List[str]: # pylint: disable=unused-argument - """ - needed for transformers compatibilty, no-op function - """ - return missing_keys - - def update_expected_keys(self, model, expected_keys: List[str], loaded_keys: List[str]) -> List[str]: # pylint: disable=unused-argument - """ - needed for transformers compatibilty, no-op function - """ - return expected_keys - - @property - def is_trainable(self): - return False - - @property - def is_serializable(self): - return False - - -@dataclass -class SDNQConfig(QuantizationConfigMixin): - """ - This is a wrapper class about all possible attributes and features that you can play with a model that has been - loaded using `sdnq`. - - Args: - weights_dtype (`str`, *optional*, defaults to `"int8"`): - The target dtype for the weights after quantization. Supported values are: - ("int8", "int6", "int5", "int4", "int3", "int2", "uint8", "uint6", "uint5", "uint4", "uint3", "uint2", "uint1", "bool", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz") - modules_to_not_convert (`list`, *optional*, default to `None`): - The list of modules to not quantize, useful for quantizing models that explicitly require to have some - modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers). - """ - - def __init__( # pylint: disable=super-init-not-called - self, - weights_dtype: str = "int8", - group_size: int = 0, - quant_conv: bool = False, - use_quantized_matmul: bool = False, - use_quantized_matmul_conv: bool = False, - modules_to_not_convert: Optional[List[str]] = None, - **kwargs, # pylint: disable=unused-argument - ): - self.weights_dtype = weights_dtype - self.quant_method = QuantizationMethod.SDNQ - self.group_size = group_size - self.quant_conv = quant_conv - self.use_quantized_matmul = use_quantized_matmul - self.use_quantized_matmul_conv = use_quantized_matmul_conv - self.modules_to_not_convert = modules_to_not_convert - self.post_init() - self.is_integer = dtype_dict[self.weights_dtype]["is_integer"] - - def post_init(self): - r""" - Safety checker that arguments are correct - """ - accepted_weights = ["int8", "int6", "int5", "int4", "int3", "int2", "uint8", "uint6", "uint5", "uint4", "uint3", "uint2", "uint1", "bool", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz"] - if self.weights_dtype not in accepted_weights: - raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}") - - -class SDNQ_T5DenseGatedActDense(torch.nn.Module): # forward can't find what self is without creating a class - def __init__(self, T5DenseGatedActDense, dtype): - super().__init__() - self.wi_0 = T5DenseGatedActDense.wi_0 - self.wi_1 = T5DenseGatedActDense.wi_1 - self.wo = T5DenseGatedActDense.wo - self.dropout = T5DenseGatedActDense.dropout - self.act = T5DenseGatedActDense.act - self.torch_dtype = dtype - - def forward(self, hidden_states): - hidden_gelu = self.act(self.wi_0(hidden_states)) - hidden_linear = self.wi_1(hidden_states) - hidden_states = hidden_gelu * hidden_linear - hidden_states = self.dropout(hidden_states) - hidden_states = hidden_states.to(self.torch_dtype) # this line needs to be forced - hidden_states = self.wo(hidden_states) - return hidden_states - - -if shared.opts.sdnq_decompress_compile: - try: - torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) - decompress_asymmetric_compiled = torch.compile(decompress_asymmetric, fullgraph=True) - decompress_symmetric_compiled = torch.compile(decompress_symmetric, fullgraph=True) - decompress_packed_int_asymmetric_compiled = torch.compile(decompress_packed_int_asymmetric, fullgraph=True) - decompress_packed_int_symmetric_compiled = torch.compile(decompress_packed_int_symmetric, fullgraph=True) - int8_matmul = torch.compile(int8_matmul, fullgraph=True) - fp8_matmul = torch.compile(fp8_matmul, fullgraph=True) - fp8_matmul_tensorwise = torch.compile(fp8_matmul_tensorwise, fullgraph=True) - conv_int8_matmul = torch.compile(conv_int8_matmul, fullgraph=True) - conv_fp8_matmul = torch.compile(conv_fp8_matmul, fullgraph=True) - conv_fp8_matmul_tensorwise = torch.compile(conv_fp8_matmul_tensorwise, fullgraph=True) - except Exception as e: - shared.log.warning(f"Quantization: type=sdnq Decompress using torch.compile is not available: {e}") - decompress_asymmetric_compiled = decompress_asymmetric - decompress_symmetric_compiled = decompress_symmetric - decompress_packed_int_asymmetric_compiled = decompress_packed_int_asymmetric - decompress_packed_int_symmetric_compiled = decompress_packed_int_symmetric -else: - decompress_asymmetric_compiled = decompress_asymmetric - decompress_symmetric_compiled = decompress_symmetric - decompress_packed_int_asymmetric_compiled = decompress_packed_int_asymmetric - decompress_packed_int_symmetric_compiled = decompress_packed_int_symmetric diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py new file mode 100644 index 000000000..fde0b2a26 --- /dev/null +++ b/modules/sdnq/__init__.py @@ -0,0 +1,428 @@ +# pylint: disable=redefined-builtin,no-member,protected-access + +from typing import Any, Dict, List, Tuple, Optional, Union +from dataclasses import dataclass +from enum import Enum +import torch +from diffusers.quantizers.base import DiffusersQuantizer +from diffusers.quantizers.quantization_config import QuantizationConfigMixin +from diffusers.utils import get_module_from_name +from modules import devices, shared + +from .common import dtype_dict, use_tensorwise_fp8_matmul, quantized_matmul_dtypes, allowed_types, conv_types, conv_transpose_types +from .decompressor import decompressor_dict +from .forward import get_forward_func + + +def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, param_name=None, pre_mode=False): + layer_class_name = layer.__class__.__name__ + if layer_class_name in allowed_types: + is_conv_type = False + is_conv_transpose_type = False + is_linear_type = False + result_shape = None + if torch_dtype is None: + torch_dtype = devices.dtype + + if layer_class_name in conv_types: + if not quant_conv: + return layer + if dtype_dict[weights_dtype]["num_bits"] < 4: + weights_dtype = "uint4" + is_conv_type = True + reduction_axes = 1 + output_channel_size, channel_size = layer.weight.shape[:2] + group_channel_size = channel_size // layer.groups + use_quantized_matmul = False + if use_quantized_matmul_conv: + use_quantized_matmul = weights_dtype in quantized_matmul_dtypes and group_channel_size >= 32 and output_channel_size >= 32 + if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: + use_quantized_matmul = output_channel_size % 16 == 0 and group_channel_size % 16 == 0 + if use_quantized_matmul: + result_shape = layer.weight.shape + layer.weight.data = layer.weight.reshape(output_channel_size, -1) + elif layer_class_name in conv_transpose_types: + if not quant_conv: + return layer + if dtype_dict[weights_dtype]["num_bits"] < 4: + weights_dtype = "uint4" + is_conv_transpose_type = True + reduction_axes = 0 + channel_size, output_channel_size = layer.weight.shape[:2] + use_quantized_matmul = False + else: + is_linear_type = True + reduction_axes = -1 + output_channel_size, channel_size = layer.weight.shape + if use_quantized_matmul: + use_quantized_matmul = weights_dtype in quantized_matmul_dtypes and channel_size >= 32 and output_channel_size >= 32 + if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: + use_quantized_matmul = output_channel_size % 16 == 0 and channel_size % 16 == 0 + + if group_size == 0: + if is_linear_type: + group_size = 2 ** (2 + dtype_dict[weights_dtype]["num_bits"]) + else: + group_size = 2 ** (1 + dtype_dict[weights_dtype]["num_bits"]) + + if not use_quantized_matmul and group_size > 0: + if group_size >= channel_size: + group_size = channel_size + num_of_groups = 1 + else: + num_of_groups = channel_size // group_size + while channel_size % group_size != 0: # find something divisible + num_of_groups -= 1 + if num_of_groups <= 1: + group_size = channel_size + num_of_groups = 1 + break + group_size = channel_size / num_of_groups + group_size = int(group_size) + num_of_groups = int(num_of_groups) + + if num_of_groups > 1: + result_shape = layer.weight.shape + new_shape = list(result_shape) + if is_conv_type: + # output_channel_size, channel_size, X, X + # output_channel_size, num_of_groups, group_size, X, X + new_shape[1] = group_size + new_shape.insert(1, num_of_groups) + reduction_axes = 2 + elif is_conv_transpose_type: + #channel_size, output_channel_size, X, X + #num_of_groups, group_size, output_channel_size, X, X + new_shape[0] = group_size + new_shape.insert(0, num_of_groups) + reduction_axes = 1 + elif is_linear_type: + # output_channel_size, channel_size + # output_channel_size, num_of_groups, group_size + last_dim_index = layer.weight.ndim + new_shape[last_dim_index - 1 : last_dim_index] = (num_of_groups, group_size) + layer.weight.data = layer.weight.reshape(new_shape) + + layer.weight.requires_grad = False + if shared.opts.diffusers_offload_mode in {"none", "model"}: + return_device = devices.device + elif pre_mode: + if shared.opts.device_map == "gpu": + return_device = devices.device + elif shared.opts.sdnq_quantize_with_gpu: + return_device = devices.cpu + else: + return_device = layer.weight.device + else: + return_device = layer.weight.device + if not pre_mode: + if shared.opts.sdnq_quantize_with_gpu: + layer.weight.data = layer.weight.to(devices.device).to(dtype=torch.float32) + else: + layer.weight.data = layer.weight.to(dtype=torch.float32) + + if dtype_dict[weights_dtype]["is_unsigned"]: + scale, zero_point = get_scale_asymmetric(layer.weight, reduction_axes, weights_dtype) + layer.weight.data.sub_(zero_point).div_(scale) + else: + scale = get_scale_symmetric(layer.weight, reduction_axes, weights_dtype) + layer.weight.data.div_(scale) + zero_point = None + if dtype_dict[weights_dtype]["is_integer"]: + layer.weight.data.round_() + layer.weight.data = layer.weight.data.clamp_(dtype_dict[weights_dtype]["min"], dtype_dict[weights_dtype]["max"]).to(dtype_dict[weights_dtype]["torch_dtype"]) + + if not shared.opts.sdnq_decompress_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"] and not use_tensorwise_fp8_matmul): + scale = scale.to(torch_dtype) + if zero_point is not None: + zero_point = zero_point.to(torch_dtype) + + if use_quantized_matmul: + scale = scale.transpose(0,1) + if dtype_dict[weights_dtype]["num_bits"] == 8: + layer.weight.data = layer.weight.transpose(0,1) + if not dtype_dict[weights_dtype]["is_integer"]: + stride = layer.weight.stride() + if stride[0] > stride[1] and stride[1] == 1: + layer.weight.data = layer.weight.t().contiguous().t() + if not use_tensorwise_fp8_matmul: + scale = scale.to(torch.float32) + + layer.sdnq_decompressor = decompressor_dict[weights_dtype]( + scale=scale, + zero_point=zero_point, + compressed_weight_shape=layer.weight.shape, + result_dtype=torch_dtype, + result_shape=result_shape, + weights_dtype=weights_dtype, + use_quantized_matmul=use_quantized_matmul, + ) + layer.weight.data = layer.sdnq_decompressor.pack_weight(layer.weight).to(return_device) + layer.sdnq_decompressor = layer.sdnq_decompressor.to(return_device) + + layer.forward = get_forward_func(layer_class_name, use_quantized_matmul, dtype_dict[weights_dtype]["is_integer"], use_tensorwise_fp8_matmul) + layer.forward = layer.forward.__get__(layer, layer.__class__) + devices.torch_gc(force=False, reason=f"SDNQ param_name: {param_name}") + return layer + + +def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, param_name=None): # pylint: disable=unused-argument + has_children = list(model.children()) + if not has_children: + return model + for module_param_name, module in model.named_children(): + if hasattr(module, "weight") and module.weight is not None: + module = sdnq_quantize_layer( + module, + weights_dtype=weights_dtype, + torch_dtype=torch_dtype, + group_size=group_size, + quant_conv=quant_conv, + use_quantized_matmul=use_quantized_matmul, + use_quantized_matmul_conv=use_quantized_matmul_conv, + param_name=module_param_name, + ) + module = apply_sdnq_to_module( + module, + weights_dtype=weights_dtype, + torch_dtype=torch_dtype, + group_size=group_size, + quant_conv=quant_conv, + use_quantized_matmul=use_quantized_matmul, + use_quantized_matmul_conv=use_quantized_matmul_conv, + param_name=module_param_name, + ) + return model + + +def get_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> Tuple[torch.FloatTensor, torch.FloatTensor]: + zero_point = torch.amin(weight, dim=reduction_axes, keepdims=True) + scale = torch.amax(weight, dim=reduction_axes, keepdims=True).sub_(zero_point).div_(dtype_dict[weights_dtype]["max"] - dtype_dict[weights_dtype]["min"]) + eps = torch.finfo(scale.dtype).eps # prevent divison by 0 + scale = torch.where(torch.abs(scale) < eps, eps, scale) + if dtype_dict[weights_dtype]["min"] != 0: + zero_point.sub_(torch.mul(scale, dtype_dict[weights_dtype]["min"])) + return scale, zero_point + + +def get_scale_symmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> torch.FloatTensor: + abs_min_values = torch.amin(weight, dim=reduction_axes, keepdims=True).abs_() + max_values = torch.amax(weight, dim=reduction_axes, keepdims=True) + scale = torch.where(abs_min_values >= max_values, abs_min_values, -max_values).div_(dtype_dict[weights_dtype]["max"]) + eps = torch.finfo(scale.dtype).eps # prevent divison by 0 + scale = torch.where(torch.abs(scale) < eps, eps, scale) + return scale + + +class QuantizationMethod(str, Enum): + SDNQ = "sdnq" + + +class SDNQQuantizer(DiffusersQuantizer): + r""" + Diffusers Quantizer for SDNQ + """ + + requires_parameters_quantization = True + use_keep_in_fp32_modules = True + requires_calibration = False + required_packages = None + torch_dtype = None + + def __init__(self, quantization_config, **kwargs): # pylint: disable=useless-parent-delegation + super().__init__(quantization_config, **kwargs) + self.modules_to_not_convert = [] + + def check_if_quantized_param( + self, + model, + param_value: "torch.Tensor", + param_name: str, + state_dict: Dict[str, Any], # pylint: disable=unused-argument + **kwargs, # pylint: disable=unused-argument + ): + if param_name.endswith(".weight"): + split_param_name = param_name.split(".") + if param_name not in self.modules_to_not_convert and not any(param in split_param_name for param in self.modules_to_not_convert): + layer_class_name = get_module_from_name(model, param_name)[0].__class__.__name__ + if layer_class_name in allowed_types: + if layer_class_name in conv_types or layer_class_name in conv_transpose_types: + if self.quantization_config.quant_conv: + return True + else: + return True + param_value.data = param_value.clone() # safetensors is unable to release the cpu memory without this + return False + + def check_quantized_param(self, *args, **kwargs) -> bool: + """ + needed for transformers compatibilty, returns self.check_if_quantized_param + """ + return self.check_if_quantized_param(*args, **kwargs) + + def create_quantized_param( # pylint: disable=arguments-differ + self, + model, + param_value: torch.FloatTensor, + param_name: str, + target_device: torch.device, + state_dict: Dict[str, Any], # pylint: disable=unused-argument + unexpected_keys: List[str], # pylint: disable=unused-argument + **kwargs, # pylint: disable=unused-argument + ): + # load the model params to target_device first + layer, _ = get_module_from_name(model, param_name) + if shared.opts.sdnq_quantize_with_gpu: + if param_value.dtype == torch.float32 and devices.same_device(param_value.device, devices.device): + param_value = param_value.clone() + else: + param_value = param_value.to(devices.device).to(dtype=torch.float32) + else: + if param_value.dtype == torch.float32 and devices.same_device(param_value.device, target_device): + param_value = param_value.clone() + else: + param_value = param_value.to(target_device).to(dtype=torch.float32) + layer.weight = torch.nn.Parameter(param_value, requires_grad=False) + layer = sdnq_quantize_layer( + layer, + weights_dtype=self.quantization_config.weights_dtype, + torch_dtype=self.torch_dtype, + group_size=self.quantization_config.group_size, + quant_conv=self.quantization_config.quant_conv, + use_quantized_matmul=self.quantization_config.use_quantized_matmul, + use_quantized_matmul_conv=self.quantization_config.use_quantized_matmul_conv, + param_name=param_name, + pre_mode=True, + ) + + def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: + max_memory = {key: val * 0.80 for key, val in max_memory.items()} + return max_memory + + def adjust_target_dtype(self, target_dtype: torch.dtype) -> torch.dtype: # pylint: disable=unused-argument,arguments-renamed + return dtype_dict[self.quantization_config.weights_dtype]["target_dtype"] + + def update_torch_dtype(self, torch_dtype: torch.dtype = None) -> torch.dtype: + if torch_dtype is None: + torch_dtype = devices.dtype + self.torch_dtype = torch_dtype + return torch_dtype + + def _process_model_before_weight_loading( # pylint: disable=arguments-differ + self, + model, + device_map, # pylint: disable=unused-argument + keep_in_fp32_modules: List[str] = [], + **kwargs, # pylint: disable=unused-argument + ): + model.config.quantization_config = self.quantization_config + self.modules_to_not_convert.extend(self.quantization_config.modules_to_not_convert) + if keep_in_fp32_modules is not None: + self.modules_to_not_convert.extend(keep_in_fp32_modules) + + def _process_model_after_weight_loading(self, model, **kwargs): # pylint: disable=unused-argument + if shared.opts.diffusers_offload_mode != "none": + model = model.to(devices.cpu) + devices.torch_gc(force=True) + return model + + def get_cuda_warm_up_factor(self): + return 32 // dtype_dict[self.quantization_config.weights_dtype]["num_bits"] + + def update_tp_plan(self, config): + """ + needed for transformers compatibilty, no-op function + """ + return config + + def update_unexpected_keys(self, model, unexpected_keys: List[str], prefix: str) -> List[str]: # pylint: disable=unused-argument + """ + needed for transformers compatibilty, no-op function + """ + return unexpected_keys + + def update_missing_keys_after_loading(self, model, missing_keys: List[str], prefix: str) -> List[str]: # pylint: disable=unused-argument + """ + needed for transformers compatibilty, no-op function + """ + return missing_keys + + def update_expected_keys(self, model, expected_keys: List[str], loaded_keys: List[str]) -> List[str]: # pylint: disable=unused-argument + """ + needed for transformers compatibilty, no-op function + """ + return expected_keys + + @property + def is_trainable(self): + return False + + @property + def is_serializable(self): + return False + + +@dataclass +class SDNQConfig(QuantizationConfigMixin): + """ + This is a wrapper class about all possible attributes and features that you can play with a model that has been + loaded using `sdnq`. + + Args: + weights_dtype (`str`, *optional*, defaults to `"int8"`): + The target dtype for the weights after quantization. Supported values are: + ("int8", "int6", "int5", "int4", "int3", "int2", "uint8", "uint6", "uint5", "uint4", "uint3", "uint2", "uint1", "bool", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz") + modules_to_not_convert (`list`, *optional*, default to `None`): + The list of modules to not quantize, useful for quantizing models that explicitly require to have some + modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers). + """ + + def __init__( # pylint: disable=super-init-not-called + self, + weights_dtype: str = "int8", + group_size: int = 0, + quant_conv: bool = False, + use_quantized_matmul: bool = False, + use_quantized_matmul_conv: bool = False, + modules_to_not_convert: Optional[List[str]] = None, + **kwargs, # pylint: disable=unused-argument + ): + self.weights_dtype = weights_dtype + self.quant_method = QuantizationMethod.SDNQ + self.group_size = group_size + self.quant_conv = quant_conv + self.use_quantized_matmul = use_quantized_matmul + self.use_quantized_matmul_conv = use_quantized_matmul_conv + self.modules_to_not_convert = modules_to_not_convert + self.post_init() + self.is_integer = dtype_dict[self.weights_dtype]["is_integer"] + + def post_init(self): + r""" + Safety checker that arguments are correct + """ + accepted_weights = ["int8", "int6", "int5", "int4", "int3", "int2", "uint8", "uint6", "uint5", "uint4", "uint3", "uint2", "uint1", "bool", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz"] + if self.weights_dtype not in accepted_weights: + raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}") + if not isinstance(self.modules_to_not_convert, list): + self.modules_to_not_convert = [self.modules_to_not_convert] + + +class SDNQ_T5DenseGatedActDense(torch.nn.Module): # forward can't find what self is without creating a class + def __init__(self, T5DenseGatedActDense, dtype): + super().__init__() + self.wi_0 = T5DenseGatedActDense.wi_0 + self.wi_1 = T5DenseGatedActDense.wi_1 + self.wo = T5DenseGatedActDense.wo + self.dropout = T5DenseGatedActDense.dropout + self.act = T5DenseGatedActDense.act + self.torch_dtype = dtype + + def forward(self, hidden_states): + hidden_gelu = self.act(self.wi_0(hidden_states)) + hidden_linear = self.wi_1(hidden_states) + hidden_states = hidden_gelu * hidden_linear + hidden_states = self.dropout(hidden_states) + hidden_states = hidden_states.to(self.torch_dtype) # this line needs to be forced + hidden_states = self.wo(hidden_states) + return hidden_states diff --git a/modules/sdnq/common.py b/modules/sdnq/common.py new file mode 100644 index 000000000..70addbe7a --- /dev/null +++ b/modules/sdnq/common.py @@ -0,0 +1,41 @@ +# pylint: disable=redefined-builtin,no-member,protected-access + +import sys +import torch +from accelerate.utils import CustomDtype +from modules import devices + +torch_version = float(torch.__version__[:3]) + +dtype_dict = { + "int8": {"min": -128, "max": 127, "num_bits": 8, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.int8, "is_unsigned": False, "is_integer": True}, + "int7": {"min": -64, "max": 63, "num_bits": 6, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, + "int6": {"min": -32, "max": 31, "num_bits": 6, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, + "int5": {"min": -16, "max": 15, "num_bits": 5, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, + "int4": {"min": -8, "max": 7, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, + "int3": {"min": -4, "max": 3, "num_bits": 3, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, + "int2": {"min": -2, "max": 1, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, + "uint8": {"min": 0, "max": 255, "num_bits": 8, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint7": {"min": 0, "max": 127, "num_bits": 6, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint6": {"min": 0, "max": 63, "num_bits": 6, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint5": {"min": 0, "max": 31, "num_bits": 5, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint4": {"min": 0, "max": 15, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint3": {"min": 0, "max": 7, "num_bits": 3, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint2": {"min": 0, "max": 3, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint1": {"min": 0, "max": 1, "num_bits": 1, "target_dtype": torch.bool, "torch_dtype": torch.bool, "storage_dtype": torch.bool, "is_unsigned": True, "is_integer": True}, + "float8_e4m3fn": {"min": -448, "max": 448, "num_bits": 8, "target_dtype": torch.float8_e4m3fn, "torch_dtype": torch.float8_e4m3fn, "storage_dtype": torch.float8_e4m3fn, "is_unsigned": False, "is_integer": False}, + "float8_e5m2": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": torch.float8_e5m2, "torch_dtype": torch.float8_e5m2, "storage_dtype": torch.float8_e5m2, "is_unsigned": False, "is_integer": False}, + "float8_e4m3fnuz": {"min": -240, "max": 240, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e4m3fnuz, "storage_dtype": torch.float8_e4m3fnuz, "is_unsigned": False, "is_integer": False}, + "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, +} +dtype_dict["bool"] = dtype_dict["uint1"] + +use_tensorwise_fp8_matmul = torch_version < 2.5 or devices.backend in {"cpu", "openvino"} or (devices.backend == "cuda" and sys.platform == "win32" and torch_version <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) +quantized_matmul_dtypes = ("int8", "int6", "int5", "int4", "int3", "int2", "float8_e4m3fn", "float8_e5m2") +if devices.backend in {"cpu", "openvino"}: + quantized_matmul_dtypes += ("float8_e4m3fnuz", "float8_e5m2fnuz") + +linear_types = ("Linear",) +conv_types = ("Conv1d", "Conv2d", "Conv3d") +conv_transpose_types = ("ConvTranspose1d", "ConvTranspose2d", "ConvTranspose3d") +allowed_types = linear_types + conv_types + conv_transpose_types diff --git a/modules/sdnq/decompressor.py b/modules/sdnq/decompressor.py new file mode 100644 index 000000000..2c37f26b2 --- /dev/null +++ b/modules/sdnq/decompressor.py @@ -0,0 +1,170 @@ +# pylint: disable=redefined-builtin,no-member,protected-access + +import torch +from modules import shared + +from .common import dtype_dict +from .packed_int import pack_int_symetric, unpack_int_symetric, packed_int_function_dict + + +def decompress_asymmetric(input: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, dtype: torch.dtype, result_shape: torch.Size) -> torch.FloatTensor: + result = torch.addcmul(zero_point, input.to(dtype=scale.dtype), scale).to(dtype=dtype) + if result_shape is not None: + result = result.reshape(result_shape) + return result + + +def decompress_symmetric(input: torch.CharTensor, scale: torch.FloatTensor, dtype: torch.dtype, result_shape: torch.Size, skip_quantized_matmul: bool = False) -> torch.FloatTensor: + if skip_quantized_matmul: + result = input.transpose(0,1).to(dtype=scale.dtype).mul_(scale.transpose(0,1)).to(dtype=dtype) + else: + result = input.to(dtype=scale.dtype).mul_(scale).to(dtype=dtype) + if result_shape is not None: + result = result.reshape(result_shape) + return result + + +def decompress_packed_int_asymmetric(input: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str) -> torch.FloatTensor: + return decompress_asymmetric(packed_int_function_dict[weights_dtype]["unpack"](input, shape), scale, zero_point, dtype, result_shape) + + +def decompress_packed_int_symmetric(input: torch.ByteTensor, scale: torch.FloatTensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str, skip_quantized_matmul: bool = False) -> torch.FloatTensor: + if skip_quantized_matmul: + return decompress_symmetric(unpack_int_symetric(input, shape, weights_dtype, dtype=scale.dtype), scale.transpose(0,1), dtype, result_shape) + else: + return decompress_symmetric(unpack_int_symetric(input, shape, weights_dtype, dtype=scale.dtype), scale, dtype, result_shape) + + +class AsymmetricWeightsDecompressor(torch.nn.Module): + def __init__( + self, + scale: torch.Tensor, + zero_point: torch.Tensor, + result_dtype: torch.dtype, + result_shape: torch.Size, + weights_dtype: str, + **kwargs, # pylint: disable=unused-argument + ): + super().__init__() + self.weights_dtype = weights_dtype + self.use_quantized_matmul = False + self.result_dtype = result_dtype + self.result_shape = result_shape + self.register_buffer("scale", scale) + self.register_buffer("zero_point", zero_point) + + def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: + return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) + + def forward(self, weight, **kwargs): # pylint: disable=unused-argument + return decompress_asymmetric(weight, self.scale, self.zero_point, self.result_dtype, self.result_shape) + + +class SymmetricWeightsDecompressor(torch.nn.Module): + def __init__( + self, + scale: torch.Tensor, + result_dtype: torch.dtype, + result_shape: torch.Size, + weights_dtype: str, + use_quantized_matmul: bool = False, + **kwargs, # pylint: disable=unused-argument + ): + super().__init__() + self.weights_dtype = weights_dtype + self.use_quantized_matmul = use_quantized_matmul + self.result_dtype = result_dtype + self.result_shape = result_shape + self.register_buffer("scale", scale) + + def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: + return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) + + def forward(self, weight, skip_quantized_matmul=False, **kwargs): # pylint: disable=unused-argument + return decompress_symmetric(weight, self.scale, self.result_dtype, self.result_shape, skip_quantized_matmul=skip_quantized_matmul) + + +class PackedINTAsymmetricWeightsDecompressor(torch.nn.Module): + def __init__( + self, + scale: torch.Tensor, + zero_point: torch.Tensor, + compressed_weight_shape: torch.Size, + result_dtype: torch.dtype, + result_shape: torch.Size, + weights_dtype: str, + **kwargs, # pylint: disable=unused-argument + ): + super().__init__() + self.weights_dtype = weights_dtype + self.use_quantized_matmul = False + self.compressed_weight_shape = compressed_weight_shape + self.result_dtype = result_dtype + self.result_shape = result_shape + self.register_buffer("scale", scale) + self.register_buffer("zero_point", zero_point) + + def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: + return packed_int_function_dict[self.weights_dtype]["pack"](weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"])) + + def forward(self, weight, **kwargs): # pylint: disable=unused-argument + return decompress_packed_int_asymmetric(weight, self.scale, self.zero_point, self.compressed_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype) + + +class PackedINTSymmetricWeightsDecompressor(torch.nn.Module): + def __init__( + self, + scale: torch.Tensor, + compressed_weight_shape: torch.Size, + result_dtype: torch.dtype, + result_shape: torch.Size, + weights_dtype: str, + use_quantized_matmul: bool = False, + **kwargs, # pylint: disable=unused-argument + ): + super().__init__() + self.weights_dtype = weights_dtype + self.use_quantized_matmul = use_quantized_matmul + self.compressed_weight_shape = compressed_weight_shape + self.result_dtype = result_dtype + self.result_shape = result_shape + self.register_buffer("scale", scale) + + def pack_weight(self, weight: torch.Tensor) -> torch.Tensor: + return pack_int_symetric(weight, self.weights_dtype) + + def forward(self, weight, skip_quantized_matmul=False, **kwargs): # pylint: disable=unused-argument + return decompress_packed_int_symmetric(weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype, skip_quantized_matmul=skip_quantized_matmul) + + +decompressor_dict = { + "int8": SymmetricWeightsDecompressor, + "int6": PackedINTSymmetricWeightsDecompressor, + "int5": PackedINTSymmetricWeightsDecompressor, + "int4": PackedINTSymmetricWeightsDecompressor, + "int3": PackedINTSymmetricWeightsDecompressor, + "int2": PackedINTSymmetricWeightsDecompressor, + "uint8": AsymmetricWeightsDecompressor, + "uint6": PackedINTAsymmetricWeightsDecompressor, + "uint5": PackedINTAsymmetricWeightsDecompressor, + "uint4": PackedINTAsymmetricWeightsDecompressor, + "uint3": PackedINTAsymmetricWeightsDecompressor, + "uint2": PackedINTAsymmetricWeightsDecompressor, + "uint1": AsymmetricWeightsDecompressor, + "bool": AsymmetricWeightsDecompressor, + "float8_e4m3fn": SymmetricWeightsDecompressor, + "float8_e4m3fnuz": SymmetricWeightsDecompressor, + "float8_e5m2": SymmetricWeightsDecompressor, + "float8_e5m2fnuz": SymmetricWeightsDecompressor, +} + + +if shared.opts.sdnq_decompress_compile: + try: + torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) + decompress_asymmetric = torch.compile(decompress_asymmetric, fullgraph=True) + decompress_symmetric = torch.compile(decompress_symmetric, fullgraph=True) + decompress_packed_int_asymmetric = torch.compile(decompress_packed_int_asymmetric, fullgraph=True) + decompress_packed_int_symmetric = torch.compile(decompress_packed_int_symmetric, fullgraph=True) + except Exception as e: + shared.log.warning(f"Quantization: type=sdnq Decompress using torch.compile is not available: {e}") diff --git a/modules/sdnq/forward.py b/modules/sdnq/forward.py new file mode 100644 index 000000000..e5f345682 --- /dev/null +++ b/modules/sdnq/forward.py @@ -0,0 +1,402 @@ +# pylint: disable=redefined-builtin,no-member,protected-access + +from typing import Callable, List, Tuple, Optional +import torch +from modules import shared + +from .common import conv_types, conv_transpose_types +from .decompressor import decompress_symmetric +from .packed_int import unpack_int_symetric + + +def get_forward_func(layer_class_name: str, use_quantized_matmul: bool, is_integer: bool, use_tensorwise_fp8_matmul: bool) -> Callable: # pylint: disable=inconsistent-return-statements + if layer_class_name in conv_types: + if use_quantized_matmul: + if is_integer: + return quantized_conv_forward_int8_matmul + else: + if use_tensorwise_fp8_matmul: + return quantized_conv_forward_fp8_matmul_tensorwise + else: + return quantized_conv_forward_fp8_matmul + else: + return quantized_conv_forward + elif layer_class_name in conv_transpose_types: + if layer_class_name.endswith("1d"): + return quantized_conv_transpose_1d_forward + elif layer_class_name.endswith("2d"): + return quantized_conv_transpose_2d_forward + elif layer_class_name.endswith("3d"): + return quantized_conv_transpose_3d_forward + else: + if use_quantized_matmul: + if is_integer: + return quantized_linear_forward_int8_matmul + else: + if use_tensorwise_fp8_matmul: + return quantized_linear_forward_fp8_matmul_tensorwise + else: + return quantized_linear_forward_fp8_matmul + else: + return quantized_linear_forward + + +def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.Tensor, torch.FloatTensor]: + input = input.flatten(0,-2).contiguous() + input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 448) + input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn) + input_scale = input_scale.to(torch.float32) + return input, input_scale + + +def quantize_fp8_matmul_input_tensorwise(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.Tensor, torch.FloatTensor]: + input = input.flatten(0,-2).contiguous() + input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 448) + input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn) + scale = torch.mul(input_scale, scale) + if scale.dtype == torch.float16: # fp16 will overflow + scale = scale.to(dtype=torch.float32) + return input, scale + + +def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.CharTensor, torch.FloatTensor]: + input = input.flatten(0,-2).contiguous() + input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 127) + input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(torch.int8) + scale = torch.mul(input_scale, scale) + if scale.dtype == torch.float16: # fp16 will overflow + scale = scale.to(dtype=torch.float32) + return input, scale + + +def fp8_matmul( + input: torch.FloatTensor, + weight: torch.Tensor, + bias: torch.FloatTensor, + scale: torch.FloatTensor, +) -> torch.FloatTensor: + return_dtype = input.dtype + output_shape = list(input.shape) + output_shape[-1] = weight.shape[-1] + input, input_scale = quantize_fp8_matmul_input(input) + return torch._scaled_mm(input, weight, scale_a=input_scale, scale_b=scale, bias=bias, out_dtype=return_dtype).reshape(output_shape) + + +# sm89 doesn't support row wise scale in Windows +def fp8_matmul_tensorwise( + input: torch.FloatTensor, + weight: torch.Tensor, + bias: torch.FloatTensor, + scale: torch.FloatTensor, +) -> torch.FloatTensor: + return_dtype = input.dtype + output_shape = list(input.shape) + output_shape[-1] = weight.shape[-1] + dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) + input, scale = quantize_fp8_matmul_input_tensorwise(input, scale) + result = decompress_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, output_shape) + if bias is not None: + result.add_(bias) + return result + + +def int8_matmul( + input: torch.FloatTensor, + weight: torch.Tensor, + bias: torch.FloatTensor, + scale: torch.FloatTensor, + compressed_weight_shape: torch.Size, + weights_dtype: str, +) -> torch.FloatTensor: + if compressed_weight_shape is not None: + weight = unpack_int_symetric(weight, compressed_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) + return_dtype = input.dtype + output_shape = list(input.shape) + output_shape[-1] = weight.shape[-1] + input, scale = quantize_int8_matmul_input(input, scale) + result = decompress_symmetric(torch._int_mm(input, weight), scale, return_dtype, output_shape) + if bias is not None: + result.add_(bias) + return result + + +def process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation): + if conv_type == 1: + batch_size, _, L_in = input.shape + C_out, _, K_l = result_shape + L_out = (L_in + 2 * padding[1] - dilation[1] * (K_l - 1) - 1) // stride[1] + 1 + mm_output_shape = (batch_size, L_out, C_out) + kernel_size = (1, K_l) + if conv_type == 2: + batch_size, _, H_in, W_in = input.shape + C_out, _, K_h, K_w = result_shape + H_out = (H_in + 2 * padding[0] - dilation[0] * (K_h - 1) - 1) // stride[0] + 1 + W_out = (W_in + 2 * padding[1] - dilation[1] * (K_w - 1) - 1) // stride[1] + 1 + mm_output_shape = (batch_size, H_out, W_out, C_out) + kernel_size = (K_h, K_w) + else: + batch_size, _, D_in, H_in, W_in = input.shape + C_out, _, K_d, K_h, K_w = result_shape + D_out = (D_in + 2 * padding[0] - dilation[0] * (K_d - 1) - 1) // stride[0] + 1 + H_out = (H_in + 2 * padding[1] - dilation[1] * (K_h - 1) - 1) // stride[1] + 1 + W_out = (W_in + 2 * padding[2] - dilation[2] * (K_w - 1) - 1) // stride[2] + 1 + mm_output_shape = (batch_size, D_out, H_out, W_out, C_out) + kernel_size = (K_d, K_h, K_w) + + if padding_mode != "zeros": + input = torch.nn.functional.pad(input, reversed_padding_repeated_twice, mode=padding_mode) + padding = (0,) * (conv_type if conv_type != 1 else 2) + elif conv_type == 3: + input = torch.nn.functional.pad(input, reversed_padding_repeated_twice) + + if conv_type == 1: + input = input.unsqueeze(2) + + if conv_type == 3: + K_D_eff = K_d + (K_d - 1) * (dilation[0] - 1) + K_H_eff = K_h + (K_h - 1) * (dilation[0] - 1) + K_W_eff = K_w + (K_w - 1) * (dilation[0] - 1) + input = input.unfold(2, K_D_eff, stride[0]).unfold(3, K_H_eff, stride[1]).unfold(4, K_W_eff, stride[2]) + if dilation[0] > 1: + input = input[..., ::dilation[0], :, :] + if dilation[1] > 1: + input = input[..., ::dilation[1], :] + if dilation[2] > 1: + input = input[..., ::dilation[2]] + input = input.permute(0, 2, 3, 4, 1, 5, 6, 7).reshape(batch_size, D_out * H_out * W_out, -1) + else: + input = torch.nn.functional.unfold(input, kernel_size=kernel_size, padding=padding, stride=stride, dilation=dilation).transpose(1,2) + return input, mm_output_shape + + +def conv_fp8_matmul( + input: torch.FloatTensor, + weight: torch.Tensor, + bias: torch.FloatTensor, + scale: torch.FloatTensor, + result_shape: torch.Size, + reversed_padding_repeated_twice: List[int], + padding_mode: str, conv_type: int, + groups: int, stride: List[int], + padding: List[int], dilation: List[int], +) -> torch.FloatTensor: + return_dtype = input.dtype + input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) + input, input_scale = quantize_fp8_matmul_input(input) + + if groups == 1: + result = torch._scaled_mm(input, weight, scale_a=input_scale, scale_b=scale, bias=bias, out_dtype=return_dtype).reshape(mm_output_shape) + else: + scale = scale.reshape(groups, 1, scale.shape[1] // groups) + input_scale = input_scale.reshape(groups, input_scale.shape[0] // groups, 1) + weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) + input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) + result = [] + for i in range(groups): + result.append(torch._scaled_mm(input[i], weight[i], scale_a=input_scale[i], scale_b=scale[i], bias=None, out_dtype=return_dtype)) + result = torch.cat(result, dim=-1).reshape(mm_output_shape) + if bias is not None: + result.add_(bias) + + if conv_type == 1: + result = result.transpose(1,2) + elif conv_type == 2: + result = result.permute(0,3,1,2) + elif conv_type == 3: + result = result.permute(0,4,1,2,3) + return result + + +def conv_fp8_matmul_tensorwise( + input: torch.FloatTensor, + weight: torch.Tensor, + bias: torch.FloatTensor, + scale: torch.FloatTensor, + result_shape: torch.Size, + reversed_padding_repeated_twice: List[int], + padding_mode: str, conv_type: int, + groups: int, stride: List[int], + padding: List[int], dilation: List[int], +) -> torch.FloatTensor: + return_dtype = input.dtype + input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) + input, scale = quantize_fp8_matmul_input_tensorwise(input, scale) + dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) + + if groups == 1: + result = decompress_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, mm_output_shape) + else: + weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) + input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) + result = [] + for i in range(groups): + result.append(torch._scaled_mm(input[i], weight[i], scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype)) + result = decompress_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) + if bias is not None: + result.add_(bias) + + if conv_type == 1: + result = result.transpose(1,2) + elif conv_type == 2: + result = result.permute(0,3,1,2) + elif conv_type == 3: + result = result.permute(0,4,1,2,3) + return result + + +def conv_int8_matmul( + input: torch.FloatTensor, + weight: torch.CharTensor, + bias: torch.FloatTensor, + scale: torch.FloatTensor, + result_shape: torch.Size, + compressed_weight_shape: torch.Size, + weights_dtype: str, + reversed_padding_repeated_twice: List[int], + padding_mode: str, conv_type: int, + groups: int, stride: List[int], + padding: List[int], dilation: List[int], +) -> torch.FloatTensor: + return_dtype = input.dtype + input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) + input, scale = quantize_int8_matmul_input(input, scale) + if compressed_weight_shape is not None: + weight = unpack_int_symetric(weight, compressed_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) + + if groups == 1: + result = decompress_symmetric(torch._int_mm(input, weight), scale, return_dtype, mm_output_shape) + else: + weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) + input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) + result = [] + for i in range(groups): + result.append(torch._int_mm(input[i], weight[i])) + result = decompress_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) + if bias is not None: + result.add_(bias) + + if conv_type == 1: + result = result.transpose(1,2) + elif conv_type == 2: + result = result.permute(0,3,1,2) + elif conv_type == 3: + result = result.permute(0,4,1,2,3) + return result + + +def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: + if torch.numel(input) / input.shape[-1] < 32: + return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) + return fp8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale) + + +def quantized_linear_forward_fp8_matmul_tensorwise(self, input: torch.FloatTensor) -> torch.FloatTensor: + if torch.numel(input) / input.shape[-1] < 32: + return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) + return fp8_matmul_tensorwise(input, self.weight, self.bias, self.sdnq_decompressor.scale) + + +def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: + if torch.numel(input) / input.shape[-1] < 32: + return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) + return int8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale, getattr(self.sdnq_decompressor, "compressed_weight_shape", None), self.sdnq_decompressor.weights_dtype) + + +def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTensor: + return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight), self.bias) + + +def get_conv_args(input_ndim: int, stride, padding, dilation): + if input_ndim == 3: + conv_type = 1 + elif input_ndim == 4: + conv_type = 2 + else: + conv_type = 3 + if isinstance(stride, int): + stride = (stride,) * conv_type + if isinstance(padding, int): + padding = (padding,) * conv_type + if isinstance(dilation, int): + dilation = (dilation,) * conv_type + if conv_type == 1: + stride = (1, stride[0]) + padding = (0, padding[0]) + dilation = (1, dilation[0]) + return conv_type, stride, padding, dilation + + +def quantized_conv_forward_fp8_matmul(self, input) -> torch.FloatTensor: + if torch.numel(input) / input.shape[2] < 32: + return self._conv_forward(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) + conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) + return conv_fp8_matmul( + input, self.weight, self.bias, + self.sdnq_decompressor.scale, + self.sdnq_decompressor.result_shape, + self._reversed_padding_repeated_twice, + self.padding_mode, conv_type, + self.groups, stride, padding, dilation, + ) + + +def quantized_conv_forward_fp8_matmul_tensorwise(self, input) -> torch.FloatTensor: + if torch.numel(input) / input.shape[2] < 32: + return self._conv_forward(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) + conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) + return conv_fp8_matmul_tensorwise( + input, self.weight, self.bias, + self.sdnq_decompressor.scale, + self.sdnq_decompressor.result_shape, + self._reversed_padding_repeated_twice, + self.padding_mode, conv_type, + self.groups, stride, padding, dilation, + ) + + +def quantized_conv_forward_int8_matmul(self, input) -> torch.FloatTensor: + if torch.numel(input) / input.shape[2] < 32: + return self._conv_forward(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) + conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) + return conv_int8_matmul( + input, self.weight, self.bias, + self.sdnq_decompressor.scale, + self.sdnq_decompressor.result_shape, + getattr(self.sdnq_decompressor, "compressed_weight_shape", None), + self.sdnq_decompressor.weights_dtype, + self._reversed_padding_repeated_twice, + self.padding_mode, conv_type, + self.groups, stride, padding, dilation, + ) + + +def quantized_conv_forward(self, input) -> torch.FloatTensor: + return self._conv_forward(input, self.sdnq_decompressor(self.weight), self.bias) + + +def quantized_conv_transpose_1d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: + output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 1, self.dilation) + return torch.nn.functional.conv_transpose1d(input, self.sdnq_decompressor(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + + +def quantized_conv_transpose_2d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: + output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 2, self.dilation) + return torch.nn.functional.conv_transpose2d(input, self.sdnq_decompressor(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + + +def quantized_conv_transpose_3d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: + output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 3, self.dilation) + return torch.nn.functional.conv_transpose3d(input, self.sdnq_decompressor(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + + +if shared.opts.sdnq_decompress_compile: + try: + torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) + int8_matmul = torch.compile(int8_matmul, fullgraph=True) + fp8_matmul = torch.compile(fp8_matmul, fullgraph=True) + fp8_matmul_tensorwise = torch.compile(fp8_matmul_tensorwise, fullgraph=True) + conv_int8_matmul = torch.compile(conv_int8_matmul, fullgraph=True) + conv_fp8_matmul = torch.compile(conv_fp8_matmul, fullgraph=True) + conv_fp8_matmul_tensorwise = torch.compile(conv_fp8_matmul_tensorwise, fullgraph=True) + except Exception as e: + shared.log.warning(f"Quantization: type=sdnq MatMul using torch.compile is not available: {e}") diff --git a/modules/sdnq/packed_int.py b/modules/sdnq/packed_int.py new file mode 100644 index 000000000..351dc7546 --- /dev/null +++ b/modules/sdnq/packed_int.py @@ -0,0 +1,212 @@ +# pylint: disable=redefined-builtin,no-member,protected-access + +from typing import Optional +import torch + +from .common import dtype_dict + + +def pack_int_symetric(tensor: torch.ByteTensor, weights_dtype: str) -> torch.ByteTensor: + return packed_int_function_dict[weights_dtype]["pack"](tensor.to(dtype=dtype_dict[weights_dtype]["torch_dtype"]).sub_(dtype_dict[weights_dtype]["min"]).to(dtype=dtype_dict[weights_dtype]["storage_dtype"])) + + +def unpack_int_symetric(packed_tensor: torch.CharTensor, shape: torch.Size, weights_dtype: str, dtype: Optional[torch.dtype] = None, transpose: Optional[bool] = False) -> torch.ByteTensor: + if dtype is None: + dtype = dtype_dict[weights_dtype]["torch_dtype"] + result = packed_int_function_dict[weights_dtype]["unpack"](packed_tensor, shape).to(dtype=dtype).add_(dtype_dict[weights_dtype]["min"]) + if transpose: + result = result.transpose(0,1) + return result + + +def pack_uint6(tensor: torch.Tensor) -> torch.Tensor: + if tensor.dtype != torch.uint8: + raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") + packed_tensor = tensor.contiguous().reshape(-1, 4) + packed_tensor = torch.stack( + ( + torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 3], 2), 192)), + torch.bitwise_or(packed_tensor[:, 1], torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 3], 4), 192)), + torch.bitwise_or(packed_tensor[:, 2], torch.bitwise_left_shift(packed_tensor[:, 3], 6)), + ), + dim=-1 + ) + return packed_tensor + + +def pack_uint5(tensor: torch.Tensor) -> torch.Tensor: + if tensor.dtype != torch.uint8: + raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") + packed_tensor = tensor.contiguous().reshape(-1, 8) + packed_tensor = torch.stack( + ( + torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 5], 5)), + torch.bitwise_or(packed_tensor[:, 1], torch.bitwise_left_shift(packed_tensor[:, 6], 5)), + torch.bitwise_or(packed_tensor[:, 2], torch.bitwise_left_shift(packed_tensor[:, 7], 5)), + torch.bitwise_or( + packed_tensor[:, 3], + torch.bitwise_or( + torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 5], 2), 96), + torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 3), 128), + ), + ), + torch.bitwise_or( + packed_tensor[:, 4], + torch.bitwise_or( + torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 6], 2), 96), + torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 4), 128), + ), + ), + ), + dim=-1 + ) + return packed_tensor + + +def pack_uint4(tensor: torch.Tensor) -> torch.Tensor: + if tensor.dtype != torch.uint8: + raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") + packed_tensor = tensor.contiguous().reshape(-1, 2) + packed_tensor = torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 1], 4)) + return packed_tensor + + +def pack_uint3(tensor: torch.Tensor) -> torch.Tensor: + if tensor.dtype != torch.uint8: + raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") + packed_tensor = tensor.contiguous().reshape(-1, 8) + packed_tensor = torch.stack( + ( + torch.bitwise_or( + torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 1], 3)), + torch.bitwise_left_shift(packed_tensor[:, 6], 6), + ), + torch.bitwise_or( + torch.bitwise_or(packed_tensor[:, 2], torch.bitwise_left_shift(packed_tensor[:, 3], 3)), + torch.bitwise_left_shift(packed_tensor[:, 7], 6), + ), + torch.bitwise_or( + torch.bitwise_or(packed_tensor[:, 4], torch.bitwise_left_shift(packed_tensor[:, 5], 3)), + torch.bitwise_or( + torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 6], 4), 64), + torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 5), 128), + ) + ), + ), + dim=-1 + ) + return packed_tensor + + +def pack_uint2(tensor: torch.Tensor) -> torch.Tensor: + if tensor.dtype != torch.uint8: + raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") + packed_tensor = tensor.contiguous().reshape(-1, 4) + packed_tensor = torch.bitwise_or( + torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_left_shift(packed_tensor[:, 1], 2)), + torch.bitwise_or(torch.bitwise_left_shift(packed_tensor[:, 2], 4), torch.bitwise_left_shift(packed_tensor[:, 3], 6)), + ) + return packed_tensor + + +def unpack_uint6(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: + result = torch.stack( + ( + torch.bitwise_and(packed_tensor[:, 0], 63), + torch.bitwise_and(packed_tensor[:, 1], 63), + torch.bitwise_and(packed_tensor[:, 2], 63), + torch.bitwise_or( + torch.bitwise_or( + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 0], 2), 48), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 1], 4), 12), + ), + torch.bitwise_right_shift(packed_tensor[:, 2], 6) + ) + ), + dim=-1 + ).reshape(shape) + return result + + +def unpack_uint5(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: + result = torch.stack( + ( + torch.bitwise_and(packed_tensor[:, 0], 31), + torch.bitwise_and(packed_tensor[:, 1], 31), + torch.bitwise_and(packed_tensor[:, 2], 31), + torch.bitwise_and(packed_tensor[:, 3], 31), + torch.bitwise_and(packed_tensor[:, 4], 31), + torch.bitwise_or( + torch.bitwise_right_shift(packed_tensor[:, 0], 5), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 3], 2), 24), + ), + torch.bitwise_or( + torch.bitwise_right_shift(packed_tensor[:, 1], 5), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 4], 2), 24), + ), + torch.bitwise_or( + torch.bitwise_right_shift(packed_tensor[:, 2], 5), + torch.bitwise_or( + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 3], 3), 16), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 4], 4), 8), + ), + ), + ), + dim=-1 + ).reshape(shape) + return result + + +def unpack_uint4(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: + result = torch.stack((torch.bitwise_and(packed_tensor, 15), torch.bitwise_right_shift(packed_tensor, 4)), dim=-1).reshape(shape) + return result + + +def unpack_uint3(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: + result = torch.stack( + ( + torch.bitwise_and(packed_tensor[:, 0], 7), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 0], 3), 7), + torch.bitwise_and(packed_tensor[:, 1], 7), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 1], 3), 7), + torch.bitwise_and(packed_tensor[:, 2], 7), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 2], 3), 7), + torch.bitwise_or( + torch.bitwise_right_shift(packed_tensor[:, 0], 6), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 2], 4), 4), + ), + torch.bitwise_or( + torch.bitwise_right_shift(packed_tensor[:, 1], 6), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 2], 5), 4), + ), + ), + dim=-1 + ).reshape(shape) + return result + + +def unpack_uint2(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: + result = torch.stack( + ( + torch.bitwise_and(packed_tensor, 3), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 2), 3), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor, 4), 3), + torch.bitwise_right_shift(packed_tensor, 6), + ), + dim=-1 + ).reshape(shape) + return result + + +packed_int_function_dict = { + "int6": {"pack": pack_uint6, "unpack": unpack_uint6}, + "int5": {"pack": pack_uint5, "unpack": unpack_uint5}, + "int4": {"pack": pack_uint4, "unpack": unpack_uint4}, + "int3": {"pack": pack_uint3, "unpack": unpack_uint3}, + "int2": {"pack": pack_uint2, "unpack": unpack_uint2}, + "uint6": {"pack": pack_uint6, "unpack": unpack_uint6}, + "uint5": {"pack": pack_uint5, "unpack": unpack_uint5}, + "uint4": {"pack": pack_uint4, "unpack": unpack_uint4}, + "uint3": {"pack": pack_uint3, "unpack": unpack_uint3}, + "uint2": {"pack": pack_uint2, "unpack": unpack_uint2}, +} From 5bd7a08877fe19944b28738f47b96aa684492cf7 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 10 Jun 2025 03:29:07 +0300 Subject: [PATCH 26/63] don't use inplace ops in quant layer --- modules/sdnq/__init__.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index fde0b2a26..345045da9 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -123,14 +123,10 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if dtype_dict[weights_dtype]["is_unsigned"]: scale, zero_point = get_scale_asymmetric(layer.weight, reduction_axes, weights_dtype) - layer.weight.data.sub_(zero_point).div_(scale) else: scale = get_scale_symmetric(layer.weight, reduction_axes, weights_dtype) - layer.weight.data.div_(scale) zero_point = None - if dtype_dict[weights_dtype]["is_integer"]: - layer.weight.data.round_() - layer.weight.data = layer.weight.data.clamp_(dtype_dict[weights_dtype]["min"], dtype_dict[weights_dtype]["max"]).to(dtype_dict[weights_dtype]["torch_dtype"]) + layer.weight.data = quantize_weight(layer.weight, scale, zero_point, weights_dtype) if not shared.opts.sdnq_decompress_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"] and not use_tensorwise_fp8_matmul): scale = scale.to(torch_dtype) @@ -214,6 +210,17 @@ def get_scale_symmetric(weight: torch.FloatTensor, reduction_axes: List[int], we return scale +def quantize_weight(weight: torch.FloatTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, weights_dtype: str) -> torch.Tensor: + if zero_point is not None: + compressed_weight = torch.sub(weight, zero_point).div_(scale) + else: + compressed_weight = torch.div(weight, scale) + if dtype_dict[weights_dtype]["is_integer"]: + compressed_weight.round_() + compressed_weight = compressed_weight.clamp_(dtype_dict[weights_dtype]["min"], dtype_dict[weights_dtype]["max"]).to(dtype_dict[weights_dtype]["torch_dtype"]) + return compressed_weight + + class QuantizationMethod(str, Enum): SDNQ = "sdnq" From 33fadf946b5188abe09e0ea543cee6bed6fa75e1 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 10 Jun 2025 11:33:06 +0300 Subject: [PATCH 27/63] SDNQ add 7 bit support --- CHANGELOG.md | 2 +- modules/sdnq/common.py | 10 +++---- modules/sdnq/decompressor.py | 2 ++ modules/sdnq/packed_int.py | 56 ++++++++++++++++++++++++++++++++++++ modules/shared.py | 2 +- wiki | 2 +- 6 files changed, 66 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63b86dbd5..53b02c6b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ - **SDNQ Quantization** - Add group size support for convolutional layers - Add quantized matmul support for for convolutional layers - - Add 5-bit and 3-bit quantization support + - Add 7-bit, 5-bit and 3-bit quantization support - Fix forced FP32 with tensorwise FP8 matmul - Fix PyTorch <= 2.4 compatibility with FP8 matmul - Fix VAE with conv quant diff --git a/modules/sdnq/common.py b/modules/sdnq/common.py index 70addbe7a..ccfdf1227 100644 --- a/modules/sdnq/common.py +++ b/modules/sdnq/common.py @@ -9,16 +9,16 @@ torch_version = float(torch.__version__[:3]) dtype_dict = { "int8": {"min": -128, "max": 127, "num_bits": 8, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.int8, "is_unsigned": False, "is_integer": True}, - "int7": {"min": -64, "max": 63, "num_bits": 6, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, + "int7": {"min": -64, "max": 63, "num_bits": 7, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "int6": {"min": -32, "max": 31, "num_bits": 6, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, - "int5": {"min": -16, "max": 15, "num_bits": 5, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, + "int5": {"min": -16, "max": 15, "num_bits": 5, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "int4": {"min": -8, "max": 7, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "int3": {"min": -4, "max": 3, "num_bits": 3, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "int2": {"min": -2, "max": 1, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.int8, "storage_dtype": torch.uint8, "is_unsigned": False, "is_integer": True}, "uint8": {"min": 0, "max": 255, "num_bits": 8, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, - "uint7": {"min": 0, "max": 127, "num_bits": 6, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint7": {"min": 0, "max": 127, "num_bits": 7, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "uint6": {"min": 0, "max": 63, "num_bits": 6, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, - "uint5": {"min": 0, "max": 31, "num_bits": 5, "target_dtype": torch.uint8, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, + "uint5": {"min": 0, "max": 31, "num_bits": 5, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "uint4": {"min": 0, "max": 15, "num_bits": 4, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "uint3": {"min": 0, "max": 7, "num_bits": 3, "target_dtype": CustomDtype.INT4, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, "uint2": {"min": 0, "max": 3, "num_bits": 2, "target_dtype": CustomDtype.INT2, "torch_dtype": torch.uint8, "storage_dtype": torch.uint8, "is_unsigned": True, "is_integer": True}, @@ -31,7 +31,7 @@ dtype_dict = { dtype_dict["bool"] = dtype_dict["uint1"] use_tensorwise_fp8_matmul = torch_version < 2.5 or devices.backend in {"cpu", "openvino"} or (devices.backend == "cuda" and sys.platform == "win32" and torch_version <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) -quantized_matmul_dtypes = ("int8", "int6", "int5", "int4", "int3", "int2", "float8_e4m3fn", "float8_e5m2") +quantized_matmul_dtypes = ("int8", "int7", "int6", "int5", "int4", "int3", "int2", "float8_e4m3fn", "float8_e5m2") if devices.backend in {"cpu", "openvino"}: quantized_matmul_dtypes += ("float8_e4m3fnuz", "float8_e5m2fnuz") diff --git a/modules/sdnq/decompressor.py b/modules/sdnq/decompressor.py index 2c37f26b2..c85ffc017 100644 --- a/modules/sdnq/decompressor.py +++ b/modules/sdnq/decompressor.py @@ -139,12 +139,14 @@ class PackedINTSymmetricWeightsDecompressor(torch.nn.Module): decompressor_dict = { "int8": SymmetricWeightsDecompressor, + "int7": PackedINTSymmetricWeightsDecompressor, "int6": PackedINTSymmetricWeightsDecompressor, "int5": PackedINTSymmetricWeightsDecompressor, "int4": PackedINTSymmetricWeightsDecompressor, "int3": PackedINTSymmetricWeightsDecompressor, "int2": PackedINTSymmetricWeightsDecompressor, "uint8": AsymmetricWeightsDecompressor, + "uint7": PackedINTAsymmetricWeightsDecompressor, "uint6": PackedINTAsymmetricWeightsDecompressor, "uint5": PackedINTAsymmetricWeightsDecompressor, "uint4": PackedINTAsymmetricWeightsDecompressor, diff --git a/modules/sdnq/packed_int.py b/modules/sdnq/packed_int.py index 351dc7546..e4717086e 100644 --- a/modules/sdnq/packed_int.py +++ b/modules/sdnq/packed_int.py @@ -19,6 +19,25 @@ def unpack_int_symetric(packed_tensor: torch.CharTensor, shape: torch.Size, weig return result +def pack_uint7(tensor: torch.Tensor) -> torch.Tensor: + if tensor.dtype != torch.uint8: + raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") + packed_tensor = tensor.contiguous().reshape(-1, 8) + packed_tensor = torch.stack( + ( + torch.bitwise_or(packed_tensor[:, 0], torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 1), 128)), + torch.bitwise_or(packed_tensor[:, 1], torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 2), 128)), + torch.bitwise_or(packed_tensor[:, 2], torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 3), 128)), + torch.bitwise_or(packed_tensor[:, 3], torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 4), 128)), + torch.bitwise_or(packed_tensor[:, 4], torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 5), 128)), + torch.bitwise_or(packed_tensor[:, 5], torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 6), 128)), + torch.bitwise_or(packed_tensor[:, 6], torch.bitwise_and(torch.bitwise_left_shift(packed_tensor[:, 7], 7), 128)), + ), + dim=-1 + ) + return packed_tensor + + def pack_uint6(tensor: torch.Tensor) -> torch.Tensor: if tensor.dtype != torch.uint8: raise RuntimeError(f"Invalid tensor dtype {tensor.type}. torch.uint8 type is supported.") @@ -109,6 +128,41 @@ def pack_uint2(tensor: torch.Tensor) -> torch.Tensor: return packed_tensor +def unpack_uint7(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: + result = torch.stack( + ( + torch.bitwise_and(packed_tensor[:, 0], 127), + torch.bitwise_and(packed_tensor[:, 1], 127), + torch.bitwise_and(packed_tensor[:, 2], 127), + torch.bitwise_and(packed_tensor[:, 3], 127), + torch.bitwise_and(packed_tensor[:, 4], 127), + torch.bitwise_and(packed_tensor[:, 5], 127), + torch.bitwise_and(packed_tensor[:, 6], 127), + torch.bitwise_or( + torch.bitwise_or( + torch.bitwise_or( + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 0], 1), 64), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 1], 2), 32), + ), + torch.bitwise_or( + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 2], 3), 16), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 3], 4), 8), + ), + ), + torch.bitwise_or( + torch.bitwise_or( + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 4], 5), 4), + torch.bitwise_and(torch.bitwise_right_shift(packed_tensor[:, 5], 6), 2), + ), + torch.bitwise_right_shift(packed_tensor[:, 6], 7), + ), + ) + ), + dim=-1 + ).reshape(shape) + return result + + def unpack_uint6(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor: result = torch.stack( ( @@ -199,11 +253,13 @@ def unpack_uint2(packed_tensor: torch.Tensor, shape: torch.Size) -> torch.Tensor packed_int_function_dict = { + "int7": {"pack": pack_uint7, "unpack": unpack_uint7}, "int6": {"pack": pack_uint6, "unpack": unpack_uint6}, "int5": {"pack": pack_uint5, "unpack": unpack_uint5}, "int4": {"pack": pack_uint4, "unpack": unpack_uint4}, "int3": {"pack": pack_uint3, "unpack": unpack_uint3}, "int2": {"pack": pack_uint2, "unpack": unpack_uint2}, + "uint7": {"pack": pack_uint7, "unpack": unpack_uint7}, "uint6": {"pack": pack_uint6, "unpack": unpack_uint6}, "uint5": {"pack": pack_uint5, "unpack": unpack_uint5}, "uint4": {"pack": pack_uint4, "unpack": unpack_uint4}, diff --git a/modules/shared.py b/modules/shared.py index c05039363..0e7099c38 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -518,7 +518,7 @@ options_templates.update(options_section(("quantization", "Quantization Settings "sdnq_quantize_sep": OptionInfo("

SDNQ: SD.Next Quantization

", "", gr.HTML), "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ["pre", "post"], "visible": native}), - "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "float8_e4m3fn", "int6", "int5", "uint4", "uint3", "uint2", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "uint8", "uint6", "uint5", "int4", "int3", "int2", "uint1"], "visible": native}), + "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "float8_e4m3fn", "int7", "int6", "int5", "uint4", "uint3", "uint2", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "uint8", "uint7", "uint6", "uint5", "int4", "int3", "int2", "uint1"], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_decompress_compile": OptionInfo(devices.has_triton(), "Decompress using torch.compile", gr.Checkbox, {"visible": native}), diff --git a/wiki b/wiki index f01d19390..88b282df8 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit f01d19390603551f4713015f7842cae21e70f6f0 +Subproject commit 88b282df8d42ff396d5e925b5d167ffd6d791d7e From f5b575db28b6cb23230510c88f784ba0a4aaad7b Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 10 Jun 2025 12:39:34 +0300 Subject: [PATCH 28/63] Update changelog and wiki --- CHANGELOG.md | 3 ++- wiki | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53b02c6b4..e4cd04bcf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2025-06-09 +## Update for 2025-06-10 - **Feature** - Support Python 3.13 @@ -27,6 +27,7 @@ - PixArt Sigma Small and Large loading - TAESD previews with PixArt - VAE Tiling with non-default tile sizes + - flash-atten repo with ROCm ## Update for 2025-06-02 diff --git a/wiki b/wiki index 88b282df8..7205474d6 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 88b282df8d42ff396d5e925b5d167ffd6d791d7e +Subproject commit 7205474d61969cdb7bf0a969b0b2d5318a2aee81 From d2ffee1b4e9b5c15da5bb0708a1a8e1cf55e7b4f Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 10 Jun 2025 13:21:37 +0300 Subject: [PATCH 29/63] ROCm don't override user set HSA_OVERRIDE_GFX_VERSION --- CHANGELOG.md | 1 + installer.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4cd04bcf..7ded29486 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Increase the medvram mode threshold from 8GB to 12GB - Set CPU backend to use FP32 by default - Relax Python version checks for Zluda + - don't override user set gfx version with ROCm - **Torch** - set default to `torch==2.7.1` diff --git a/installer.py b/installer.py index f92082dbe..5eb1a86f3 100644 --- a/installer.py +++ b/installer.py @@ -696,7 +696,7 @@ def install_rocm_zluda(): log.debug(f'ROCm hipBLASLt: arch={device.name} available={device.blaslt_supported}') rocm.set_blaslt_enabled(device.blaslt_supported) - if device is None: + if device is None or os.environ.get("HSA_OVERRIDE_GFX_VERSION", None) is not None: log.debug('ROCm: HSA_OVERRIDE_GFX_VERSION auto config skipped') else: gfx_ver = device.get_gfx_version() From a6b58efe4540fd3d4bbbbc495f1d1dfaddd873ab Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 10 Jun 2025 13:42:43 +0300 Subject: [PATCH 30/63] ROCm 6.4 support with --use-nightly --- CHANGELOG.md | 3 ++- installer.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ded29486..1593fbf57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,8 @@ - don't override user set gfx version with ROCm - **Torch** - - set default to `torch==2.7.1` + - Set default to `torch==2.7.1` + - Support ROCm 6.4 with `---use-nightly` - **SDNQ Quantization** - Add group size support for convolutional layers diff --git a/installer.py b/installer.py index 5eb1a86f3..cf45e3737 100644 --- a/installer.py +++ b/installer.py @@ -669,7 +669,9 @@ def install_rocm_zluda(): os.environ.setdefault('TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL', '1') if args.use_nightly: - if rocm.version is None or float(rocm.version) >= 6.3: # assume the latest if version check fails + if rocm.version is None or float(rocm.version) >= 6.4: # assume the latest if version check fails + torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm6.4') + elif rocm.version == "6.3": torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm6.3') else: # oldest rocm version on nightly is 6.2.4 torch_command = os.environ.get('TORCH_COMMAND', '--upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm6.2.4') From 7ccd94ed4fe12092e3c4ff659e3bea649f6f8a4c Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 10 Jun 2025 13:59:32 +0300 Subject: [PATCH 31/63] Force upgrade pip when installing Torch --- CHANGELOG.md | 1 + installer.py | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1593fbf57..d6b7f0cd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - **Torch** - Set default to `torch==2.7.1` - Support ROCm 6.4 with `---use-nightly` + - Force upgrade pip when installing Torch - **SDNQ Quantization** - Add group size support for convolutional layers diff --git a/installer.py b/installer.py index cf45e3737..0769808e9 100644 --- a/installer.py +++ b/installer.py @@ -869,6 +869,7 @@ def check_torch(): if 'torch' in torch_command and not args.version: if not installed('torch'): log.info(f'Torch: download and install in progress... cmd="{torch_command}"') + install('--upgrade pip', 'pip', reinstall=True) # pytorch rocm is too large for older pip install(torch_command, 'torch torchvision', quiet=True) else: try: From 4436a583aa69f936b85da8a4f694d82abe23a7fd Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 10 Jun 2025 14:12:49 +0300 Subject: [PATCH 32/63] Cleanup --- installer.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/installer.py b/installer.py index 0769808e9..a86f25f6c 100644 --- a/installer.py +++ b/installer.py @@ -711,7 +711,7 @@ def install_rocm_zluda(): return torch_command -def install_ipex(torch_command): +def install_ipex(): t_start = time.time() #check_python(supported_minors=[9, 10, 11, 12, 13], reason='IPEX backend requires a Python version between 3.9 and 3.12') args.use_ipex = True # pylint: disable=attribute-defined-outside-init @@ -744,7 +744,7 @@ def install_ipex(torch_command): return torch_command -def install_openvino(torch_command): +def install_openvino(): t_start = time.time() #check_python(supported_minors=[9, 10, 11, 12, 13], reason='OpenVINO backend requires a Python version between 3.9 and 3.12') log.info('OpenVINO: selected') @@ -842,15 +842,15 @@ def check_torch(): elif is_rocm_available and (args.use_rocm or args.use_zluda): # prioritize rocm torch_command = install_rocm_zluda() elif allow_ipex and args.use_ipex: # prioritize ipex - torch_command = install_ipex(torch_command) + torch_command = install_ipex() elif allow_openvino and args.use_openvino: # prioritize openvino - torch_command = install_openvino(torch_command) + torch_command = install_openvino() elif is_cuda_available: torch_command = install_cuda() elif is_rocm_available: torch_command = install_rocm_zluda() elif is_ipex_available: - torch_command = install_ipex(torch_command) + torch_command = install_ipex() else: machine = platform.machine() if sys.platform == 'darwin': @@ -1192,15 +1192,15 @@ def install_requirements(): if args.skip_requirements and not args.requirements: return if int(sys.version_info.minor) >= 13: - install("audioop-lts") + install('audioop-lts') # gcc 15 patch - backup_cmake_policy = os.environ.get("CMAKE_POLICY_VERSION_MINIMUM", None) - backup_cxxflags = os.environ.get("CXXFLAGS", None) - os.environ.setdefault("CMAKE_POLICY_VERSION_MINIMUM", "3.5") - os.environ.setdefault("CXXFLAGS", "-include cstdint") - install("git+https://github.com/google/sentencepiece#subdirectory=python", "sentencepiece") - os.environ.setdefault("CMAKE_POLICY_VERSION_MINIMUM", backup_cmake_policy) - os.environ.setdefault("CXXFLAGS", backup_cxxflags) + backup_cmake_policy = os.environ.get('CMAKE_POLICY_VERSION_MINIMUM', None) + backup_cxxflags = os.environ.get('CXXFLAGS', None) + os.environ.setdefault('CMAKE_POLICY_VERSION_MINIMUM', '3.5') + os.environ.setdefault('CXXFLAGS', '-include cstdint') + install('git+https://github.com/google/sentencepiece#subdirectory=python', 'sentencepiece') + os.environ.setdefault('CMAKE_POLICY_VERSION_MINIMUM', backup_cmake_policy) + os.environ.setdefault('CXXFLAGS', backup_cxxflags) if not installed('diffusers', quiet=True): # diffusers are not installed, so run initial installation global quick_allowed # pylint: disable=global-statement quick_allowed = False From 78f99abec80f10b4c5a510444782fcc7a35420be Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 10 Jun 2025 15:29:24 +0300 Subject: [PATCH 33/63] SDNQ use group_size / 2 for convs --- modules/sdnq/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index 345045da9..ef62c432f 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -64,6 +64,8 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz group_size = 2 ** (2 + dtype_dict[weights_dtype]["num_bits"]) else: group_size = 2 ** (1 + dtype_dict[weights_dtype]["num_bits"]) + elif group_size != -1 and not is_linear_type: + group_size = max(group_size // 2, 1) if not use_quantized_matmul and group_size > 0: if group_size >= channel_size: @@ -322,10 +324,11 @@ class SDNQQuantizer(DiffusersQuantizer): keep_in_fp32_modules: List[str] = [], **kwargs, # pylint: disable=unused-argument ): - model.config.quantization_config = self.quantization_config - self.modules_to_not_convert.extend(self.quantization_config.modules_to_not_convert) if keep_in_fp32_modules is not None: self.modules_to_not_convert.extend(keep_in_fp32_modules) + self.modules_to_not_convert.extend(self.quantization_config.modules_to_not_convert) + self.quantization_config.modules_to_not_convert = self.modules_to_not_convert + model.config.quantization_config = self.quantization_config def _process_model_after_weight_loading(self, model, **kwargs): # pylint: disable=unused-argument if shared.opts.diffusers_offload_mode != "none": @@ -366,7 +369,7 @@ class SDNQQuantizer(DiffusersQuantizer): @property def is_serializable(self): - return False + return True @dataclass From c81b712ddbb07a805522cef575e3b23c13ab2fbb Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 10 Jun 2025 15:56:19 +0300 Subject: [PATCH 34/63] Make VAE options not require model reload --- CHANGELOG.md | 3 ++- modules/processing_vae.py | 2 ++ modules/sd_models.py | 11 ++++++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6b7f0cd6..d8e022c8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ - Increase the medvram mode threshold from 8GB to 12GB - Set CPU backend to use FP32 by default - Relax Python version checks for Zluda - - don't override user set gfx version with ROCm + - Don't override user set gfx version with ROCm + - Make VAE options not require model reload - **Torch** - Set default to `torch==2.7.1` diff --git a/modules/processing_vae.py b/modules/processing_vae.py index b8e5dc4ae..290c0d489 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -117,6 +117,7 @@ def full_vae_decode(latents, model): elif shared.opts.diffusers_offload_mode != "sequential": sd_models.move_model(model.vae, devices.device) + sd_models.set_vae_options(model, vae=None, op='decode') upcast = (model.vae.dtype == torch.float16) and (getattr(model.vae.config, 'force_upcast', False) or shared.opts.no_half_vae) if upcast: if hasattr(model, 'upcast_vae'): # this is done by diffusers automatically if output_type != 'latent' @@ -193,6 +194,7 @@ def full_vae_encode(image, model): vae_name = sd_vae.loaded_vae_file if sd_vae.loaded_vae_file is not None else "default" log_debug(f'Encode vae="{vae_name}" dtype={model.vae.dtype} upcast={model.vae.config.get("force_upcast", None)}') + sd_models.set_vae_options(model, vae=None, op='encode') upcast = (model.vae.dtype == torch.float16) and (getattr(model.vae.config, 'force_upcast', False) or shared.opts.no_half_vae) if upcast: if hasattr(model, 'upcast_vae'): # this is done by diffusers automatically if output_type != 'latent' diff --git a/modules/sd_models.py b/modules/sd_models.py index ce35d0aba..c81b50366 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -92,7 +92,7 @@ def set_vae_options(sd_model, vae=None, op:str='model', quiet:bool=False): if shared.opts.diffusers_vae_upcast != 'default': sd_model.vae.config.force_upcast = True if shared.opts.diffusers_vae_upcast == 'true' else False shared.log.quiet(quiet, f'Setting {op}: component=VAE upcast={sd_model.vae.config.force_upcast}') - if shared.opts.no_half_vae: + if shared.opts.no_half_vae and op not in {'decode', 'encode'}: devices.dtype_vae = torch.float32 sd_model.vae.to(devices.dtype_vae) shared.log.quiet(quiet, f'Setting {op}: component=VAE no-half=True') @@ -105,11 +105,20 @@ def set_vae_options(sd_model, vae=None, op:str='model', quiet:bool=False): if hasattr(sd_model, "enable_vae_tiling"): if shared.opts.diffusers_vae_tiling: if hasattr(sd_model, 'vae') and hasattr(sd_model.vae, 'config') and hasattr(sd_model.vae.config, 'sample_size') and isinstance(sd_model.vae.config.sample_size, int): + if getattr(sd_model.vae, "tile_sample_min_size_backup", None) is None: + sd_model.vae.tile_sample_min_size_backup = sd_model.vae.tile_sample_min_size + sd_model.vae.tile_latent_min_size_backup = sd_model.vae.tile_latent_min_size + sd_model.vae.tile_overlap_factor_backup = sd_model.vae.tile_overlap_factor if shared.opts.diffusers_vae_tile_size > 0: sd_model.vae.tile_sample_min_size = int(shared.opts.diffusers_vae_tile_size) sd_model.vae.tile_latent_min_size = int(shared.opts.diffusers_vae_tile_size / (2 ** (len(sd_model.vae.config.block_out_channels) - 1))) + else: + sd_model.vae.tile_sample_min_size = getattr(sd_model.vae, "tile_sample_min_size_backup", sd_model.vae.tile_sample_min_size) + sd_model.vae.tile_latent_min_size = getattr(sd_model.vae, "tile_latent_min_size_backup", sd_model.vae.tile_latent_min_size) if shared.opts.diffusers_vae_tile_overlap != 0.25: sd_model.vae.tile_overlap_factor = float(shared.opts.diffusers_vae_tile_overlap) + else: + sd_model.vae.tile_overlap_factor = getattr(sd_model.vae, "tile_overlap_factor_backup", sd_model.vae.tile_overlap_factor) shared.log.quiet(quiet, f'Setting {op}: component=VAE tiling=True tile={sd_model.vae.tile_sample_min_size} overlap={sd_model.vae.tile_overlap_factor}') else: shared.log.quiet(quiet, f'Setting {op}: component=VAE tiling=True') From 64f49fb40fb3d20a1c09b9afa2b6541feb9bbafa Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 11 Jun 2025 14:58:12 +0300 Subject: [PATCH 35/63] ROCm log HSA_OVERRIDE_GFX_VERSION skip --- installer.py | 10 +++++----- wiki | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/installer.py b/installer.py index a86f25f6c..73f85f010 100644 --- a/installer.py +++ b/installer.py @@ -633,7 +633,7 @@ def install_rocm_zluda(): log.info(msg) if sys.platform == "win32": # TODO install: enable ROCm for windows when available - #check_python(supported_minors=[9, 10, 11, 12], reason='ZLUDA backend requires a Python version between 3.9 and 3.12') + #check_python(supported_minors=[9, 10, 11, 12, 13], reason='ZLUDA backend requires a Python version between 3.9 and 3.13') if args.device_id is not None: if os.environ.get('HIP_VISIBLE_DEVICES', None) is not None: @@ -663,7 +663,7 @@ def install_rocm_zluda(): log.info('Using CPU-only torch') torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') else: - #check_python(supported_minors=[9, 10, 11, 12, 13], reason='ROCm backend requires a Python version between 3.9 and 3.12') + #check_python(supported_minors=[9, 10, 11, 12, 13], reason='ROCm backend requires a Python version between 3.9 and 3.13') if os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", None) is None: os.environ.setdefault('TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL', '1') @@ -699,7 +699,7 @@ def install_rocm_zluda(): rocm.set_blaslt_enabled(device.blaslt_supported) if device is None or os.environ.get("HSA_OVERRIDE_GFX_VERSION", None) is not None: - log.debug('ROCm: HSA_OVERRIDE_GFX_VERSION auto config skipped') + log.info(f'ROCm: HSA_OVERRIDE_GFX_VERSION auto config skipped: version={os.environ.get("HSA_OVERRIDE_GFX_VERSION", None)}') else: gfx_ver = device.get_gfx_version() if gfx_ver is not None: @@ -713,7 +713,7 @@ def install_rocm_zluda(): def install_ipex(): t_start = time.time() - #check_python(supported_minors=[9, 10, 11, 12, 13], reason='IPEX backend requires a Python version between 3.9 and 3.12') + #check_python(supported_minors=[9, 10, 11, 12, 13], reason='IPEX backend requires a Python version between 3.9 and 3.13') args.use_ipex = True # pylint: disable=attribute-defined-outside-init log.info('IPEX: Intel OneAPI toolkit detected') @@ -746,7 +746,7 @@ def install_ipex(): def install_openvino(): t_start = time.time() - #check_python(supported_minors=[9, 10, 11, 12, 13], reason='OpenVINO backend requires a Python version between 3.9 and 3.12') + #check_python(supported_minors=[9, 10, 11, 12, 13], reason='OpenVINO backend requires a Python version between 3.9 and 3.13') log.info('OpenVINO: selected') if sys.platform == 'darwin': torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1 torchvision==0.22.1') diff --git a/wiki b/wiki index 7205474d6..70ea13a0c 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 7205474d61969cdb7bf0a969b0b2d5318a2aee81 +Subproject commit 70ea13a0c1af02184777d562e7b5d908077e639d From df6b13ea47c167a06a6305e498ca62191946da29 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 11 Jun 2025 15:09:03 +0300 Subject: [PATCH 36/63] Don't set gfx override with RX 9000 and above --- modules/rocm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/rocm.py b/modules/rocm.py index cc3268860..7553f3dfc 100644 --- a/modules/rocm.py +++ b/modules/rocm.py @@ -82,7 +82,7 @@ class Agent: def get_gfx_version(self) -> Union[str, None]: if self.gfx_version >= 0x1200: - return "12.0.0" + return None # 12.0.1 is RX 9070, 12.0.0 is RX 9060 elif self.gfx_version >= 0x1100: return "11.0.0" elif self.gfx_version >= 0x1000: From fd0c5b0e3eb71a74efcbe5fbd31ea77ffaae7d70 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 11 Jun 2025 15:12:22 +0300 Subject: [PATCH 37/63] Update changelog --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8e022c8e..7dcd87c17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,18 @@ - Increase the medvram mode threshold from 8GB to 12GB - Set CPU backend to use FP32 by default - Relax Python version checks for Zluda - - Don't override user set gfx version with ROCm - Make VAE options not require model reload - **Torch** - Set default to `torch==2.7.1` - - Support ROCm 6.4 with `---use-nightly` - Force upgrade pip when installing Torch +- **ROCm** + - Support ROCm 6.4 with `---use-nightly` + - Don't override user set gfx version + - Don't override gfx version with RX 9000 + - Fix flash-atten repo + - **SDNQ Quantization** - Add group size support for convolutional layers - Add quantized matmul support for for convolutional layers @@ -31,7 +35,6 @@ - PixArt Sigma Small and Large loading - TAESD previews with PixArt - VAE Tiling with non-default tile sizes - - flash-atten repo with ROCm ## Update for 2025-06-02 From 71be3c7d4547dd4e64e677ee8aea9819c9936a18 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 11 Jun 2025 15:47:25 +0300 Subject: [PATCH 38/63] ROCm don't override gfx with gfx1100 and gfx1101 + rocm 6.4 --- installer.py | 5 ++--- modules/rocm.py | 11 +++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/installer.py b/installer.py index 73f85f010..f1997ec2e 100644 --- a/installer.py +++ b/installer.py @@ -699,13 +699,12 @@ def install_rocm_zluda(): rocm.set_blaslt_enabled(device.blaslt_supported) if device is None or os.environ.get("HSA_OVERRIDE_GFX_VERSION", None) is not None: - log.info(f'ROCm: HSA_OVERRIDE_GFX_VERSION auto config skipped: version={os.environ.get("HSA_OVERRIDE_GFX_VERSION", None)}') + log.info(f'ROCm: HSA_OVERRIDE_GFX_VERSION auto config skipped: device={device.name if device is not None else None} version={os.environ.get("HSA_OVERRIDE_GFX_VERSION", None)}') else: gfx_ver = device.get_gfx_version() if gfx_ver is not None: os.environ.setdefault('HSA_OVERRIDE_GFX_VERSION', gfx_ver) - else: - log.warning(f'ROCm: device={device.name} could not auto-detect HSA version') + log.info(f'ROCm: HSA_OVERRIDE_GFX_VERSION config overridden: device={device.name} version={os.environ.get("HSA_OVERRIDE_GFX_VERSION", None)}') ts('amd', t_start) return torch_command diff --git a/modules/rocm.py b/modules/rocm.py index 7553f3dfc..262c15620 100644 --- a/modules/rocm.py +++ b/modules/rocm.py @@ -81,11 +81,14 @@ class Agent: self.blaslt_supported = os.path.exists(os.path.join(blaslt_tensile_libpath, f"Kernels.so-000-{name}.hsaco" if sys.platform == "win32" else f"extop_{name}.co")) def get_gfx_version(self) -> Union[str, None]: - if self.gfx_version >= 0x1200: - return None # 12.0.1 is RX 9070, 12.0.0 is RX 9060 - elif self.gfx_version >= 0x1100: + if self.gfx_version >= 0x1102 and self.gfx_version < 0x1200: return "11.0.0" - elif self.gfx_version >= 0x1000: + elif self.gfx_version == 0x1101: + if version is None or float(version) < 6.4: + return "11.0.0" # gfx1101 requires rocm 6.4.1 + else: + return None + elif self.gfx_version >= 0x1000 and self.gfx_version < 0x1100: # gfx1010 users had to override gfx version to 10.3.0 in Linux # it is unknown whether overriding is needed in ZLUDA return "10.3.0" From 6aa5c08fb0e4c4ff956f44594fae7094daebc621 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 11 Jun 2025 16:03:33 +0300 Subject: [PATCH 39/63] Cleanup and update changelog --- CHANGELOG.md | 2 +- modules/rocm.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dcd87c17..1bbde29b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ - **ROCm** - Support ROCm 6.4 with `---use-nightly` - Don't override user set gfx version - - Don't override gfx version with RX 9000 + - Don't override gfx version with RX 9000 and gfx1101 - Fix flash-atten repo - **SDNQ Quantization** diff --git a/modules/rocm.py b/modules/rocm.py index 262c15620..1a21b2f6e 100644 --- a/modules/rocm.py +++ b/modules/rocm.py @@ -81,13 +81,13 @@ class Agent: self.blaslt_supported = os.path.exists(os.path.join(blaslt_tensile_libpath, f"Kernels.so-000-{name}.hsaco" if sys.platform == "win32" else f"extop_{name}.co")) def get_gfx_version(self) -> Union[str, None]: - if self.gfx_version >= 0x1102 and self.gfx_version < 0x1200: - return "11.0.0" - elif self.gfx_version == 0x1101: + if self.gfx_version == 0x1101: if version is None or float(version) < 6.4: return "11.0.0" # gfx1101 requires rocm 6.4.1 else: return None + elif self.gfx_version >= 0x1102 and self.gfx_version < 0x1200: + return "11.0.0" elif self.gfx_version >= 0x1000 and self.gfx_version < 0x1100: # gfx1010 users had to override gfx version to 10.3.0 in Linux # it is unknown whether overriding is needed in ZLUDA From 74b6edf2dfb4237567d55517ba1cd740f441bd4a Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 11 Jun 2025 19:25:05 +0300 Subject: [PATCH 40/63] revert gfx1101 --- CHANGELOG.md | 2 +- modules/rocm.py | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bbde29b0..7dcd87c17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ - **ROCm** - Support ROCm 6.4 with `---use-nightly` - Don't override user set gfx version - - Don't override gfx version with RX 9000 and gfx1101 + - Don't override gfx version with RX 9000 - Fix flash-atten repo - **SDNQ Quantization** diff --git a/modules/rocm.py b/modules/rocm.py index 1a21b2f6e..816a3b10c 100644 --- a/modules/rocm.py +++ b/modules/rocm.py @@ -81,14 +81,9 @@ class Agent: self.blaslt_supported = os.path.exists(os.path.join(blaslt_tensile_libpath, f"Kernels.so-000-{name}.hsaco" if sys.platform == "win32" else f"extop_{name}.co")) def get_gfx_version(self) -> Union[str, None]: - if self.gfx_version == 0x1101: - if version is None or float(version) < 6.4: - return "11.0.0" # gfx1101 requires rocm 6.4.1 - else: - return None - elif self.gfx_version >= 0x1102 and self.gfx_version < 0x1200: + if self.gfx_version >= 0x1101 and self.gfx_version < 0x1200: return "11.0.0" - elif self.gfx_version >= 0x1000 and self.gfx_version < 0x1100: + elif self.gfx_version != 0x1030 and self.gfx_version >= 0x1000 and self.gfx_version < 0x1100: # gfx1010 users had to override gfx version to 10.3.0 in Linux # it is unknown whether overriding is needed in ZLUDA return "10.3.0" From 5cefa64a60d18f8e3b17c14ea68a869450f80639 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 11 Jun 2025 20:58:54 +0300 Subject: [PATCH 41/63] SDNQ update accepted dtypes --- modules/sdnq/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index ef62c432f..64134723a 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -381,7 +381,7 @@ class SDNQConfig(QuantizationConfigMixin): Args: weights_dtype (`str`, *optional*, defaults to `"int8"`): The target dtype for the weights after quantization. Supported values are: - ("int8", "int6", "int5", "int4", "int3", "int2", "uint8", "uint6", "uint5", "uint4", "uint3", "uint2", "uint1", "bool", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz") + ("int8", "int7", "int6", "int5", "int4", "int3", "int2", "uint8", "uint7", "uint6", "uint5", "uint4", "uint3", "uint2", "uint1", "bool", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz") modules_to_not_convert (`list`, *optional*, default to `None`): The list of modules to not quantize, useful for quantizing models that explicitly require to have some modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers). @@ -411,7 +411,7 @@ class SDNQConfig(QuantizationConfigMixin): r""" Safety checker that arguments are correct """ - accepted_weights = ["int8", "int6", "int5", "int4", "int3", "int2", "uint8", "uint6", "uint5", "uint4", "uint3", "uint2", "uint1", "bool", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz"] + accepted_weights = ["int8", "int7", "int6", "int5", "int4", "int3", "int2", "uint8", "uint7", "uint6", "uint5", "uint4", "uint3", "uint2", "uint1", "bool", "float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz"] if self.weights_dtype not in accepted_weights: raise ValueError(f"Only support weights in {accepted_weights} but found {self.weights_dtype}") if not isinstance(self.modules_to_not_convert, list): From dd84fb541f718b800c7a112db1897d96214666ed Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 11 Jun 2025 21:43:48 +0300 Subject: [PATCH 42/63] Always set sdpa params --- modules/devices.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/devices.py b/modules/devices.py index 2e8dc73ce..1c35f2683 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -426,8 +426,6 @@ def override_ipex_math(): def set_sdpa_params(): try: - if opts.cross_attention_optimization != "Scaled-Dot-Product": - return try: global sdpa_original # pylint: disable=global-statement if sdpa_original is not None: From 26545b6483a838100ab5d155b35217e65b381d11 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 11 Jun 2025 21:59:59 +0300 Subject: [PATCH 43/63] Add warning for incompatible attention processors --- modules/sd_models.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index c81b50366..7cf4f4993 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -908,12 +908,14 @@ def set_diffuser_pipe(pipe, new_pipe_type): def set_diffusers_attention(pipe, quiet:bool=False): import diffusers.models.attention_processor as p - def set_attn(pipe, attention): + def set_attn(pipe, attention, name:str=None, quiet:bool=False): if attention is None: return # other models uses their own attention processor if pipe.__class__.__name__.startswith("StableDiffusion") and hasattr(pipe, "unet"): pipe.unet.set_attn_processor(attention) + elif not quiet: + shared.log.warning(f"Attention: {name if name is not None else attention.__class__.__name__} is not compatible with {pipe.__class__.__name__}") # if hasattr(pipe, 'pipe'): # set_diffusers_attention(pipe.pipe) @@ -924,16 +926,22 @@ def set_diffusers_attention(pipe, quiet:bool=False): if shared.opts.cross_attention_optimization == "Disabled": pass # do nothing elif shared.opts.cross_attention_optimization == "Scaled-Dot-Product": # The default set by Diffusers - set_attn(pipe, p.AttnProcessor2_0()) - elif shared.opts.cross_attention_optimization == "xFormers" and hasattr(pipe, 'enable_xformers_memory_efficient_attention'): - pipe.enable_xformers_memory_efficient_attention() - elif shared.opts.cross_attention_optimization == "Split attention" and hasattr(pipe, "enable_attention_slicing"): - pipe.enable_attention_slicing() + set_attn(pipe, p.AttnProcessor2_0(), name="Scaled-Dot-Product", quiet=True) + elif shared.opts.cross_attention_optimization == "xFormers": + if hasattr(pipe, 'enable_xformers_memory_efficient_attention'): + pipe.enable_xformers_memory_efficient_attention() + else: + shared.log.warning(f"Attention: xFormers is not compatible with {pipe.__class__.__name__}") + elif shared.opts.cross_attention_optimization == "Split attention": + if hasattr(pipe, "enable_attention_slicing"): + pipe.enable_attention_slicing() + else: + shared.log.warning(f"Attention: Split attention is not compatible with {pipe.__class__.__name__}") elif shared.opts.cross_attention_optimization == "Batch matrix-matrix": - set_attn(pipe, p.AttnProcessor()) + set_attn(pipe, p.AttnProcessor(), name="Batch matrix-matrix") elif shared.opts.cross_attention_optimization == "Dynamic Attention BMM": from modules.sd_hijack_dynamic_atten import DynamicAttnProcessorBMM - set_attn(pipe, DynamicAttnProcessorBMM()) + set_attn(pipe, DynamicAttnProcessorBMM(), name="Dynamic Attention BMM") pipe.current_attn_name = shared.opts.cross_attention_optimization From 2d05396b4ee9d122b08b27984d246d1898ab5ef4 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 12 Jun 2025 02:26:04 +0300 Subject: [PATCH 44/63] SDNQ simplify sym scale formula --- modules/sdnq/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index 64134723a..717e130da 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -204,9 +204,7 @@ def get_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: List[int], w def get_scale_symmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> torch.FloatTensor: - abs_min_values = torch.amin(weight, dim=reduction_axes, keepdims=True).abs_() - max_values = torch.amax(weight, dim=reduction_axes, keepdims=True) - scale = torch.where(abs_min_values >= max_values, abs_min_values, -max_values).div_(dtype_dict[weights_dtype]["max"]) + scale = torch.amax(weight.abs(), dim=reduction_axes, keepdims=True).div_(dtype_dict[weights_dtype]["max"]) eps = torch.finfo(scale.dtype).eps # prevent divison by 0 scale = torch.where(torch.abs(scale) < eps, eps, scale) return scale From 5e013fb1543acaa1ab6f1372cd08a9c0c11cb719 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 12 Jun 2025 12:06:57 +0300 Subject: [PATCH 45/63] SDNQ optimize input quantization and use the word quantize instead of compress --- modules/lora/lora_apply.py | 26 +++--- modules/sdnq/__init__.py | 41 ++++----- .../sdnq/{decompressor.py => dequantizer.py} | 92 +++++++++---------- modules/sdnq/forward.py | 78 ++++++++-------- modules/shared.py | 4 +- wiki | 2 +- 6 files changed, 120 insertions(+), 123 deletions(-) rename modules/sdnq/{decompressor.py => dequantizer.py} (63%) diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index b265fcce2..62e9cbdfa 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -45,8 +45,8 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n self.network_weights_backup = True else: self.network_weights_backup = weight.clone().to(devices.cpu) - if hasattr(self, "sdnq_decompressor"): - self.sdnq_decompressor_backup = self.sdnq_decompressor.to(devices.cpu) + if hasattr(self, "sdnq_dequantizer"): + self.sdnq_dequantizer_backup = self.sdnq_dequantizer.to(devices.cpu) if bias_backup is None: if getattr(self, 'bias', None) is not None: @@ -79,8 +79,8 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. continue try: t0 = time.time() - if hasattr(self, "sdnq_decompressor"): - weight = self.sdnq_decompressor.to(devices.device)(self.weight.to(devices.device), skip_quantized_matmul=self.sdnq_decompressor.use_quantized_matmul) + if hasattr(self, "sdnq_dequantizer"): + weight = self.sdnq_dequantizer.to(devices.device)(self.weight.to(devices.device), skip_quantized_matmul=self.sdnq_dequantizer.use_quantized_matmul) else: weight = self.weight.to(devices.device) # must perform calc on gpu due to performance updown, ex_bias = module.calc_updown(weight) @@ -136,20 +136,20 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G # weight._quantize(devices.device) / weight.to(device=device) except Exception as e: shared.log.error(f'Network load: type=LoRA quant=bnb cls={self.__class__.__name__} type={self.quant_type} blocksize={self.blocksize} state={vars(self.quant_state)} weight={self.weight} bias={lora_weights} {e}') - elif not bias and hasattr(self, "sdnq_decompressor"): + elif not bias and hasattr(self, "sdnq_dequantizer"): try: from modules.sdnq import sdnq_quantize_layer - if hasattr(self, "sdnq_decompressor_backup"): - sdnq_decompressor = self.sdnq_decompressor_backup.to(devices.device) + if hasattr(self, "sdnq_dequantizer_backup"): + sdnq_dequantizer = self.sdnq_dequantizer_backup.to(devices.device) else: - sdnq_decompressor = self.sdnq_decompressor.to(devices.device) - dequant_weight = sdnq_decompressor(model_weights.to(devices.device), skip_quantized_matmul=sdnq_decompressor.use_quantized_matmul) + sdnq_dequantizer = self.sdnq_dequantizer.to(devices.device) + dequant_weight = sdnq_dequantizer(model_weights.to(devices.device), skip_quantized_matmul=sdnq_dequantizer.use_quantized_matmul) new_weight = dequant_weight.to(devices.device, dtype=torch.float32) + lora_weights.to(devices.device, dtype=torch.float32) self.weight = torch.nn.Parameter(new_weight, requires_grad=False) - self.sdnq_decompressor = None + self.sdnq_dequantizer = None self = sdnq_quantize_layer( self, - sdnq_decompressor.weights_dtype, + sdnq_dequantizer.weights_dtype, torch_dtype=devices.dtype, group_size=shared.opts.sdnq_quantize_weights_group_size, quant_conv=shared.opts.sdnq_quantize_conv_layers, @@ -223,8 +223,8 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn network_add_weights(self, model_weights=weights_backup, lora_weights=updown, deactivate=deactivate, device=device, bias=False) else: self.weight = torch.nn.Parameter(weights_backup.to(device), requires_grad=False) - if hasattr(self, "sdnq_decompressor_backup"): - self.sdnq_decompressor = self.sdnq_decompressor_backup.to(device) + if hasattr(self, "sdnq_dequantizer_backup"): + self.sdnq_dequantizer = self.sdnq_dequantizer_backup.to(device) if bias_backup is not None: self.bias = None diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index 717e130da..f7607fcd1 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -10,7 +10,7 @@ from diffusers.utils import get_module_from_name from modules import devices, shared from .common import dtype_dict, use_tensorwise_fp8_matmul, quantized_matmul_dtypes, allowed_types, conv_types, conv_transpose_types -from .decompressor import decompressor_dict +from .dequantizer import dequantizer_dict from .forward import get_forward_func @@ -123,14 +123,8 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz else: layer.weight.data = layer.weight.to(dtype=torch.float32) - if dtype_dict[weights_dtype]["is_unsigned"]: - scale, zero_point = get_scale_asymmetric(layer.weight, reduction_axes, weights_dtype) - else: - scale = get_scale_symmetric(layer.weight, reduction_axes, weights_dtype) - zero_point = None - layer.weight.data = quantize_weight(layer.weight, scale, zero_point, weights_dtype) - - if not shared.opts.sdnq_decompress_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"] and not use_tensorwise_fp8_matmul): + layer.weight.data, scale, zero_point = quantize_weight(layer.weight, reduction_axes, weights_dtype) + if not shared.opts.sdnq_dequantize_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"] and not use_tensorwise_fp8_matmul): scale = scale.to(torch_dtype) if zero_point is not None: zero_point = zero_point.to(torch_dtype) @@ -146,17 +140,17 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if not use_tensorwise_fp8_matmul: scale = scale.to(torch.float32) - layer.sdnq_decompressor = decompressor_dict[weights_dtype]( + layer.sdnq_dequantizer = dequantizer_dict[weights_dtype]( scale=scale, zero_point=zero_point, - compressed_weight_shape=layer.weight.shape, + quantized_weight_shape=layer.weight.shape, result_dtype=torch_dtype, result_shape=result_shape, weights_dtype=weights_dtype, use_quantized_matmul=use_quantized_matmul, ) - layer.weight.data = layer.sdnq_decompressor.pack_weight(layer.weight).to(return_device) - layer.sdnq_decompressor = layer.sdnq_decompressor.to(return_device) + layer.weight.data = layer.sdnq_dequantizer.pack_weight(layer.weight).to(return_device) + layer.sdnq_dequantizer = layer.sdnq_dequantizer.to(return_device) layer.forward = get_forward_func(layer_class_name, use_quantized_matmul, dtype_dict[weights_dtype]["is_integer"], use_tensorwise_fp8_matmul) layer.forward = layer.forward.__get__(layer, layer.__class__) @@ -193,7 +187,7 @@ def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_si return model -def get_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> Tuple[torch.FloatTensor, torch.FloatTensor]: +def get_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: Union[int, List[int]], weights_dtype: str) -> Tuple[torch.FloatTensor, torch.FloatTensor]: zero_point = torch.amin(weight, dim=reduction_axes, keepdims=True) scale = torch.amax(weight, dim=reduction_axes, keepdims=True).sub_(zero_point).div_(dtype_dict[weights_dtype]["max"] - dtype_dict[weights_dtype]["min"]) eps = torch.finfo(scale.dtype).eps # prevent divison by 0 @@ -203,22 +197,25 @@ def get_scale_asymmetric(weight: torch.FloatTensor, reduction_axes: List[int], w return scale, zero_point -def get_scale_symmetric(weight: torch.FloatTensor, reduction_axes: List[int], weights_dtype: str) -> torch.FloatTensor: +def get_scale_symmetric(weight: torch.FloatTensor, reduction_axes: Union[int, List[int]], weights_dtype: str) -> torch.FloatTensor: scale = torch.amax(weight.abs(), dim=reduction_axes, keepdims=True).div_(dtype_dict[weights_dtype]["max"]) eps = torch.finfo(scale.dtype).eps # prevent divison by 0 scale = torch.where(torch.abs(scale) < eps, eps, scale) return scale -def quantize_weight(weight: torch.FloatTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, weights_dtype: str) -> torch.Tensor: - if zero_point is not None: - compressed_weight = torch.sub(weight, zero_point).div_(scale) +def quantize_weight(weight: torch.FloatTensor, reduction_axes: Union[int, List[int]], weights_dtype: str): + if dtype_dict[weights_dtype]["is_unsigned"]: + scale, zero_point = get_scale_asymmetric(weight, reduction_axes, weights_dtype) + quantized_weight = torch.sub(weight, zero_point).div_(scale) else: - compressed_weight = torch.div(weight, scale) + scale = get_scale_symmetric(weight, reduction_axes, weights_dtype) + quantized_weight = torch.div(weight, scale) + zero_point = None if dtype_dict[weights_dtype]["is_integer"]: - compressed_weight.round_() - compressed_weight = compressed_weight.clamp_(dtype_dict[weights_dtype]["min"], dtype_dict[weights_dtype]["max"]).to(dtype_dict[weights_dtype]["torch_dtype"]) - return compressed_weight + quantized_weight.round_() + quantized_weight = quantized_weight.clamp_(dtype_dict[weights_dtype]["min"], dtype_dict[weights_dtype]["max"]).to(dtype_dict[weights_dtype]["torch_dtype"]) + return quantized_weight, scale, zero_point class QuantizationMethod(str, Enum): diff --git a/modules/sdnq/decompressor.py b/modules/sdnq/dequantizer.py similarity index 63% rename from modules/sdnq/decompressor.py rename to modules/sdnq/dequantizer.py index c85ffc017..1f2b36df1 100644 --- a/modules/sdnq/decompressor.py +++ b/modules/sdnq/dequantizer.py @@ -7,14 +7,14 @@ from .common import dtype_dict from .packed_int import pack_int_symetric, unpack_int_symetric, packed_int_function_dict -def decompress_asymmetric(input: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, dtype: torch.dtype, result_shape: torch.Size) -> torch.FloatTensor: +def dequantize_asymmetric(input: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, dtype: torch.dtype, result_shape: torch.Size) -> torch.FloatTensor: result = torch.addcmul(zero_point, input.to(dtype=scale.dtype), scale).to(dtype=dtype) if result_shape is not None: result = result.reshape(result_shape) return result -def decompress_symmetric(input: torch.CharTensor, scale: torch.FloatTensor, dtype: torch.dtype, result_shape: torch.Size, skip_quantized_matmul: bool = False) -> torch.FloatTensor: +def dequantize_symmetric(input: torch.CharTensor, scale: torch.FloatTensor, dtype: torch.dtype, result_shape: torch.Size, skip_quantized_matmul: bool = False) -> torch.FloatTensor: if skip_quantized_matmul: result = input.transpose(0,1).to(dtype=scale.dtype).mul_(scale.transpose(0,1)).to(dtype=dtype) else: @@ -24,18 +24,18 @@ def decompress_symmetric(input: torch.CharTensor, scale: torch.FloatTensor, dtyp return result -def decompress_packed_int_asymmetric(input: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str) -> torch.FloatTensor: - return decompress_asymmetric(packed_int_function_dict[weights_dtype]["unpack"](input, shape), scale, zero_point, dtype, result_shape) +def dequantize_packed_int_asymmetric(input: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str) -> torch.FloatTensor: + return dequantize_asymmetric(packed_int_function_dict[weights_dtype]["unpack"](input, shape), scale, zero_point, dtype, result_shape) -def decompress_packed_int_symmetric(input: torch.ByteTensor, scale: torch.FloatTensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str, skip_quantized_matmul: bool = False) -> torch.FloatTensor: +def dequantize_packed_int_symmetric(input: torch.ByteTensor, scale: torch.FloatTensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str, skip_quantized_matmul: bool = False) -> torch.FloatTensor: if skip_quantized_matmul: - return decompress_symmetric(unpack_int_symetric(input, shape, weights_dtype, dtype=scale.dtype), scale.transpose(0,1), dtype, result_shape) + return dequantize_symmetric(unpack_int_symetric(input, shape, weights_dtype, dtype=scale.dtype), scale.transpose(0,1), dtype, result_shape) else: - return decompress_symmetric(unpack_int_symetric(input, shape, weights_dtype, dtype=scale.dtype), scale, dtype, result_shape) + return dequantize_symmetric(unpack_int_symetric(input, shape, weights_dtype, dtype=scale.dtype), scale, dtype, result_shape) -class AsymmetricWeightsDecompressor(torch.nn.Module): +class AsymmetricWeightsDequantizer(torch.nn.Module): def __init__( self, scale: torch.Tensor, @@ -57,10 +57,10 @@ class AsymmetricWeightsDecompressor(torch.nn.Module): return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) def forward(self, weight, **kwargs): # pylint: disable=unused-argument - return decompress_asymmetric(weight, self.scale, self.zero_point, self.result_dtype, self.result_shape) + return dequantize_asymmetric(weight, self.scale, self.zero_point, self.result_dtype, self.result_shape) -class SymmetricWeightsDecompressor(torch.nn.Module): +class SymmetricWeightsDequantizer(torch.nn.Module): def __init__( self, scale: torch.Tensor, @@ -81,15 +81,15 @@ class SymmetricWeightsDecompressor(torch.nn.Module): return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) def forward(self, weight, skip_quantized_matmul=False, **kwargs): # pylint: disable=unused-argument - return decompress_symmetric(weight, self.scale, self.result_dtype, self.result_shape, skip_quantized_matmul=skip_quantized_matmul) + return dequantize_symmetric(weight, self.scale, self.result_dtype, self.result_shape, skip_quantized_matmul=skip_quantized_matmul) -class PackedINTAsymmetricWeightsDecompressor(torch.nn.Module): +class PackedINTAsymmetricWeightsDequantizer(torch.nn.Module): def __init__( self, scale: torch.Tensor, zero_point: torch.Tensor, - compressed_weight_shape: torch.Size, + quantized_weight_shape: torch.Size, result_dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str, @@ -98,7 +98,7 @@ class PackedINTAsymmetricWeightsDecompressor(torch.nn.Module): super().__init__() self.weights_dtype = weights_dtype self.use_quantized_matmul = False - self.compressed_weight_shape = compressed_weight_shape + self.quantized_weight_shape = quantized_weight_shape self.result_dtype = result_dtype self.result_shape = result_shape self.register_buffer("scale", scale) @@ -108,14 +108,14 @@ class PackedINTAsymmetricWeightsDecompressor(torch.nn.Module): return packed_int_function_dict[self.weights_dtype]["pack"](weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"])) def forward(self, weight, **kwargs): # pylint: disable=unused-argument - return decompress_packed_int_asymmetric(weight, self.scale, self.zero_point, self.compressed_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype) + return dequantize_packed_int_asymmetric(weight, self.scale, self.zero_point, self.quantized_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype) -class PackedINTSymmetricWeightsDecompressor(torch.nn.Module): +class PackedINTSymmetricWeightsDequantizer(torch.nn.Module): def __init__( self, scale: torch.Tensor, - compressed_weight_shape: torch.Size, + quantized_weight_shape: torch.Size, result_dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str, @@ -125,7 +125,7 @@ class PackedINTSymmetricWeightsDecompressor(torch.nn.Module): super().__init__() self.weights_dtype = weights_dtype self.use_quantized_matmul = use_quantized_matmul - self.compressed_weight_shape = compressed_weight_shape + self.quantized_weight_shape = quantized_weight_shape self.result_dtype = result_dtype self.result_shape = result_shape self.register_buffer("scale", scale) @@ -134,39 +134,39 @@ class PackedINTSymmetricWeightsDecompressor(torch.nn.Module): return pack_int_symetric(weight, self.weights_dtype) def forward(self, weight, skip_quantized_matmul=False, **kwargs): # pylint: disable=unused-argument - return decompress_packed_int_symmetric(weight, self.scale, self.compressed_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype, skip_quantized_matmul=skip_quantized_matmul) + return dequantize_packed_int_symmetric(weight, self.scale, self.quantized_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype, skip_quantized_matmul=skip_quantized_matmul) -decompressor_dict = { - "int8": SymmetricWeightsDecompressor, - "int7": PackedINTSymmetricWeightsDecompressor, - "int6": PackedINTSymmetricWeightsDecompressor, - "int5": PackedINTSymmetricWeightsDecompressor, - "int4": PackedINTSymmetricWeightsDecompressor, - "int3": PackedINTSymmetricWeightsDecompressor, - "int2": PackedINTSymmetricWeightsDecompressor, - "uint8": AsymmetricWeightsDecompressor, - "uint7": PackedINTAsymmetricWeightsDecompressor, - "uint6": PackedINTAsymmetricWeightsDecompressor, - "uint5": PackedINTAsymmetricWeightsDecompressor, - "uint4": PackedINTAsymmetricWeightsDecompressor, - "uint3": PackedINTAsymmetricWeightsDecompressor, - "uint2": PackedINTAsymmetricWeightsDecompressor, - "uint1": AsymmetricWeightsDecompressor, - "bool": AsymmetricWeightsDecompressor, - "float8_e4m3fn": SymmetricWeightsDecompressor, - "float8_e4m3fnuz": SymmetricWeightsDecompressor, - "float8_e5m2": SymmetricWeightsDecompressor, - "float8_e5m2fnuz": SymmetricWeightsDecompressor, +dequantizer_dict = { + "int8": SymmetricWeightsDequantizer, + "int7": PackedINTSymmetricWeightsDequantizer, + "int6": PackedINTSymmetricWeightsDequantizer, + "int5": PackedINTSymmetricWeightsDequantizer, + "int4": PackedINTSymmetricWeightsDequantizer, + "int3": PackedINTSymmetricWeightsDequantizer, + "int2": PackedINTSymmetricWeightsDequantizer, + "uint8": AsymmetricWeightsDequantizer, + "uint7": PackedINTAsymmetricWeightsDequantizer, + "uint6": PackedINTAsymmetricWeightsDequantizer, + "uint5": PackedINTAsymmetricWeightsDequantizer, + "uint4": PackedINTAsymmetricWeightsDequantizer, + "uint3": PackedINTAsymmetricWeightsDequantizer, + "uint2": PackedINTAsymmetricWeightsDequantizer, + "uint1": AsymmetricWeightsDequantizer, + "bool": AsymmetricWeightsDequantizer, + "float8_e4m3fn": SymmetricWeightsDequantizer, + "float8_e4m3fnuz": SymmetricWeightsDequantizer, + "float8_e5m2": SymmetricWeightsDequantizer, + "float8_e5m2fnuz": SymmetricWeightsDequantizer, } -if shared.opts.sdnq_decompress_compile: +if shared.opts.sdnq_dequantize_compile: try: torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) - decompress_asymmetric = torch.compile(decompress_asymmetric, fullgraph=True) - decompress_symmetric = torch.compile(decompress_symmetric, fullgraph=True) - decompress_packed_int_asymmetric = torch.compile(decompress_packed_int_asymmetric, fullgraph=True) - decompress_packed_int_symmetric = torch.compile(decompress_packed_int_symmetric, fullgraph=True) + dequantize_asymmetric = torch.compile(dequantize_asymmetric, fullgraph=True) + dequantize_symmetric = torch.compile(dequantize_symmetric, fullgraph=True) + dequantize_packed_int_asymmetric = torch.compile(dequantize_packed_int_asymmetric, fullgraph=True) + dequantize_packed_int_symmetric = torch.compile(dequantize_packed_int_symmetric, fullgraph=True) except Exception as e: - shared.log.warning(f"Quantization: type=sdnq Decompress using torch.compile is not available: {e}") + shared.log.warning(f"Quantization: type=sdnq Dequantize using torch.compile is not available: {e}") diff --git a/modules/sdnq/forward.py b/modules/sdnq/forward.py index e5f345682..be4a957ea 100644 --- a/modules/sdnq/forward.py +++ b/modules/sdnq/forward.py @@ -5,7 +5,7 @@ import torch from modules import shared from .common import conv_types, conv_transpose_types -from .decompressor import decompress_symmetric +from .dequantizer import dequantize_symmetric from .packed_int import unpack_int_symetric @@ -43,7 +43,7 @@ def get_forward_func(layer_class_name: str, use_quantized_matmul: bool, is_integ def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.Tensor, torch.FloatTensor]: input = input.flatten(0,-2).contiguous() - input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 448) + input_scale = torch.amax(input.abs(), dim=-1, keepdims=True).div_(448) input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn) input_scale = input_scale.to(torch.float32) return input, input_scale @@ -51,7 +51,7 @@ def quantize_fp8_matmul_input(input: torch.FloatTensor) -> Tuple[torch.Tensor, t def quantize_fp8_matmul_input_tensorwise(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.Tensor, torch.FloatTensor]: input = input.flatten(0,-2).contiguous() - input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 448) + input_scale = torch.amax(input.abs(), dim=-1, keepdims=True).div_(448) input = torch.div(input, input_scale).clamp_(-448, 448).to(torch.float8_e4m3fn) scale = torch.mul(input_scale, scale) if scale.dtype == torch.float16: # fp16 will overflow @@ -61,7 +61,7 @@ def quantize_fp8_matmul_input_tensorwise(input: torch.FloatTensor, scale: torch. def quantize_int8_matmul_input(input: torch.FloatTensor, scale: torch.FloatTensor) -> Tuple[torch.CharTensor, torch.FloatTensor]: input = input.flatten(0,-2).contiguous() - input_scale = torch.div(input.abs().amax(dim=-1, keepdims=True), 127) + input_scale = torch.amax(input.abs(), dim=-1, keepdims=True).div_(127) input = torch.div(input, input_scale).round_().clamp_(-128, 127).to(torch.int8) scale = torch.mul(input_scale, scale) if scale.dtype == torch.float16: # fp16 will overflow @@ -94,7 +94,7 @@ def fp8_matmul_tensorwise( output_shape[-1] = weight.shape[-1] dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) input, scale = quantize_fp8_matmul_input_tensorwise(input, scale) - result = decompress_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, output_shape) + result = dequantize_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, output_shape) if bias is not None: result.add_(bias) return result @@ -105,16 +105,16 @@ def int8_matmul( weight: torch.Tensor, bias: torch.FloatTensor, scale: torch.FloatTensor, - compressed_weight_shape: torch.Size, + quantized_weight_shape: torch.Size, weights_dtype: str, ) -> torch.FloatTensor: - if compressed_weight_shape is not None: - weight = unpack_int_symetric(weight, compressed_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) + if quantized_weight_shape is not None: + weight = unpack_int_symetric(weight, quantized_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) return_dtype = input.dtype output_shape = list(input.shape) output_shape[-1] = weight.shape[-1] input, scale = quantize_int8_matmul_input(input, scale) - result = decompress_symmetric(torch._int_mm(input, weight), scale, return_dtype, output_shape) + result = dequantize_symmetric(torch._int_mm(input, weight), scale, return_dtype, output_shape) if bias is not None: result.add_(bias) return result @@ -224,14 +224,14 @@ def conv_fp8_matmul_tensorwise( dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) if groups == 1: - result = decompress_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, mm_output_shape) + result = dequantize_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, mm_output_shape) else: weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) result = [] for i in range(groups): result.append(torch._scaled_mm(input[i], weight[i], scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype)) - result = decompress_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) + result = dequantize_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) if bias is not None: result.add_(bias) @@ -250,7 +250,7 @@ def conv_int8_matmul( bias: torch.FloatTensor, scale: torch.FloatTensor, result_shape: torch.Size, - compressed_weight_shape: torch.Size, + quantized_weight_shape: torch.Size, weights_dtype: str, reversed_padding_repeated_twice: List[int], padding_mode: str, conv_type: int, @@ -260,18 +260,18 @@ def conv_int8_matmul( return_dtype = input.dtype input, mm_output_shape = process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation) input, scale = quantize_int8_matmul_input(input, scale) - if compressed_weight_shape is not None: - weight = unpack_int_symetric(weight, compressed_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) + if quantized_weight_shape is not None: + weight = unpack_int_symetric(weight, quantized_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) if groups == 1: - result = decompress_symmetric(torch._int_mm(input, weight), scale, return_dtype, mm_output_shape) + result = dequantize_symmetric(torch._int_mm(input, weight), scale, return_dtype, mm_output_shape) else: weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) result = [] for i in range(groups): result.append(torch._int_mm(input[i], weight[i])) - result = decompress_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) + result = dequantize_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) if bias is not None: result.add_(bias) @@ -286,24 +286,24 @@ def conv_int8_matmul( def quantized_linear_forward_fp8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: if torch.numel(input) / input.shape[-1] < 32: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) - return fp8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale) + return torch.nn.functional.linear(input, self.sdnq_dequantizer(self.weight, skip_quantized_matmul=True), self.bias) + return fp8_matmul(input, self.weight, self.bias, self.sdnq_dequantizer.scale) def quantized_linear_forward_fp8_matmul_tensorwise(self, input: torch.FloatTensor) -> torch.FloatTensor: if torch.numel(input) / input.shape[-1] < 32: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) - return fp8_matmul_tensorwise(input, self.weight, self.bias, self.sdnq_decompressor.scale) + return torch.nn.functional.linear(input, self.sdnq_dequantizer(self.weight, skip_quantized_matmul=True), self.bias) + return fp8_matmul_tensorwise(input, self.weight, self.bias, self.sdnq_dequantizer.scale) def quantized_linear_forward_int8_matmul(self, input: torch.FloatTensor) -> torch.FloatTensor: if torch.numel(input) / input.shape[-1] < 32: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) - return int8_matmul(input, self.weight, self.bias, self.sdnq_decompressor.scale, getattr(self.sdnq_decompressor, "compressed_weight_shape", None), self.sdnq_decompressor.weights_dtype) + return torch.nn.functional.linear(input, self.sdnq_dequantizer(self.weight, skip_quantized_matmul=True), self.bias) + return int8_matmul(input, self.weight, self.bias, self.sdnq_dequantizer.scale, getattr(self.sdnq_dequantizer, "quantized_weight_shape", None), self.sdnq_dequantizer.weights_dtype) def quantized_linear_forward(self, input: torch.FloatTensor) -> torch.FloatTensor: - return torch.nn.functional.linear(input, self.sdnq_decompressor(self.weight), self.bias) + return torch.nn.functional.linear(input, self.sdnq_dequantizer(self.weight), self.bias) def get_conv_args(input_ndim: int, stride, padding, dilation): @@ -328,12 +328,12 @@ def get_conv_args(input_ndim: int, stride, padding, dilation): def quantized_conv_forward_fp8_matmul(self, input) -> torch.FloatTensor: if torch.numel(input) / input.shape[2] < 32: - return self._conv_forward(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) + return self._conv_forward(input, self.sdnq_dequantizer(self.weight, skip_quantized_matmul=True), self.bias) conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) return conv_fp8_matmul( input, self.weight, self.bias, - self.sdnq_decompressor.scale, - self.sdnq_decompressor.result_shape, + self.sdnq_dequantizer.scale, + self.sdnq_dequantizer.result_shape, self._reversed_padding_repeated_twice, self.padding_mode, conv_type, self.groups, stride, padding, dilation, @@ -342,12 +342,12 @@ def quantized_conv_forward_fp8_matmul(self, input) -> torch.FloatTensor: def quantized_conv_forward_fp8_matmul_tensorwise(self, input) -> torch.FloatTensor: if torch.numel(input) / input.shape[2] < 32: - return self._conv_forward(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) + return self._conv_forward(input, self.sdnq_dequantizer(self.weight, skip_quantized_matmul=True), self.bias) conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) return conv_fp8_matmul_tensorwise( input, self.weight, self.bias, - self.sdnq_decompressor.scale, - self.sdnq_decompressor.result_shape, + self.sdnq_dequantizer.scale, + self.sdnq_dequantizer.result_shape, self._reversed_padding_repeated_twice, self.padding_mode, conv_type, self.groups, stride, padding, dilation, @@ -356,14 +356,14 @@ def quantized_conv_forward_fp8_matmul_tensorwise(self, input) -> torch.FloatTens def quantized_conv_forward_int8_matmul(self, input) -> torch.FloatTensor: if torch.numel(input) / input.shape[2] < 32: - return self._conv_forward(input, self.sdnq_decompressor(self.weight, skip_quantized_matmul=True), self.bias) + return self._conv_forward(input, self.sdnq_dequantizer(self.weight, skip_quantized_matmul=True), self.bias) conv_type, stride, padding, dilation = get_conv_args(input.ndim, self.stride, self.padding, self.dilation) return conv_int8_matmul( input, self.weight, self.bias, - self.sdnq_decompressor.scale, - self.sdnq_decompressor.result_shape, - getattr(self.sdnq_decompressor, "compressed_weight_shape", None), - self.sdnq_decompressor.weights_dtype, + self.sdnq_dequantizer.scale, + self.sdnq_dequantizer.result_shape, + getattr(self.sdnq_dequantizer, "quantized_weight_shape", None), + self.sdnq_dequantizer.weights_dtype, self._reversed_padding_repeated_twice, self.padding_mode, conv_type, self.groups, stride, padding, dilation, @@ -371,25 +371,25 @@ def quantized_conv_forward_int8_matmul(self, input) -> torch.FloatTensor: def quantized_conv_forward(self, input) -> torch.FloatTensor: - return self._conv_forward(input, self.sdnq_decompressor(self.weight), self.bias) + return self._conv_forward(input, self.sdnq_dequantizer(self.weight), self.bias) def quantized_conv_transpose_1d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 1, self.dilation) - return torch.nn.functional.conv_transpose1d(input, self.sdnq_decompressor(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + return torch.nn.functional.conv_transpose1d(input, self.sdnq_dequantizer(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) def quantized_conv_transpose_2d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 2, self.dilation) - return torch.nn.functional.conv_transpose2d(input, self.sdnq_decompressor(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + return torch.nn.functional.conv_transpose2d(input, self.sdnq_dequantizer(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) def quantized_conv_transpose_3d_forward(self, input: torch.FloatTensor, output_size: Optional[list[int]] = None) -> torch.FloatTensor: output_padding = self._output_padding(input, output_size, self.stride, self.padding, self.kernel_size, 3, self.dilation) - return torch.nn.functional.conv_transpose3d(input, self.sdnq_decompressor(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) + return torch.nn.functional.conv_transpose3d(input, self.sdnq_dequantizer(self.weight), self.bias, self.stride, self.padding, output_padding, self.groups, self.dilation) -if shared.opts.sdnq_decompress_compile: +if shared.opts.sdnq_dequantize_compile: try: torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) int8_matmul = torch.compile(int8_matmul, fullgraph=True) diff --git a/modules/shared.py b/modules/shared.py index 0e7099c38..4232d3c65 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -521,11 +521,11 @@ options_templates.update(options_section(("quantization", "Quantization Settings "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "float8_e4m3fn", "int7", "int6", "int5", "uint4", "uint3", "uint2", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "uint8", "uint7", "uint6", "uint5", "int4", "int3", "int2", "uint1"], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), - "sdnq_decompress_compile": OptionInfo(devices.has_triton(), "Decompress using torch.compile", gr.Checkbox, {"visible": native}), + "sdnq_dequantize_compile": OptionInfo(devices.has_triton(), "Dequantize using torch.compile", gr.Checkbox, {"visible": native}), "sdnq_use_quantized_matmul": OptionInfo(False, "Use Quantized MatMul", gr.Checkbox, {"visible": native}), "sdnq_use_quantized_matmul_conv": OptionInfo(False, "Use Quantized MatMul with convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_quantize_with_gpu": OptionInfo(True, "Quantize with the GPU", gr.Checkbox, {"visible": native}), - "sdnq_decompress_fp32": OptionInfo(False, "Decompress using full precision", gr.Checkbox, {"visible": native}), + "sdnq_dequantize_fp32": OptionInfo(False, "Dequantize using full precision", gr.Checkbox, {"visible": native}), "sdnq_quantize_shuffle_weights": OptionInfo(False, "Shuffle weights in post mode", gr.Checkbox, {"visible": native}), "bnb_quantization_sep": OptionInfo("

BitsAndBytes

", "", gr.HTML), diff --git a/wiki b/wiki index 70ea13a0c..04cfb75b8 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 70ea13a0c1af02184777d562e7b5d908077e639d +Subproject commit 04cfb75b8911c227109c0b0dbe64f11f71ef5619 From 41f14df8f55a6f2e51048f38ddf02b1da102aa30 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 12 Jun 2025 14:17:36 +0300 Subject: [PATCH 46/63] Fix TAESD and double downloading with Lumina2 --- CHANGELOG.md | 2 +- modules/model_lumina.py | 7 +++---- modules/sd_vae_taesd.py | 8 +++----- modules/shared_items.py | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dcd87c17..77cb827e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ - Meissonic with multiple generators - Kandinsky V2.2 invalid attention processor - PixArt Sigma Small and Large loading - - TAESD previews with PixArt + - TAESD previews with PixArt and Lumina 2 - VAE Tiling with non-default tile sizes ## Update for 2025-06-02 diff --git a/modules/model_lumina.py b/modules/model_lumina.py index f9d3b9abd..5f46da24f 100644 --- a/modules/model_lumina.py +++ b/modules/model_lumina.py @@ -23,7 +23,7 @@ def load_lumina2(checkpoint_info, diffusers_load_config={}): transformer = diffusers.Lumina2Transformer2DModel.from_pretrained( repo_id, subfolder="transformer", - cache_dir=shared.opts.hfcache_dir, + cache_dir=shared.opts.diffusers_dir, **load_config, **quant_config, ) @@ -32,14 +32,13 @@ def load_lumina2(checkpoint_info, diffusers_load_config={}): text_encoder = transformers.AutoModel.from_pretrained( repo_id, subfolder="text_encoder", - cache_dir=shared.opts.hfcache_dir, - torch_dtype=devices.dtype, + cache_dir=shared.opts.diffusers_dir, **load_config, **quant_config, ) load_config, quant_config = model_quant.get_dit_args(diffusers_load_config, allow_quant=False) - pipe = diffusers.Lumina2Text2ImgPipeline.from_pretrained( + pipe = diffusers.Lumina2Pipeline.from_pretrained( repo_id, cache_dir=shared.opts.diffusers_dir, text_encoder=text_encoder, diff --git a/modules/sd_vae_taesd.py b/modules/sd_vae_taesd.py index 7f6bf191b..3bcd1b322 100644 --- a/modules/sd_vae_taesd.py +++ b/modules/sd_vae_taesd.py @@ -36,7 +36,7 @@ prev_cls = '' prev_type = '' prev_model = '' lock = threading.Lock() -supported = ['sd', 'sdxl', 'f1', 'h1', 'hunyuanvideo', 'wanvideo', 'mochivideo', 'pixartsigma', 'pixartalpha'] +supported = ['sd', 'sdxl', 'f1', 'h1', 'lumina2', 'hunyuanvideo', 'wanvideo', 'mochivideo', 'pixartsigma', 'pixartalpha'] def warn_once(msg, variant=None): @@ -53,14 +53,12 @@ def get_model(model_type = 'decoder', variant = None): global prev_cls, prev_type, prev_model # pylint: disable=global-statement from modules import shared cls = shared.sd_model_type - if cls == 'ldm': # original backend + if cls in {'ldm', 'pixartalpha'}: cls = 'sd' - elif cls == 'h1': # hidream uses flux vae + elif cls in {'h1', 'lumina2'}: cls = 'f1' elif cls == 'pixartsigma': cls = 'sdxl' - elif cls == 'pixartalpha': - cls = 'sd' elif cls not in supported: warn_once(f'cls={shared.sd_model.__class__.__name__} type={cls} unsuppported', variant=variant) variant = variant or shared.opts.taesd_variant diff --git a/modules/shared_items.py b/modules/shared_items.py index d8796daf5..9a09b8c7f 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -29,7 +29,7 @@ pipelines = { 'FLEX': getattr(diffusers, 'AutoPipelineForText2Image', None), 'Sana': getattr(diffusers, 'SanaPipeline', None), 'Lumina-Next': getattr(diffusers, 'LuminaText2ImgPipeline', None), - 'Lumina 2': getattr(diffusers, 'Lumina2Text2ImgPipeline', None), + 'Lumina 2': getattr(diffusers, 'Lumina2Pipeline', None), 'AuraFlow': getattr(diffusers, 'AuraFlowPipeline', None), 'Kandinsky 2.1': getattr(diffusers, 'KandinskyCombinedPipeline', None), 'Kandinsky 2.2': getattr(diffusers, 'KandinskyV22CombinedPipeline', None), From c8f947827b1812a59793e63056aa675aca9bdbf8 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 12 Jun 2025 19:46:05 +0300 Subject: [PATCH 47/63] IPEX fix Lumina2 --- CHANGELOG.md | 1 + modules/intel/ipex/diffusers.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77cb827e4..7b4dc84c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ - PixArt Sigma Small and Large loading - TAESD previews with PixArt and Lumina 2 - VAE Tiling with non-default tile sizes + - Lumina 2 with IPEX ## Update for 2025-06-02 diff --git a/modules/intel/ipex/diffusers.py b/modules/intel/ipex/diffusers.py index 033b74cbe..d3487fefd 100644 --- a/modules/intel/ipex/diffusers.py +++ b/modules/intel/ipex/diffusers.py @@ -81,14 +81,46 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos, output_type="np"): return emb +def apply_rotary_emb(x, freqs_cis, use_real: bool = True, use_real_unbind_dim: int = -1): + if use_real: + cos, sin = freqs_cis # [S, D] + cos = cos[None, None] + sin = sin[None, None] + cos, sin = cos.to(x.device), sin.to(x.device) + + if use_real_unbind_dim == -1: + # Used for flux, cogvideox, hunyuan-dit + x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) # [B, S, H, D//2] + x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3) + elif use_real_unbind_dim == -2: + # Used for Stable Audio, OmniGen, CogView4 and Cosmos + x_real, x_imag = x.reshape(*x.shape[:-1], 2, -1).unbind(-2) # [B, S, H, D//2] + x_rotated = torch.cat([-x_imag, x_real], dim=-1) + else: + raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.") + + out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + return out + else: + # used for lumina + # force cpu with Alchemist + x_rotated = torch.view_as_complex(x.to("cpu").float().reshape(*x.shape[:-1], -1, 2)) + freqs_cis = freqs_cis.to("cpu").unsqueeze(2) + x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(3) + return x_out.type_as(x).to(x.device) + + def ipex_diffusers(device_supports_fp64=False): diffusers.utils.torch_utils.fourier_filter = fourier_filter if not device_supports_fp64: # get around lazy imports + from diffusers.models import embeddings as diffusers_embeddings # pylint: disable=import-error, unused-import # noqa: F401 from diffusers.models import transformers as diffusers_transformers # pylint: disable=import-error, unused-import # noqa: F401 from diffusers.models import controlnets as diffusers_controlnets # pylint: disable=import-error, unused-import # noqa: F401 diffusers.models.embeddings.get_1d_sincos_pos_embed_from_grid = get_1d_sincos_pos_embed_from_grid diffusers.models.embeddings.FluxPosEmbed = FluxPosEmbed + diffusers.models.embeddings.apply_rotary_emb = apply_rotary_emb diffusers.models.transformers.transformer_flux.FluxPosEmbed = FluxPosEmbed + diffusers.models.transformers.transformer_lumina2.apply_rotary_emb = apply_rotary_emb diffusers.models.controlnets.controlnet_flux.FluxPosEmbed = FluxPosEmbed diffusers.models.transformers.transformer_hidream_image.rope = hidream_rope From cb4684cbebac0aed5f673fce6913f7c19b3729ad Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 13 Jun 2025 12:42:57 +0300 Subject: [PATCH 48/63] SNDQ add separate quant mode option for Text Encoders --- CHANGELOG.md | 1 + modules/model_quant.py | 17 ++++++++++++++--- modules/shared.py | 1 + wiki | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b4dc84c4..ea24e138b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - Add group size support for convolutional layers - Add quantized matmul support for for convolutional layers - Add 7-bit, 5-bit and 3-bit quantization support + - Add separate quant mode option for Text Encoders - Fix forced FP32 with tensorwise FP8 matmul - Fix PyTorch <= 2.4 compatibility with FP8 matmul - Fix VAE with conv quant diff --git a/modules/model_quant.py b/modules/model_quant.py index 6e2429e35..b90297ee6 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -113,14 +113,20 @@ def create_sdnq_config(kwargs = None, allow_sdnq: bool = True, module: str = 'Mo diffusers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["sdnq"] = SDNQConfig transformers.quantizers.auto.AUTO_QUANTIZATION_CONFIG_MAPPING["sdnq"] = SDNQConfig + if weights_dtype is None: + if shared.opts.sdnq_quantize_weights_mode_te != "default" and module in {"TE", "LLM"}: + weights_dtype = shared.opts.sdnq_quantize_weights_mode_te + else: + weights_dtype = shared.opts.sdnq_quantize_weights_mode + sdnq_config = SDNQConfig( - weights_dtype=weights_dtype if weights_dtype is not None else shared.opts.sdnq_quantize_weights_mode, + weights_dtype=weights_dtype, group_size=shared.opts.sdnq_quantize_weights_group_size, quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, ) - log.debug(f'Quantization: module="{module}" type=sdnq dtype={shared.opts.sdnq_quantize_weights_mode}') + log.debug(f'Quantization: module="{module}" type=sdnq dtype={weights_dtype}') if kwargs is None: return sdnq_config else: @@ -320,9 +326,14 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc=True): if hasattr(model, "get_input_embeddings"): backup_embeddings = copy.deepcopy(model.get_input_embeddings()) + if shared.opts.sdnq_quantize_weights_mode_te != "default" and op is not None and "text_encoder" in op: + weights_dtype = shared.opts.sdnq_quantize_weights_mode_te + else: + weights_dtype = shared.opts.sdnq_quantize_weights_mode + model = apply_sdnq_to_module( model, - weights_dtype=shared.opts.sdnq_quantize_weights_mode, + weights_dtype=weights_dtype, torch_dtype=devices.dtype, group_size=shared.opts.sdnq_quantize_weights_group_size, quant_conv=shared.opts.sdnq_quantize_conv_layers, diff --git a/modules/shared.py b/modules/shared.py index 4232d3c65..2231e0484 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -519,6 +519,7 @@ options_templates.update(options_section(("quantization", "Quantization Settings "sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "Transformer", "VAE", "TE", "Video", "LLM", "ControlNet"], "visible": native}), "sdnq_quantize_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ["pre", "post"], "visible": native}), "sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": ["int8", "float8_e4m3fn", "int7", "int6", "int5", "uint4", "uint3", "uint2", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "uint8", "uint7", "uint6", "uint5", "int4", "int3", "int2", "uint1"], "visible": native}), + "sdnq_quantize_weights_mode_te": OptionInfo("default", "Quantization type for Text Encoders", gr.Dropdown, {"choices": ["default", "int8", "float8_e4m3fn", "int7", "int6", "int5", "uint4", "uint3", "uint2", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz", "uint8", "uint7", "uint6", "uint5", "int4", "int3", "int2", "uint1"], "visible": native}), "sdnq_quantize_weights_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1, "visible": native}), "sdnq_quantize_conv_layers": OptionInfo(False, "Quantize the convolutional layers", gr.Checkbox, {"visible": native}), "sdnq_dequantize_compile": OptionInfo(devices.has_triton(), "Dequantize using torch.compile", gr.Checkbox, {"visible": native}), diff --git a/wiki b/wiki index 04cfb75b8..19a1dca01 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 04cfb75b8911c227109c0b0dbe64f11f71ef5619 +Subproject commit 19a1dca01821204cc114f7a955705322b6186ee0 From e68f9272e8d17565b6bc56f657a5940913082d54 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 13 Jun 2025 13:05:46 +0300 Subject: [PATCH 49/63] Disable custom atten processors for non SD 1.5 / SDXL models --- CHANGELOG.md | 3 ++- modules/sd_models.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea24e138b..abe785428 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Set CPU backend to use FP32 by default - Relax Python version checks for Zluda - Make VAE options not require model reload + - Add warning about incompatible attention processors - **Torch** - Set default to `torch==2.7.1` @@ -32,7 +33,7 @@ - **Fixes** - Meissonic with multiple generators - - Kandinsky V2.2 invalid attention processor + - Invalid attention processors - PixArt Sigma Small and Large loading - TAESD previews with PixArt and Lumina 2 - VAE Tiling with non-default tile sizes diff --git a/modules/sd_models.py b/modules/sd_models.py index 7cf4f4993..0bea615ff 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -920,8 +920,13 @@ def set_diffusers_attention(pipe, quiet:bool=False): # if hasattr(pipe, 'pipe'): # set_diffusers_attention(pipe.pipe) - if 'ControlNet' in pipe.__class__.__name__: # do not replace attention in ControlNet pipelines + if 'ControlNet' in pipe.__class__.__name__ or not (pipe.__class__.__name__.startswith("StableDiffusion") and hasattr(pipe, "unet")): + if shared.opts.cross_attention_optimization not in {"Scaled-Dot-Product", "Disabled"}: + shared.log.warning(f"Attention: {shared.opts.cross_attention_optimization} is not compatible with {pipe.__class__.__name__}") + else: + pipe.current_attn_name = shared.opts.cross_attention_optimization return + shared.log.quiet(quiet, f'Setting model: attention="{shared.opts.cross_attention_optimization}"') if shared.opts.cross_attention_optimization == "Disabled": pass # do nothing From 1fca56517853876bd58d2d0ff8ee5d82deac5d35 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 13 Jun 2025 13:37:12 +0300 Subject: [PATCH 50/63] Cleanup --- modules/model_flux.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/model_flux.py b/modules/model_flux.py index 17ebe06c7..4381e1e50 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -14,7 +14,6 @@ debug = shared.log.trace if os.environ.get('SD_LOAD_DEBUG', None) is not None el def load_flux_quanto(checkpoint_info): transformer, text_encoder_2 = None, None quanto = model_quant.load_quanto('Load model: type=FLUX') - quanto.tensor.qbits.QBitsTensor.create = lambda *args, **kwargs: quanto.tensor.qbits.QBitsTensor(*args, **kwargs) if isinstance(checkpoint_info, str): repo_path = checkpoint_info From fb7280c3f4afefa73e0f2b1441a4af21667c9192 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 13 Jun 2025 13:40:44 +0300 Subject: [PATCH 51/63] Flux quanto fix logged dtype --- modules/model_flux.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/model_flux.py b/modules/model_flux.py index 4381e1e50..3a5068377 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -35,11 +35,12 @@ def load_flux_quanto(checkpoint_info): quanto.requantize(transformer, state_dict, quantization_map, device=torch.device("cpu")) if shared.opts.diffusers_eval: transformer.eval() - if transformer.dtype != devices.dtype: + transformer_dtype = transformer.dtype + if transformer_dtype != devices.dtype: try: transformer = transformer.to(dtype=devices.dtype) except Exception: - shared.log.error(f"Load model: type=FLUX Failed to cast transformer to {devices.dtype}, set dtype to {transformer.dtype}") + shared.log.error(f"Load model: type=FLUX Failed to cast transformer to {devices.dtype}, set dtype to {transformer_dtype}") except Exception as e: shared.log.error(f"Load model: type=FLUX failed to load Quanto transformer: {e}") if debug: From 90e76b2023598d1eae27e6e9d649b0316c44a053 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 13 Jun 2025 13:42:13 +0300 Subject: [PATCH 52/63] Cleanup --- modules/model_flux.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/model_flux.py b/modules/model_flux.py index 3a5068377..dab00d86a 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -63,11 +63,12 @@ def load_flux_quanto(checkpoint_info): quanto.requantize(text_encoder_2, state_dict, quantization_map, device=torch.device("cpu")) if shared.opts.diffusers_eval: text_encoder_2.eval() - if text_encoder_2.dtype != devices.dtype: + text_encoder_2_dtype = text_encoder_2.dtype + if text_encoder_2_dtype != devices.dtype: try: text_encoder_2 = text_encoder_2.to(dtype=devices.dtype) except Exception: - shared.log.error(f"Load model: type=FLUX Failed to cast text encoder to {devices.dtype}, set dtype to {text_encoder_2.dtype}") + shared.log.error(f"Load model: type=FLUX Failed to cast text encoder to {devices.dtype}, set dtype to {text_encoder_2_dtype}") except Exception as e: shared.log.error(f"Load model: type=FLUX failed to load Quanto text encoder: {e}") if debug: From 45827a923f18b9ad9715bc536c82e19d72472000 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 13 Jun 2025 16:20:21 +0300 Subject: [PATCH 53/63] IPEX fix torch.cuda.set_device --- modules/intel/ipex/__init__.py | 1 - modules/intel/ipex/hijacks.py | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py index 369367ef8..a44531f35 100644 --- a/modules/intel/ipex/__init__.py +++ b/modules/intel/ipex/__init__.py @@ -39,7 +39,6 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.is_available = torch.xpu.is_available torch.cuda.is_initialized = torch.xpu.is_initialized torch.cuda.is_current_stream_capturing = lambda: False - torch.cuda.set_device = torch.xpu.set_device torch.cuda.stream = torch.xpu.stream torch.cuda.Event = torch.xpu.Event torch.cuda.Stream = torch.xpu.Stream diff --git a/modules/intel/ipex/hijacks.py b/modules/intel/ipex/hijacks.py index 0ce8abdc5..d81d7b05c 100644 --- a/modules/intel/ipex/hijacks.py +++ b/modules/intel/ipex/hijacks.py @@ -379,6 +379,12 @@ def torch_cuda_device(device): else: return torch.xpu.device(device) +@wraps(torch.cuda.set_device) +def torch_cuda_set_device(device): + if check_cuda(device): + torch.xpu.set_device(return_xpu(device)) + else: + torch.xpu.set_device(device) # torch.Generator has to be a class for isinstance checks original_torch_Generator = torch.Generator @@ -412,6 +418,7 @@ def ipex_hijacks(): torch.load = torch_load torch.cuda.synchronize = torch_cuda_synchronize torch.cuda.device = torch_cuda_device + torch.cuda.set_device = torch_cuda_set_device torch.Generator = torch_Generator torch._C.Generator = torch_Generator From fb72c6f54090d5775a000f9ddf07d998ab5b63f8 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 13 Jun 2025 21:32:06 +0300 Subject: [PATCH 54/63] Zluda use exact torch version --- installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer.py b/installer.py index f1997ec2e..0c617b92c 100644 --- a/installer.py +++ b/installer.py @@ -655,7 +655,7 @@ def install_rocm_zluda(): if error is None: try: zluda_installer.load() - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1 torchvision --index-url https://download.pytorch.org/whl/cu118') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.7.1+cu118 torchvision==0.22.1+cu118 --index-url https://download.pytorch.org/whl/cu118') except Exception as e: error = e log.warning(f'Failed to load ZLUDA: {e}') From 2ba64abcde9cfabdfac05bdc21947a47f6c279b5 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 14 Jun 2025 00:54:18 +0300 Subject: [PATCH 55/63] Cleanup --- modules/lora/lora_apply.py | 2 ++ modules/model_quant.py | 4 ++++ modules/sdnq/__init__.py | 20 +++++++++++++++----- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index 62e9cbdfa..205a64e96 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -155,6 +155,8 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, + quantize_with_gpu=shared.opts.sdnq_quantize_with_gpu, + dequantize_fp32=shared.opts.sdnq_dequantize_fp32, param_name=getattr(self, 'network_layer_name', None), ) self = self.to(device) diff --git a/modules/model_quant.py b/modules/model_quant.py index b90297ee6..29ae5dd7f 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -125,6 +125,8 @@ def create_sdnq_config(kwargs = None, allow_sdnq: bool = True, module: str = 'Mo quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, + quantize_with_gpu=shared.opts.sdnq_quantize_with_gpu, + dequantize_fp32=shared.opts.sdnq_dequantize_fp32, ) log.debug(f'Quantization: module="{module}" type=sdnq dtype={weights_dtype}') if kwargs is None: @@ -339,6 +341,8 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc=True): quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, + quantize_with_gpu=shared.opts.sdnq_quantize_with_gpu, + dequantize_fp32=shared.opts.sdnq_dequantize_fp32, param_name=op, ) model.quantization_method = 'SDNQ' diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index f7607fcd1..fff1a3126 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -14,7 +14,7 @@ from .dequantizer import dequantizer_dict from .forward import get_forward_func -def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, param_name=None, pre_mode=False): +def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, quantize_with_gpu=True, dequantize_fp32=False, param_name=None, pre_mode=False): layer_class_name = layer.__class__.__name__ if layer_class_name in allowed_types: is_conv_type = False @@ -111,20 +111,20 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz elif pre_mode: if shared.opts.device_map == "gpu": return_device = devices.device - elif shared.opts.sdnq_quantize_with_gpu: + elif quantize_with_gpu: return_device = devices.cpu else: return_device = layer.weight.device else: return_device = layer.weight.device if not pre_mode: - if shared.opts.sdnq_quantize_with_gpu: + if quantize_with_gpu: layer.weight.data = layer.weight.to(devices.device).to(dtype=torch.float32) else: layer.weight.data = layer.weight.to(dtype=torch.float32) layer.weight.data, scale, zero_point = quantize_weight(layer.weight, reduction_axes, weights_dtype) - if not shared.opts.sdnq_dequantize_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"] and not use_tensorwise_fp8_matmul): + if not dequantize_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"] and not use_tensorwise_fp8_matmul): scale = scale.to(torch_dtype) if zero_point is not None: zero_point = zero_point.to(torch_dtype) @@ -158,7 +158,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz return layer -def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, param_name=None): # pylint: disable=unused-argument +def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, quantize_with_gpu=True, dequantize_fp32=False, param_name=None): # pylint: disable=unused-argument has_children = list(model.children()) if not has_children: return model @@ -172,6 +172,8 @@ def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_si quant_conv=quant_conv, use_quantized_matmul=use_quantized_matmul, use_quantized_matmul_conv=use_quantized_matmul_conv, + quantize_with_gpu=quantize_with_gpu, + dequantize_fp32=dequantize_fp32, param_name=module_param_name, ) module = apply_sdnq_to_module( @@ -182,6 +184,8 @@ def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_si quant_conv=quant_conv, use_quantized_matmul=use_quantized_matmul, use_quantized_matmul_conv=use_quantized_matmul_conv, + quantize_with_gpu=quantize_with_gpu, + dequantize_fp32=dequantize_fp32, param_name=module_param_name, ) return model @@ -295,6 +299,8 @@ class SDNQQuantizer(DiffusersQuantizer): quant_conv=self.quantization_config.quant_conv, use_quantized_matmul=self.quantization_config.use_quantized_matmul, use_quantized_matmul_conv=self.quantization_config.use_quantized_matmul_conv, + quantize_with_gpu=self.quantization_config.quantize_with_gpu, + dequantize_fp32=self.quantization_config.dequantize_fp32, param_name=param_name, pre_mode=True, ) @@ -389,6 +395,8 @@ class SDNQConfig(QuantizationConfigMixin): quant_conv: bool = False, use_quantized_matmul: bool = False, use_quantized_matmul_conv: bool = False, + quantize_with_gpu: bool = True, + dequantize_fp32: bool = False, modules_to_not_convert: Optional[List[str]] = None, **kwargs, # pylint: disable=unused-argument ): @@ -398,6 +406,8 @@ class SDNQConfig(QuantizationConfigMixin): self.quant_conv = quant_conv self.use_quantized_matmul = use_quantized_matmul self.use_quantized_matmul_conv = use_quantized_matmul_conv + self.quantize_with_gpu = quantize_with_gpu, + self.dequantize_fp32 = dequantize_fp32, self.modules_to_not_convert = modules_to_not_convert self.post_init() self.is_integer = dtype_dict[self.weights_dtype]["is_integer"] From 8f8e5ce1b0081651c4ec4c1e2c5d5333c101f438 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 14 Jun 2025 01:08:25 +0300 Subject: [PATCH 56/63] Cleanup x2 --- modules/sdnq/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index fff1a3126..41e17b643 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -280,7 +280,7 @@ class SDNQQuantizer(DiffusersQuantizer): ): # load the model params to target_device first layer, _ = get_module_from_name(model, param_name) - if shared.opts.sdnq_quantize_with_gpu: + if self.quantization_config.quantize_with_gpu: if param_value.dtype == torch.float32 and devices.same_device(param_value.device, devices.device): param_value = param_value.clone() else: From c01802d9fffcc735cd53bdf52cb5eb4cba3b51ed Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 14 Jun 2025 01:13:51 +0300 Subject: [PATCH 57/63] SDNQ fix transformers llm --- modules/sdnq/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index 41e17b643..bf6da021c 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -372,6 +372,10 @@ class SDNQQuantizer(DiffusersQuantizer): def is_serializable(self): return True + @property + def is_compileable(self): + return True + @dataclass class SDNQConfig(QuantizationConfigMixin): From fd583523f718c66ee1dca78249fd79cdcf1ef7b7 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 14 Jun 2025 11:47:34 +0300 Subject: [PATCH 58/63] Update requirements --- installer.py | 2 +- requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/installer.py b/installer.py index 0c617b92c..bf4d0b408 100644 --- a/installer.py +++ b/installer.py @@ -546,7 +546,7 @@ def check_diffusers(): t_start = time.time() if args.skip_all or args.skip_git or args.experimental: return - sha = '6508da6f06a0da1054ae6a808d0025c04b70f0e8' # diffusers commit hash + sha = '8adc6003ba4dbf5b61bb4f1ce571e9e55e145a99' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/requirements.txt b/requirements.txt index 1de823e6d..f5eb92014 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ pi-heif # versioned rich==14.0.0 safetensors==0.5.3 -tensordict==0.1.2 +tensordict==0.8.3 peft==0.15.2 httpx==0.24.1 compel==2.0.3 @@ -45,7 +45,7 @@ accelerate==1.7.0 opencv-contrib-python-headless==4.9.0.80 einops==0.4.1 gradio==3.43.2 -huggingface_hub==0.31.2 +huggingface_hub==0.33.0 numexpr==2.10.2 numpy==1.26.4 pandas==2.3.0 @@ -53,7 +53,7 @@ numba==0.61.2 protobuf==4.25.3 pytorch_lightning==1.9.4 tokenizers==0.21.1 -transformers==4.52.3 +transformers==4.52.4 urllib3==1.26.19 Pillow==10.4.0 timm==0.9.16 From 24194201cf11d1b3270dca8fec174e2b90e0b44c Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 14 Jun 2025 19:55:43 +0300 Subject: [PATCH 59/63] Fix OmniGen --- CHANGELOG.md | 1 + modules/omnigen/transformer.py | 5 +++++ modules/processing_vae.py | 4 ++-- modules/sd_models_utils.py | 2 ++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abe785428..d3ec2c882 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ - **Fixes** - Meissonic with multiple generators + - OmniGen with new transformers - Invalid attention processors - PixArt Sigma Small and Large loading - TAESD previews with PixArt and Lumina 2 diff --git a/modules/omnigen/transformer.py b/modules/omnigen/transformer.py index f3bcdb15a..d166309ca 100644 --- a/modules/omnigen/transformer.py +++ b/modules/omnigen/transformer.py @@ -99,6 +99,9 @@ class Phi3Transformer(Phi3Model): hidden_states = inputs_embeds + # create position embeddings to be shared across the decoder layers + position_embeddings = self.rotary_emb(hidden_states, position_ids) + # decoder layers all_hidden_states = () if output_hidden_states else None all_self_attns = () if output_attentions else None @@ -118,6 +121,7 @@ class Phi3Transformer(Phi3Model): output_attentions, use_cache, cache_position, + position_embeddings, ) else: layer_outputs = decoder_layer( @@ -128,6 +132,7 @@ class Phi3Transformer(Phi3Model): output_attentions=output_attentions, use_cache=use_cache, cache_position=cache_position, + position_embeddings=position_embeddings, ) hidden_states = layer_outputs[0] diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 290c0d489..36d5d0dda 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -256,8 +256,8 @@ def vae_postprocess(tensor, model, output_type='np'): if output_type == "pil": images = model.numpy_to_pil(images) else: - import diffusers - model.image_processor = diffusers.image_processor.VaeImageProcessor() + from diffusers.image_processor import VaeImageProcessor + model.image_processor = VaeImageProcessor() images = model.image_processor.postprocess(tensor, output_type=output_type) else: images = tensor if isinstance(tensor, list) or isinstance(tensor, np.ndarray) else [tensor] diff --git a/modules/sd_models_utils.py b/modules/sd_models_utils.py index 0e1b72f4c..546297d25 100644 --- a/modules/sd_models_utils.py +++ b/modules/sd_models_utils.py @@ -155,6 +155,8 @@ def apply_function_to_model(sd_model, function, options, op=None): if hasattr(sd_model, 'transformer') and hasattr(sd_model.transformer, 'config'): sd_model.transformer = function(sd_model.transformer, op="transformer", sd_model=sd_model) if "Model" in options: + if hasattr(sd_model, 'model') and (hasattr(sd_model.model, 'config') or isinstance(sd_model.model, torch.nn.Module)): + sd_model.model = function(sd_model.model, op="model", sd_model=sd_model) if hasattr(sd_model, 'unet') and hasattr(sd_model.unet, 'config'): sd_model.unet = function(sd_model.unet, op="unet", sd_model=sd_model) if hasattr(sd_model, 'decoder_pipe') and hasattr(sd_model, 'decoder'): From 25fc0094a989080a4a0b957195661e2e8b263378 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 14 Jun 2025 21:29:08 +0300 Subject: [PATCH 60/63] SDNQ use quantize_device and return_device args and fix decompress_fp32 always being on --- modules/lora/lora_apply.py | 3 +- modules/model_quant.py | 31 ++++++++++++++++-- modules/sdnq/__init__.py | 66 ++++++++++++++++++-------------------- wiki | 2 +- 4 files changed, 62 insertions(+), 40 deletions(-) diff --git a/modules/lora/lora_apply.py b/modules/lora/lora_apply.py index 205a64e96..b971541cd 100644 --- a/modules/lora/lora_apply.py +++ b/modules/lora/lora_apply.py @@ -155,8 +155,9 @@ def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.G quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, - quantize_with_gpu=shared.opts.sdnq_quantize_with_gpu, dequantize_fp32=shared.opts.sdnq_dequantize_fp32, + quantization_device=devices.device, + return_device=device, param_name=getattr(self, 'network_layer_name', None), ) self = self.to(device) diff --git a/modules/model_quant.py b/modules/model_quant.py index 29ae5dd7f..3c57ac67c 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -104,7 +104,7 @@ def create_quanto_config(kwargs = None, allow_quanto: bool = True, module: str = def create_sdnq_config(kwargs = None, allow_sdnq: bool = True, module: str = 'Model', weights_dtype: str = None): - from modules import shared + from modules import devices, shared if len(shared.opts.sdnq_quantize_weights) > 0 and (shared.opts.sdnq_quantize_mode == 'pre') and allow_sdnq: if 'Model' in shared.opts.sdnq_quantize_weights or (module is not None and module in shared.opts.sdnq_quantize_weights) or module == 'any': from modules.sdnq import SDNQQuantizer, SDNQConfig @@ -119,14 +119,28 @@ def create_sdnq_config(kwargs = None, allow_sdnq: bool = True, module: str = 'Mo else: weights_dtype = shared.opts.sdnq_quantize_weights_mode + if shared.opts.device_map == "gpu": + quantization_device = devices.device + return_device = devices.device + elif shared.opts.diffusers_offload_mode in {"none", "model"}: + quantization_device = devices.device if shared.opts.sdnq_quantize_with_gpu else devices.cpu + return_device = devices.device + elif shared.opts.sdnq_quantize_with_gpu: + quantization_device = devices.device + return_device = devices.cpu + else: + quantization_device = None + return_device = None + sdnq_config = SDNQConfig( weights_dtype=weights_dtype, group_size=shared.opts.sdnq_quantize_weights_group_size, quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, - quantize_with_gpu=shared.opts.sdnq_quantize_with_gpu, dequantize_fp32=shared.opts.sdnq_dequantize_fp32, + quantization_device=quantization_device, + return_device=return_device, ) log.debug(f'Quantization: module="{module}" type=sdnq dtype={weights_dtype}') if kwargs is None: @@ -333,6 +347,16 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc=True): else: weights_dtype = shared.opts.sdnq_quantize_weights_mode + if shared.opts.diffusers_offload_mode in {"none", "model"}: + quantization_device = devices.device if shared.opts.sdnq_quantize_with_gpu else devices.cpu + return_device = devices.device + elif shared.opts.sdnq_quantize_with_gpu: + quantization_device = devices.device + return_device = getattr(model, "device", devices.cpu) + else: + quantization_device = None + return_device = None + model = apply_sdnq_to_module( model, weights_dtype=weights_dtype, @@ -341,8 +365,9 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc=True): quant_conv=shared.opts.sdnq_quantize_conv_layers, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul, use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv, - quantize_with_gpu=shared.opts.sdnq_quantize_with_gpu, dequantize_fp32=shared.opts.sdnq_dequantize_fp32, + quantization_device=quantization_device, + return_device=return_device, param_name=op, ) model.quantization_method = 'SDNQ' diff --git a/modules/sdnq/__init__.py b/modules/sdnq/__init__.py index bf6da021c..f5b7179a7 100644 --- a/modules/sdnq/__init__.py +++ b/modules/sdnq/__init__.py @@ -14,7 +14,7 @@ from .dequantizer import dequantizer_dict from .forward import get_forward_func -def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, quantize_with_gpu=True, dequantize_fp32=False, param_name=None, pre_mode=False): +def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, dequantize_fp32=False, quantization_device=None, return_device=None, param_name=None): layer_class_name = layer.__class__.__name__ if layer_class_name in allowed_types: is_conv_type = False @@ -106,22 +106,12 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz layer.weight.data = layer.weight.reshape(new_shape) layer.weight.requires_grad = False - if shared.opts.diffusers_offload_mode in {"none", "model"}: - return_device = devices.device - elif pre_mode: - if shared.opts.device_map == "gpu": - return_device = devices.device - elif quantize_with_gpu: - return_device = devices.cpu - else: - return_device = layer.weight.device - else: + if return_device is None: return_device = layer.weight.device - if not pre_mode: - if quantize_with_gpu: - layer.weight.data = layer.weight.to(devices.device).to(dtype=torch.float32) - else: - layer.weight.data = layer.weight.to(dtype=torch.float32) + if quantization_device is not None: + layer.weight.data = layer.weight.to(quantization_device) + if layer.weight.dtype != torch.float32: + layer.weight.data = layer.weight.to(dtype=torch.float32) layer.weight.data, scale, zero_point = quantize_weight(layer.weight, reduction_axes, weights_dtype) if not dequantize_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"] and not use_tensorwise_fp8_matmul): @@ -158,7 +148,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz return layer -def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, quantize_with_gpu=True, dequantize_fp32=False, param_name=None): # pylint: disable=unused-argument +def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_size=0, quant_conv=False, use_quantized_matmul=False, use_quantized_matmul_conv=False, dequantize_fp32=False, quantization_device=None, return_device=None, param_name=None): # pylint: disable=unused-argument has_children = list(model.children()) if not has_children: return model @@ -172,8 +162,9 @@ def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_si quant_conv=quant_conv, use_quantized_matmul=use_quantized_matmul, use_quantized_matmul_conv=use_quantized_matmul_conv, - quantize_with_gpu=quantize_with_gpu, dequantize_fp32=dequantize_fp32, + quantization_device=quantization_device, + return_device=return_device, param_name=module_param_name, ) module = apply_sdnq_to_module( @@ -184,8 +175,9 @@ def apply_sdnq_to_module(model, weights_dtype="int8", torch_dtype=None, group_si quant_conv=quant_conv, use_quantized_matmul=use_quantized_matmul, use_quantized_matmul_conv=use_quantized_matmul_conv, - quantize_with_gpu=quantize_with_gpu, dequantize_fp32=dequantize_fp32, + quantization_device=quantization_device, + return_device=return_device, param_name=module_param_name, ) return model @@ -278,18 +270,20 @@ class SDNQQuantizer(DiffusersQuantizer): unexpected_keys: List[str], # pylint: disable=unused-argument **kwargs, # pylint: disable=unused-argument ): - # load the model params to target_device first - layer, _ = get_module_from_name(model, param_name) - if self.quantization_config.quantize_with_gpu: - if param_value.dtype == torch.float32 and devices.same_device(param_value.device, devices.device): - param_value = param_value.clone() - else: - param_value = param_value.to(devices.device).to(dtype=torch.float32) + if self.quantization_config.return_device is not None: + return_device = self.quantization_config.return_device else: - if param_value.dtype == torch.float32 and devices.same_device(param_value.device, target_device): - param_value = param_value.clone() - else: - param_value = param_value.to(target_device).to(dtype=torch.float32) + return_device = target_device + + if self.quantization_config.quantization_device is not None: + target_device = self.quantization_config.quantization_device + + if param_value.dtype == torch.float32 and devices.same_device(param_value.device, target_device): + param_value = param_value.clone() + else: + param_value = param_value.to(target_device).to(dtype=torch.float32) + + layer, _ = get_module_from_name(model, param_name) layer.weight = torch.nn.Parameter(param_value, requires_grad=False) layer = sdnq_quantize_layer( layer, @@ -299,10 +293,10 @@ class SDNQQuantizer(DiffusersQuantizer): quant_conv=self.quantization_config.quant_conv, use_quantized_matmul=self.quantization_config.use_quantized_matmul, use_quantized_matmul_conv=self.quantization_config.use_quantized_matmul_conv, - quantize_with_gpu=self.quantization_config.quantize_with_gpu, dequantize_fp32=self.quantization_config.dequantize_fp32, + quantization_device=None, + return_device=return_device, param_name=param_name, - pre_mode=True, ) def adjust_max_memory(self, max_memory: Dict[str, Union[int, str]]) -> Dict[str, Union[int, str]]: @@ -399,8 +393,9 @@ class SDNQConfig(QuantizationConfigMixin): quant_conv: bool = False, use_quantized_matmul: bool = False, use_quantized_matmul_conv: bool = False, - quantize_with_gpu: bool = True, dequantize_fp32: bool = False, + quantization_device: Optional[torch.device] = None, + return_device: Optional[torch.device] = None, modules_to_not_convert: Optional[List[str]] = None, **kwargs, # pylint: disable=unused-argument ): @@ -410,8 +405,9 @@ class SDNQConfig(QuantizationConfigMixin): self.quant_conv = quant_conv self.use_quantized_matmul = use_quantized_matmul self.use_quantized_matmul_conv = use_quantized_matmul_conv - self.quantize_with_gpu = quantize_with_gpu, - self.dequantize_fp32 = dequantize_fp32, + self.dequantize_fp32 = dequantize_fp32 + self.quantization_device = quantization_device + self.return_device = return_device self.modules_to_not_convert = modules_to_not_convert self.post_init() self.is_integer = dtype_dict[self.weights_dtype]["is_integer"] diff --git a/wiki b/wiki index 19a1dca01..34e99de10 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 19a1dca01821204cc114f7a955705322b6186ee0 +Subproject commit 34e99de10210375593daec696f1b651d82ef0cf0 From d31df8c1eb37c4e02e5433f594c403fbe800c4ce Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 14 Jun 2025 22:10:10 +0300 Subject: [PATCH 61/63] SDNQ fuse bias into dequantizer with matmul --- modules/sdnq/dequantizer.py | 29 ++++++++++++++++++-------- modules/sdnq/forward.py | 41 ++++++++++++++++++++++--------------- 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/modules/sdnq/dequantizer.py b/modules/sdnq/dequantizer.py index 1f2b36df1..be59019a3 100644 --- a/modules/sdnq/dequantizer.py +++ b/modules/sdnq/dequantizer.py @@ -24,6 +24,10 @@ def dequantize_symmetric(input: torch.CharTensor, scale: torch.FloatTensor, dtyp return result +def dequantize_symmetric_with_bias(input: torch.CharTensor, bias: torch.FloatTensor, scale: torch.FloatTensor, dtype: torch.dtype, result_shape: torch.Size) -> torch.FloatTensor: + return torch.addcmul(bias, input.to(dtype=scale.dtype), scale).to(dtype=dtype).reshape(result_shape) + + def dequantize_packed_int_asymmetric(input: torch.ByteTensor, scale: torch.FloatTensor, zero_point: torch.FloatTensor, shape: torch.Size, dtype: torch.dtype, result_shape: torch.Size, weights_dtype: str) -> torch.FloatTensor: return dequantize_asymmetric(packed_int_function_dict[weights_dtype]["unpack"](input, shape), scale, zero_point, dtype, result_shape) @@ -57,7 +61,7 @@ class AsymmetricWeightsDequantizer(torch.nn.Module): return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) def forward(self, weight, **kwargs): # pylint: disable=unused-argument - return dequantize_asymmetric(weight, self.scale, self.zero_point, self.result_dtype, self.result_shape) + return dequantize_asymmetric_compiled(weight, self.scale, self.zero_point, self.result_dtype, self.result_shape) class SymmetricWeightsDequantizer(torch.nn.Module): @@ -81,7 +85,7 @@ class SymmetricWeightsDequantizer(torch.nn.Module): return weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"]) def forward(self, weight, skip_quantized_matmul=False, **kwargs): # pylint: disable=unused-argument - return dequantize_symmetric(weight, self.scale, self.result_dtype, self.result_shape, skip_quantized_matmul=skip_quantized_matmul) + return dequantize_symmetric_compiled(weight, self.scale, self.result_dtype, self.result_shape, skip_quantized_matmul=skip_quantized_matmul) class PackedINTAsymmetricWeightsDequantizer(torch.nn.Module): @@ -108,7 +112,7 @@ class PackedINTAsymmetricWeightsDequantizer(torch.nn.Module): return packed_int_function_dict[self.weights_dtype]["pack"](weight.to(dtype=dtype_dict[self.weights_dtype]["torch_dtype"])) def forward(self, weight, **kwargs): # pylint: disable=unused-argument - return dequantize_packed_int_asymmetric(weight, self.scale, self.zero_point, self.quantized_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype) + return dequantize_packed_int_asymmetric_compiled(weight, self.scale, self.zero_point, self.quantized_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype) class PackedINTSymmetricWeightsDequantizer(torch.nn.Module): @@ -134,7 +138,7 @@ class PackedINTSymmetricWeightsDequantizer(torch.nn.Module): return pack_int_symetric(weight, self.weights_dtype) def forward(self, weight, skip_quantized_matmul=False, **kwargs): # pylint: disable=unused-argument - return dequantize_packed_int_symmetric(weight, self.scale, self.quantized_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype, skip_quantized_matmul=skip_quantized_matmul) + return dequantize_packed_int_symmetric_compiled(weight, self.scale, self.quantized_weight_shape, self.result_dtype, self.result_shape, self.weights_dtype, skip_quantized_matmul=skip_quantized_matmul) dequantizer_dict = { @@ -164,9 +168,18 @@ dequantizer_dict = { if shared.opts.sdnq_dequantize_compile: try: torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) - dequantize_asymmetric = torch.compile(dequantize_asymmetric, fullgraph=True) - dequantize_symmetric = torch.compile(dequantize_symmetric, fullgraph=True) - dequantize_packed_int_asymmetric = torch.compile(dequantize_packed_int_asymmetric, fullgraph=True) - dequantize_packed_int_symmetric = torch.compile(dequantize_packed_int_symmetric, fullgraph=True) + dequantize_asymmetric_compiled = torch.compile(dequantize_asymmetric, fullgraph=True) + dequantize_symmetric_compiled = torch.compile(dequantize_symmetric, fullgraph=True) + dequantize_packed_int_asymmetric_compiled = torch.compile(dequantize_packed_int_asymmetric, fullgraph=True) + dequantize_packed_int_symmetric_compiled = torch.compile(dequantize_packed_int_symmetric, fullgraph=True) except Exception as e: shared.log.warning(f"Quantization: type=sdnq Dequantize using torch.compile is not available: {e}") + dequantize_asymmetric_compiled = dequantize_asymmetric + dequantize_symmetric_compiled = dequantize_symmetric + dequantize_packed_int_asymmetric_compiled = dequantize_packed_int_asymmetric + dequantize_packed_int_symmetric_compiled = dequantize_packed_int_symmetric +else: + dequantize_asymmetric_compiled = dequantize_asymmetric + dequantize_symmetric_compiled = dequantize_symmetric + dequantize_packed_int_asymmetric_compiled = dequantize_packed_int_asymmetric + dequantize_packed_int_symmetric_compiled = dequantize_packed_int_symmetric diff --git a/modules/sdnq/forward.py b/modules/sdnq/forward.py index be4a957ea..9caa12f7d 100644 --- a/modules/sdnq/forward.py +++ b/modules/sdnq/forward.py @@ -5,7 +5,7 @@ import torch from modules import shared from .common import conv_types, conv_transpose_types -from .dequantizer import dequantize_symmetric +from .dequantizer import dequantize_symmetric, dequantize_symmetric_with_bias from .packed_int import unpack_int_symetric @@ -94,10 +94,10 @@ def fp8_matmul_tensorwise( output_shape[-1] = weight.shape[-1] dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) input, scale = quantize_fp8_matmul_input_tensorwise(input, scale) - result = dequantize_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, output_shape) if bias is not None: - result.add_(bias) - return result + return dequantize_symmetric_with_bias(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), bias, scale, return_dtype, output_shape) + else: + return dequantize_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, output_shape) def int8_matmul( @@ -114,10 +114,10 @@ def int8_matmul( output_shape = list(input.shape) output_shape[-1] = weight.shape[-1] input, scale = quantize_int8_matmul_input(input, scale) - result = dequantize_symmetric(torch._int_mm(input, weight), scale, return_dtype, output_shape) if bias is not None: - result.add_(bias) - return result + return dequantize_symmetric_with_bias(torch._int_mm(input, weight), bias, scale, return_dtype, output_shape) + else: + return dequantize_symmetric(torch._int_mm(input, weight), scale, return_dtype, output_shape) def process_conv_input(conv_type, input, reversed_padding_repeated_twice, padding_mode, result_shape, stride, padding, dilation): @@ -192,11 +192,14 @@ def conv_fp8_matmul( weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) result = [] - for i in range(groups): - result.append(torch._scaled_mm(input[i], weight[i], scale_a=input_scale[i], scale_b=scale[i], bias=None, out_dtype=return_dtype)) - result = torch.cat(result, dim=-1).reshape(mm_output_shape) if bias is not None: - result.add_(bias) + bias = bias.reshape(groups, bias.shape[0] // groups) + for i in range(groups): + result.append(torch._scaled_mm(input[i], weight[i], scale_a=input_scale[i], scale_b=scale[i], bias=bias[i], out_dtype=return_dtype)) + else: + for i in range(groups): + result.append(torch._scaled_mm(input[i], weight[i], scale_a=input_scale[i], scale_b=scale[i], bias=None, out_dtype=return_dtype)) + result = torch.cat(result, dim=-1).reshape(mm_output_shape) if conv_type == 1: result = result.transpose(1,2) @@ -224,16 +227,18 @@ def conv_fp8_matmul_tensorwise( dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) if groups == 1: - result = dequantize_symmetric(torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype), scale, return_dtype, mm_output_shape) + result = torch._scaled_mm(input, weight, scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype) else: weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) result = [] for i in range(groups): result.append(torch._scaled_mm(input[i], weight[i], scale_a=dummy_input_scale, scale_b=dummy_input_scale, bias=None, out_dtype=scale.dtype)) - result = dequantize_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) + result = torch.cat(result, dim=-1) if bias is not None: - result.add_(bias) + dequantize_symmetric_with_bias(result, bias, scale, return_dtype, mm_output_shape) + else: + dequantize_symmetric(result, scale, return_dtype, mm_output_shape) if conv_type == 1: result = result.transpose(1,2) @@ -264,16 +269,18 @@ def conv_int8_matmul( weight = unpack_int_symetric(weight, quantized_weight_shape, weights_dtype, dtype=torch.int8, transpose=True) if groups == 1: - result = dequantize_symmetric(torch._int_mm(input, weight), scale, return_dtype, mm_output_shape) + result = torch._int_mm(input, weight) else: weight = weight.reshape(weight.shape[0], groups, weight.shape[1] // groups).transpose(0,1) input = input.reshape(input.shape[0], groups, input.shape[1] // groups).transpose(0,1) result = [] for i in range(groups): result.append(torch._int_mm(input[i], weight[i])) - result = dequantize_symmetric(torch.cat(result, dim=-1), scale, return_dtype, mm_output_shape) + result = torch.cat(result, dim=-1) if bias is not None: - result.add_(bias) + result = dequantize_symmetric_with_bias(result, bias, scale, return_dtype, mm_output_shape) + else: + result = dequantize_symmetric(result, scale, return_dtype, mm_output_shape) if conv_type == 1: result = result.transpose(1,2) From 223a01dc71bc3309852385e2827846d5ac09d0eb Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 15 Jun 2025 03:22:54 +0300 Subject: [PATCH 62/63] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3ec2c882..98bb764c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2025-06-10 +## Update for 2025-06-15 - **Feature** - Support Python 3.13 @@ -30,6 +30,7 @@ - Fix forced FP32 with tensorwise FP8 matmul - Fix PyTorch <= 2.4 compatibility with FP8 matmul - Fix VAE with conv quant + - Don't ignore the Quantize with GPU option with offload mode `none` and `model` - **Fixes** - Meissonic with multiple generators From c307906813d6c49983465d0ca254b49f370777df Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 15 Jun 2025 12:40:24 +0300 Subject: [PATCH 63/63] Update CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98bb764c7..c2f459f35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Update for 2025-06-15 - **Feature** - - Support Python 3.13 + - Support for Python 3.13 - **Changes** - Increase the medvram mode threshold from 8GB to 12GB @@ -17,7 +17,7 @@ - Force upgrade pip when installing Torch - **ROCm** - - Support ROCm 6.4 with `---use-nightly` + - Support ROCm 6.4 with `--use-nightly` - Don't override user set gfx version - Don't override gfx version with RX 9000 - Fix flash-atten repo