From 3f3a986c0e70b997ab31a355891b268fad1d145c Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 2 Jun 2025 23:18:10 +0300 Subject: [PATCH 1/6] SDNQ fix scale staying in fp32 with tensorwise fp8 matmul --- modules/model_quant_sdnq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 8231a3828..b923df209 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -128,7 +128,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz zero_point = None layer.weight.data = quantize_weight(layer.weight, scale, zero_point, weights_dtype) - if not shared.opts.sdnq_decompress_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]): + if not shared.opts.sdnq_decompress_fp32 and not (use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"] and not use_tensorwise_fp8_matmul): scale = scale.to(torch_dtype) if zero_point is not None: zero_point = zero_point.to(torch_dtype) From 6f637f41cc777b4eb8b65cbe6b25b2651ec90545 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 2 Jun 2025 23:28:49 +0300 Subject: [PATCH 2/6] Use tensorwise fp8 matmul with torch < 2.5 --- modules/model_quant_sdnq.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index b923df209..828fc5509 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -11,6 +11,7 @@ from diffusers.utils import get_module_from_name from accelerate.utils import CustomDtype from modules import devices, shared +torch_version = float(torch.__version__[:3]) dtype_dict = { "int8": {"min": -128, "max": 127, "num_bits": 8, "target_dtype": torch.int8, "torch_dtype": torch.int8, "storage_dtype": torch.int8, "is_unsigned": False, "is_integer": True}, @@ -73,7 +74,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz use_quantized_matmul = weights_dtype in quantized_matmul_dtypes and channel_size >= 32 and output_channel_size >= 32 if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: use_quantized_matmul = output_channel_size % 16 == 0 and channel_size % 16 == 0 - use_tensorwise_fp8_matmul = devices.backend in {"cpu", "openvino"} or (devices.backend == "cuda" and sys.platform == "win32" and float(torch.__version__[:3]) <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) + use_tensorwise_fp8_matmul = torch_version < 2.5 or devices.backend in {"cpu", "openvino"} or (devices.backend == "cuda" and sys.platform == "win32" and torch_version <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) if not use_quantized_matmul and (group_size > 0 or (dtype_dict[weights_dtype]["num_bits"] < 6 and group_size != -1)): if group_size == 0: From e9ff242e03de6daf72a17c0b070b656e91ec5f62 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 3 Jun 2025 02:16:52 +0300 Subject: [PATCH 3/6] SDNQ add group size support for convs --- modules/model_quant_sdnq.py | 75 ++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 26 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 828fc5509..49fdeb6bb 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -57,15 +57,18 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if layer_class_name in conv_types: if not quant_conv: return layer - reduction_axes = [i for i in range(layer.weight.ndim) if i != 0] + reduction_axes = list(range(layer.weight.ndim))[1:] use_quantized_matmul = False is_conv_type = True + output_channel_size, channel_size = layer.weight.shape[:2] elif layer_class_name in conv_transpose_types: if not quant_conv: return layer - reduction_axes = [i for i in range(layer.weight.ndim) if i != 1] + reduction_axes = list(range(layer.weight.ndim)) + reduction_axes.pop(1) use_quantized_matmul = False is_conv_transpose_type = True + channel_size, output_channel_size = layer.weight.shape[:2] else: is_linear_type = True reduction_axes = -1 @@ -76,33 +79,53 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz use_quantized_matmul = output_channel_size % 16 == 0 and channel_size % 16 == 0 use_tensorwise_fp8_matmul = torch_version < 2.5 or devices.backend in {"cpu", "openvino"} or (devices.backend == "cuda" and sys.platform == "win32" and torch_version <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) - if not use_quantized_matmul and (group_size > 0 or (dtype_dict[weights_dtype]["num_bits"] < 6 and group_size != -1)): - if group_size == 0: - if dtype_dict[weights_dtype]["num_bits"] < 4: - group_size = 32 - else: - group_size = 64 - num_of_groups = channel_size // group_size + if group_size == 0: + if is_linear_type: + if dtype_dict[weights_dtype]["num_bits"] < 6: + group_size = 2 ** (2 + dtype_dict[weights_dtype]["num_bits"]) + else: + group_size = 2 ** dtype_dict[weights_dtype]["num_bits"] - if group_size >= channel_size: - group_size = channel_size - num_of_groups = 1 - else: - num_of_groups = channel_size // group_size - while channel_size % group_size != 0: # find something divisible - num_of_groups -= 1 - if num_of_groups <= 1: - group_size = channel_size - num_of_groups = 1 - break - group_size = channel_size / num_of_groups + if not use_quantized_matmul and group_size > 0: + if group_size >= channel_size: + group_size = channel_size + num_of_groups = 1 + else: + num_of_groups = channel_size // group_size + while channel_size % group_size != 0: # find something divisible + num_of_groups -= 1 + if num_of_groups <= 1: + group_size = channel_size + num_of_groups = 1 + break + group_size = channel_size / num_of_groups + group_size = int(group_size) + num_of_groups = int(num_of_groups) - if num_of_groups > 1: - result_shape = layer.weight.shape - new_shape = list(result_shape) + if num_of_groups > 1: + result_shape = layer.weight.shape + new_shape = list(result_shape) + if is_conv_type: + # output_channel_size, channel_size, X, X + # output_channel_size, num_of_groups, group_size, X, X + new_shape[1] = group_size + new_shape.insert(1, num_of_groups) + reduction_axes.pop(0) + reduction_axes.append(layer.weight.ndim) + elif is_conv_transpose_type: + #channel_size, output_channel_size, X, X + #num_of_groups, group_size, output_channel_size, X, X + new_shape[0] = group_size + new_shape.insert(0, num_of_groups) + reduction_axes = list(range(layer.weight.ndim + 1)) + reduction_axes.pop(2) + reduction_axes.pop(0) + elif is_linear_type: + # output_channel_size, channel_size + # output_channel_size, num_of_groups, group_size last_dim_index = layer.weight.ndim - new_shape[last_dim_index - 1 : last_dim_index] = (int(num_of_groups), int(group_size)) - layer.weight.data = layer.weight.reshape(new_shape) + new_shape[last_dim_index - 1 : last_dim_index] = (num_of_groups, group_size) + layer.weight.data = layer.weight.reshape(new_shape) layer.weight.requires_grad = False if shared.opts.diffusers_offload_mode in {"none", "model"}: From 79ec23dd9d0ee1fdd243766a03f441904556dd3a Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 3 Jun 2025 02:37:29 +0300 Subject: [PATCH 4/6] SDNQ override conv dtype --- modules/model_quant_sdnq.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 49fdeb6bb..33d68bd12 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -61,6 +61,10 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz use_quantized_matmul = False is_conv_type = True output_channel_size, channel_size = layer.weight.shape[:2] + if dtype_dict[weights_dtype]["num_bits"] < 4: + weights_dtype = "uint4" + elif dtype_dict[weights_dtype]["num_bits"] < 6: + weights_dtype = "int6" elif layer_class_name in conv_transpose_types: if not quant_conv: return layer @@ -69,6 +73,10 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz use_quantized_matmul = False is_conv_transpose_type = True channel_size, output_channel_size = layer.weight.shape[:2] + if dtype_dict[weights_dtype]["num_bits"] < 4: + weights_dtype = "uint4" + elif dtype_dict[weights_dtype]["num_bits"] < 6: + weights_dtype = "int6" else: is_linear_type = True reduction_axes = -1 From ce3a6e27fa32c638f2b3a2b780513b473c596b69 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 3 Jun 2025 12:30:19 +0300 Subject: [PATCH 5/6] SDNQ use channelwise only quant --- modules/model_quant_sdnq.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 33d68bd12..1281b05e4 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -57,26 +57,21 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if layer_class_name in conv_types: if not quant_conv: return layer - reduction_axes = list(range(layer.weight.ndim))[1:] - use_quantized_matmul = False is_conv_type = True + reduction_axes = 1 output_channel_size, channel_size = layer.weight.shape[:2] + use_quantized_matmul = False if dtype_dict[weights_dtype]["num_bits"] < 4: weights_dtype = "uint4" - elif dtype_dict[weights_dtype]["num_bits"] < 6: - weights_dtype = "int6" elif layer_class_name in conv_transpose_types: if not quant_conv: return layer - reduction_axes = list(range(layer.weight.ndim)) - reduction_axes.pop(1) - use_quantized_matmul = False is_conv_transpose_type = True + reduction_axes = 0 channel_size, output_channel_size = layer.weight.shape[:2] + use_quantized_matmul = False if dtype_dict[weights_dtype]["num_bits"] < 4: weights_dtype = "uint4" - elif dtype_dict[weights_dtype]["num_bits"] < 6: - weights_dtype = "int6" else: is_linear_type = True reduction_axes = -1 @@ -92,7 +87,8 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if dtype_dict[weights_dtype]["num_bits"] < 6: group_size = 2 ** (2 + dtype_dict[weights_dtype]["num_bits"]) else: - group_size = 2 ** dtype_dict[weights_dtype]["num_bits"] + if dtype_dict[weights_dtype]["num_bits"] < 8: + group_size = 2 ** (1 + dtype_dict[weights_dtype]["num_bits"]) if not use_quantized_matmul and group_size > 0: if group_size >= channel_size: @@ -118,16 +114,13 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz # output_channel_size, num_of_groups, group_size, X, X new_shape[1] = group_size new_shape.insert(1, num_of_groups) - reduction_axes.pop(0) - reduction_axes.append(layer.weight.ndim) + reduction_axes = 2 elif is_conv_transpose_type: #channel_size, output_channel_size, X, X #num_of_groups, group_size, output_channel_size, X, X new_shape[0] = group_size new_shape.insert(0, num_of_groups) - reduction_axes = list(range(layer.weight.ndim + 1)) - reduction_axes.pop(2) - reduction_axes.pop(0) + reduction_axes = 1 elif is_linear_type: # output_channel_size, channel_size # output_channel_size, num_of_groups, group_size From 0777f67573ccc65254927c1f9f85056e73752db7 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 3 Jun 2025 18:12:12 +0300 Subject: [PATCH 6/6] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5816d5f9b..968bb8e1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Take a look at [Docs](https://github.com/vladmandic/sdnext/wiki/Docs), [Hints](h - `INT4` -> `uint4` - Add `float8_e4m3fn`, `float8_e5m2`, `float8_e4m3fnuz`, `float8_e5m2fnuz`, `int6`, `uint6`, `int2`, `uint2` and `uint1` support - Add quantized matmul support for `float8_e4m3fn` and `float8_e5m2` + - Add group size support for convolutional layers - Set the default quant mode to `pre` - Use per token input quant with int8 and fp8 quantized matmul - Implement better layer hijacks