From f12caf81f96868aaa00e9d4344375235094dd6e1 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 17 Oct 2025 17:25:50 +0300 Subject: [PATCH] SDNQ skip bad layers on svd and fix svd with dequantize_fp32 --- modules/processing_callbacks.py | 4 ++-- modules/sdnq/layers/conv/conv_fp8_tensorwise.py | 2 +- modules/sdnq/layers/conv/conv_int8.py | 2 +- .../sdnq/layers/linear/linear_fp8_tensorwise.py | 2 +- modules/sdnq/layers/linear/linear_int8.py | 2 +- modules/sdnq/quantizer.py | 15 +++++++++------ 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 62f9b624b..3aec83fca 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -53,9 +53,9 @@ def diffusers_callback_legacy(step: int, timestep: int, latents: typing.Union[to def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {}): t0 = time.time() - if devices.backend == "ipex": # xe driver on linux needs this + if devices.backend == "ipex": torch.xpu.synchronize(devices.device) - elif (devices.backend == "zluda") or (devices.backend == "rocm") or (devices.backend == "cuda"): + elif devices.backend in {"cuda", "zluda", "rocm"}: torch.cuda.synchronize(devices.device) latents = kwargs.get('latents', None) if debug: diff --git a/modules/sdnq/layers/conv/conv_fp8_tensorwise.py b/modules/sdnq/layers/conv/conv_fp8_tensorwise.py index 2dc9fbda3..9010445a6 100644 --- a/modules/sdnq/layers/conv/conv_fp8_tensorwise.py +++ b/modules/sdnq/layers/conv/conv_fp8_tensorwise.py @@ -29,7 +29,7 @@ def conv_fp8_matmul_tensorwise( if svd_up is not None: input = input.flatten(0,-2) if bias is not None: - bias = torch.addmm(bias, torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) + bias = torch.addmm(bias.to(dtype=svd_down.dtype), torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) else: bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) diff --git a/modules/sdnq/layers/conv/conv_int8.py b/modules/sdnq/layers/conv/conv_int8.py index 3332537d1..a1e297cc1 100644 --- a/modules/sdnq/layers/conv/conv_int8.py +++ b/modules/sdnq/layers/conv/conv_int8.py @@ -32,7 +32,7 @@ def conv_int8_matmul( if svd_up is not None: input = input.flatten(0,-2) if bias is not None: - bias = torch.addmm(bias, torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) + bias = torch.addmm(bias.to(dtype=svd_down.dtype), torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) else: bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) diff --git a/modules/sdnq/layers/linear/linear_fp8_tensorwise.py b/modules/sdnq/layers/linear/linear_fp8_tensorwise.py index c7978ef0a..d58b6fbb1 100644 --- a/modules/sdnq/layers/linear/linear_fp8_tensorwise.py +++ b/modules/sdnq/layers/linear/linear_fp8_tensorwise.py @@ -31,7 +31,7 @@ def fp8_matmul_tensorwise( if svd_up is not None: input.flatten(0,-2) if bias is not None: - bias = torch.addmm(bias, torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) + bias = torch.addmm(bias.to(dtype=svd_down.dtype), torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) else: bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32) diff --git a/modules/sdnq/layers/linear/linear_int8.py b/modules/sdnq/layers/linear/linear_int8.py index 6d7f6f2b8..7fa60a87f 100644 --- a/modules/sdnq/layers/linear/linear_int8.py +++ b/modules/sdnq/layers/linear/linear_int8.py @@ -36,7 +36,7 @@ def int8_matmul( if svd_up is not None: input = input.flatten(0,-2) if bias is not None: - bias = torch.addmm(bias, torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) + bias = torch.addmm(bias.to(dtype=svd_down.dtype), torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) else: bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up) input, scale = quantize_int8_matmul_input(input, scale) diff --git a/modules/sdnq/quantizer.py b/modules/sdnq/quantizer.py index 29f3b61d8..3a39122db 100644 --- a/modules/sdnq/quantizer.py +++ b/modules/sdnq/quantizer.py @@ -199,10 +199,13 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz layer.weight.data = layer.weight.to(dtype=torch.float32) if use_svd: - layer.weight.data, svd_up, svd_down = apply_svdquant(layer.weight, rank=svd_rank, niter=svd_steps) - if use_quantized_matmul: - svd_up = svd_up.t_() - svd_down = svd_down.t_() + try: + layer.weight.data, svd_up, svd_down = apply_svdquant(layer.weight, rank=svd_rank, niter=svd_steps) + if use_quantized_matmul: + svd_up = svd_up.t_() + svd_down = svd_down.t_() + except Exception: + svd_up, svd_down = None, None else: svd_up, svd_down = None, None @@ -210,9 +213,9 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz if use_quantized_matmul and dtype_dict[weights_dtype]["num_bits"] >= 6: group_size = -1 elif is_linear_type: - group_size = 2 ** ((2 if not use_svd else 3) + dtype_dict[weights_dtype]["num_bits"]) + group_size = 2 ** ((2 if svd_up is None else 3) + dtype_dict[weights_dtype]["num_bits"]) else: - group_size = 2 ** ((1 if not use_svd else 2) + dtype_dict[weights_dtype]["num_bits"]) + group_size = 2 ** ((1 if svd_up is None else 2) + dtype_dict[weights_dtype]["num_bits"]) elif use_quantized_matmul and dtype_dict[weights_dtype]["num_bits"] == 8: group_size = -1 # override user value, re-quantizing 8bit into 8bit is pointless elif group_size != -1 and not is_linear_type: