SDNQ skip bad layers on svd and fix svd with dequantize_fp32

modules/processing_callbacks.py

@@ -53,9 +53,9 @@ def diffusers_callback_legacy(step: int, timestep: int, latents: typing.Union[to
 
 def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {}):
     t0 = time.time()
-    if devices.backend == "ipex": # xe driver on linux needs this
+    if devices.backend == "ipex":
         torch.xpu.synchronize(devices.device)
-    elif (devices.backend == "zluda") or (devices.backend == "rocm") or (devices.backend == "cuda"):
+    elif devices.backend in {"cuda", "zluda", "rocm"}:
         torch.cuda.synchronize(devices.device)
     latents = kwargs.get('latents', None)
     if debug:

modules/sdnq/layers/conv/conv_fp8_tensorwise.py

@@ -29,7 +29,7 @@ def conv_fp8_matmul_tensorwise(
     if svd_up is not None:
         input = input.flatten(0,-2)
         if bias is not None:
-            bias = torch.addmm(bias, torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
+            bias = torch.addmm(bias.to(dtype=svd_down.dtype), torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
         else:
             bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
 

modules/sdnq/layers/conv/conv_int8.py

@@ -32,7 +32,7 @@ def conv_int8_matmul(
     if svd_up is not None:
         input = input.flatten(0,-2)
         if bias is not None:
-            bias = torch.addmm(bias, torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
+            bias = torch.addmm(bias.to(dtype=svd_down.dtype), torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
         else:
             bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
 

modules/sdnq/layers/linear/linear_fp8_tensorwise.py

@@ -31,7 +31,7 @@ def fp8_matmul_tensorwise(
     if svd_up is not None:
         input.flatten(0,-2)
         if bias is not None:
-            bias = torch.addmm(bias, torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
+            bias = torch.addmm(bias.to(dtype=svd_down.dtype), torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
         else:
             bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
     dummy_input_scale = torch.ones(1, device=input.device, dtype=torch.float32)

modules/sdnq/layers/linear/linear_int8.py

@@ -36,7 +36,7 @@ def int8_matmul(
     if svd_up is not None:
         input = input.flatten(0,-2)
         if bias is not None:
-            bias = torch.addmm(bias, torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
+            bias = torch.addmm(bias.to(dtype=svd_down.dtype), torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
         else:
             bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
     input, scale = quantize_int8_matmul_input(input, scale)

modules/sdnq/quantizer.py

@@ -199,10 +199,13 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz
             layer.weight.data = layer.weight.to(dtype=torch.float32)
 
         if use_svd:
-            layer.weight.data, svd_up, svd_down = apply_svdquant(layer.weight, rank=svd_rank, niter=svd_steps)
-            if use_quantized_matmul:
-                svd_up = svd_up.t_()
-                svd_down = svd_down.t_()
+            try:
+                layer.weight.data, svd_up, svd_down = apply_svdquant(layer.weight, rank=svd_rank, niter=svd_steps)
+                if use_quantized_matmul:
+                    svd_up = svd_up.t_()
+                    svd_down = svd_down.t_()
+            except Exception:
+                svd_up, svd_down = None, None
         else:
             svd_up, svd_down = None, None
 
@@ -210,9 +213,9 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz
             if use_quantized_matmul and dtype_dict[weights_dtype]["num_bits"] >= 6:
                 group_size = -1
             elif is_linear_type:
-                group_size = 2 ** ((2 if not use_svd else 3) + dtype_dict[weights_dtype]["num_bits"])
+                group_size = 2 ** ((2 if svd_up is None else 3) + dtype_dict[weights_dtype]["num_bits"])
             else:
-                group_size = 2 ** ((1 if not use_svd else 2) + dtype_dict[weights_dtype]["num_bits"])
+                group_size = 2 ** ((1 if svd_up is None else 2) + dtype_dict[weights_dtype]["num_bits"])
         elif use_quantized_matmul and dtype_dict[weights_dtype]["num_bits"] == 8:
             group_size = -1 # override user value, re-quantizing 8bit into 8bit is pointless
         elif group_size != -1 and not is_linear_type: