Merge branch 'dev' of https://github.com/vladmandic/sdnext into dev

modules/sdnq/dequantizer.py

@@ -84,8 +84,7 @@ def quantize_int_mm(input: torch.FloatTensor, dim: int = -1, matmul_dtype: str =
 @devices.inference_context()
 def quantize_int_mm_sr(input: torch.FloatTensor, dim: int = -1, matmul_dtype: str = "int8") -> Tuple[torch.Tensor, torch.FloatTensor]:
     scale = torch.amax(input.abs(), dim=dim, keepdims=True).div_(dtype_dict[matmul_dtype]["max"])
-    input = torch.normal(0, 0.1, input.shape, device=input.device, dtype=input.dtype
-    ).addcdiv_(input, scale).round_().clamp_(dtype_dict[matmul_dtype]["min"], dtype_dict[matmul_dtype]["max"]).to(dtype=dtype_dict[matmul_dtype]["torch_dtype"])
+    input = torch.div(input, scale).add_(torch.randn_like(input), alpha=0.1).round_().clamp_(dtype_dict[matmul_dtype]["min"], dtype_dict[matmul_dtype]["max"]).to(dtype=dtype_dict[matmul_dtype]["torch_dtype"])
     return input, scale
 
 

modules/sdnq/layers/conv/conv_fp16.py

@@ -35,6 +35,7 @@ def conv_fp16_matmul(
             bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
 
     input, scale = quantize_fp_mm_input_tensorwise(input, scale, matmul_dtype="float16")
+    weight = weight.to(dtype=torch.float16) # fp8 weights
     input, weight = check_mats(input, weight)
 
     if groups == 1:

modules/sdnq/layers/linear/linear_fp16.py

@@ -26,6 +26,7 @@ def fp16_matmul(
         else:
             bias = torch.mm(torch.mm(input.to(dtype=svd_down.dtype), svd_down), svd_up)
     input, scale = quantize_fp_mm_input_tensorwise(input, scale, matmul_dtype="float16")
+    weight = weight.to(dtype=torch.float16) # fp8 weights
     input, weight = check_mats(input, weight)
     if bias is not None:
         return dequantize_symmetric_with_bias(fp_mm_func(input, weight), scale, bias, dtype=return_dtype, result_shape=output_shape)

modules/sdnq/quantizer.py

@@ -46,23 +46,15 @@ def quantize_weight(weight: torch.FloatTensor, reduction_axes: Union[int, List[i
 
     if dtype_dict[weights_dtype]["is_unsigned"]:
         scale, zero_point = get_scale_asymmetric(weight, reduction_axes, weights_dtype)
-        quantized_weight = torch.sub(weight, zero_point)
-        scale_inplace = True
+        quantized_weight = torch.sub(weight, zero_point).div_(scale)
     else:
         scale = get_scale_symmetric(weight, reduction_axes, weights_dtype)
-        quantized_weight = weight
-        scale_inplace = False
+        quantized_weight = torch.div(weight, scale)
         zero_point = None
 
-    is_integer = dtype_dict[weights_dtype]["is_integer"]
-    if use_stochastic_rounding and is_integer: # this case can be fused with addcdiv_
-        quantized_weight = torch.normal(0, 0.1, weight.shape, device=weight.device, dtype=weight.dtype).addcdiv_(quantized_weight, scale)
-    elif scale_inplace:
-        quantized_weight.div_(scale)
-    else:
-        quantized_weight = torch.div(quantized_weight, scale)
-
-    if is_integer:
+    if dtype_dict[weights_dtype]["is_integer"]:
+        if use_stochastic_rounding:
+            quantized_weight.add_(torch.randn_like(quantized_weight), alpha=0.1)
         quantized_weight.round_()
     else:
         if use_stochastic_rounding:
@@ -335,7 +327,7 @@ def sdnq_quantize_layer_weight(weight, layer_class_name=None, weights_dtype="int
         else:
             group_size = -1
 
-    weight, scale, zero_point = quantize_weight(weight, reduction_axes, weights_dtype)
+    weight, scale, zero_point = quantize_weight(weight, reduction_axes, weights_dtype, use_stochastic_rounding=use_stochastic_rounding)
     if (
         not dequantize_fp32
         and dtype_dict[weights_dtype]["num_bits"] <= 8