From 8f1a1d7311e4a0252cb2dd18d90c5c149a4000f9 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 2 Jun 2025 00:28:29 +0300 Subject: [PATCH] SDNQ expand quantized_matmul_dtypes for CPU --- modules/model_quant_sdnq.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index bded102a3..8dc4e0ed8 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -28,7 +28,9 @@ dtype_dict = { "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, } -quantized_matmul_dtypes = ("int8", "int6", "int4", "float8_e4m3fn") +quantized_matmul_dtypes = ("int8", "int6", "int4", "int2", "float8_e4m3fn") +if devices.backend in {"cpu", "openvino"}: + quantized_matmul_dtypes += ("float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz") linear_types = ("Linear",) conv_types = ("Conv1d", "Conv2d", "Conv3d") @@ -71,7 +73,7 @@ def sdnq_quantize_layer(layer, weights_dtype="int8", torch_dtype=None, group_siz use_quantized_matmul = weights_dtype in quantized_matmul_dtypes and channel_size >= 32 and output_channel_size >= 32 if use_quantized_matmul and not dtype_dict[weights_dtype]["is_integer"]: use_quantized_matmul = output_channel_size % 16 == 0 and channel_size % 16 == 0 - use_tensorwise_fp8_matmul = devices.backend == "cpu" or (devices.backend == "cuda" and sys.platform == "win32" and float(torch.__version__[:3]) <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) + use_tensorwise_fp8_matmul = devices.backend in {"cpu", "openvino"} or (devices.backend == "cuda" and sys.platform == "win32" and float(torch.__version__[:3]) <= 2.7 and torch.cuda.get_device_capability(devices.device) == (8,9)) if not use_quantized_matmul and (group_size > 0 or (dtype_dict[weights_dtype]["num_bits"] < 6 and group_size != -1)): if group_size == 0: