diff --git a/CHANGELOG.md b/CHANGELOG.md index 46cf6b8dc..dfae0d239 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ Also unlike most traditional methods, its also applicable to nearly all model ty - `INT4_SYM` -> `int4` - `INT4` -> `uint4` - Add `float8_e4m3fn`, `float8_e5m2`, `float8_e4m3fnuz`, `float8_e5m2fnuz`, `int6`, `uint6`, `int2`, `uint2` and `uint1` support - - Add quantized matmul support for `float8_e4m3fn` + - Add quantized matmul support for `float8_e4m3fn` and `float8_e5m2` - Set the default quant mode to `pre` - Use per token input quant with int8 and fp8 quantized matmul - Implement better layer hijacks diff --git a/modules/model_quant_sdnq.py b/modules/model_quant_sdnq.py index 8dc4e0ed8..4b5288958 100644 --- a/modules/model_quant_sdnq.py +++ b/modules/model_quant_sdnq.py @@ -28,9 +28,9 @@ dtype_dict = { "float8_e5m2fnuz": {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": CustomDtype.FP8, "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}, } -quantized_matmul_dtypes = ("int8", "int6", "int4", "int2", "float8_e4m3fn") +quantized_matmul_dtypes = ("int8", "int6", "int4", "int2", "float8_e4m3fn", "float8_e5m2") if devices.backend in {"cpu", "openvino"}: - quantized_matmul_dtypes += ("float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz") + quantized_matmul_dtypes += ("float8_e4m3fnuz", "float8_e5m2fnuz") linear_types = ("Linear",) conv_types = ("Conv1d", "Conv2d", "Conv3d")