From 34c2a624aaf6bd443d3e937fed0c4403a620c752 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 2 Oct 2025 19:40:07 +0300 Subject: [PATCH] SDNQ autodetect fp8 tw fallback and disable dynamic compile --- modules/sdnq/common.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/sdnq/common.py b/modules/sdnq/common.py index 783a7ab2f..b70c7b06b 100644 --- a/modules/sdnq/common.py +++ b/modules/sdnq/common.py @@ -4,7 +4,7 @@ import os from functools import partial import torch -from modules import shared +from modules import shared, devices torch_version = float(torch.__version__[:3]) @@ -34,7 +34,12 @@ if hasattr(torch, "float8_e5m2fnuz"): dtype_dict["float8_e5m2fnuz"] = {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": "fp8", "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False} use_torch_compile = shared.opts.sdnq_dequantize_compile # this setting requires a full restart of the webui to apply -use_tensorwise_fp8_matmul = os.environ.get("SDNQ_USE_TENSORWISE_FP8_MATMUL", "1").lower() not in {"0", "false", "no"} # row-wise FP8 only exist on H100 hardware, sdnq will use software row-wise with tensorwise hardware with this setting + +if devices.backend == "cuda" and os.environ.get("SDNQ_USE_TENSORWISE_FP8_MATMUL", None) is None: + # row-wise FP8 only exist on H100 hardware, sdnq will use software row-wise with tensorwise hardware with this setting + use_tensorwise_fp8_matmul = torch.cuda.get_device_capability(devices.device) < (9,0) +else: + use_tensorwise_fp8_matmul = os.environ.get("SDNQ_USE_TENSORWISE_FP8_MATMUL", "1").lower() not in {"0", "false", "no"} linear_types = ("Linear",) conv_types = ("Conv1d", "Conv2d", "Conv3d") @@ -44,7 +49,7 @@ allowed_types = linear_types + conv_types + conv_transpose_types if use_torch_compile: torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit) torch._dynamo.config.accumulated_recompile_limit = max(8192, torch._dynamo.config.accumulated_recompile_limit) - compile_func = partial(torch.compile, fullgraph=True) + compile_func = partial(torch.compile, fullgraph=True, dynamic=False) else: def compile_func(fn, **kwargs): # pylint: disable=unused-argument return fn