Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 17:24:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

SDNQ autodetect fp8 tw fallback and disable dynamic compile

Browse Source
This commit is contained in:
Disty0
2025-10-02 19:40:07 +03:00
parent a0b375ae3d
commit 34c2a624aa
1 changed files with 8 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/sdnq/common.py
+8 -3
View File
@@ -4,7 +4,7 @@ import os
from functools import partial
import torch
from modules import shared
from modules import shared, devices
torch_version = float(torch.__version__[:3])
@@ -34,7 +34,12 @@ if hasattr(torch, "float8_e5m2fnuz"):
dtype_dict["float8_e5m2fnuz"] = {"min": -57344, "max": 57344, "num_bits": 8, "target_dtype": "fp8", "torch_dtype": torch.float8_e5m2fnuz, "storage_dtype": torch.float8_e5m2fnuz, "is_unsigned": False, "is_integer": False}
use_torch_compile = shared.opts.sdnq_dequantize_compile # this setting requires a full restart of the webui to apply
use_tensorwise_fp8_matmul = os.environ.get("SDNQ_USE_TENSORWISE_FP8_MATMUL", "1").lower() not in {"0", "false", "no"} # row-wise FP8 only exist on H100 hardware, sdnq will use software row-wise with tensorwise hardware with this setting
if devices.backend == "cuda" and os.environ.get("SDNQ_USE_TENSORWISE_FP8_MATMUL", None) is None:
# row-wise FP8 only exist on H100 hardware, sdnq will use software row-wise with tensorwise hardware with this setting
use_tensorwise_fp8_matmul = torch.cuda.get_device_capability(devices.device) < (9,0)
else:
use_tensorwise_fp8_matmul = os.environ.get("SDNQ_USE_TENSORWISE_FP8_MATMUL", "1").lower() not in {"0", "false", "no"}
linear_types = ("Linear",)
conv_types = ("Conv1d", "Conv2d", "Conv3d")
@@ -44,7 +49,7 @@ allowed_types = linear_types + conv_types + conv_transpose_types
if use_torch_compile:
torch._dynamo.config.cache_size_limit = max(8192, torch._dynamo.config.cache_size_limit)
torch._dynamo.config.accumulated_recompile_limit = max(8192, torch._dynamo.config.accumulated_recompile_limit)
compile_func = partial(torch.compile, fullgraph=True)
compile_func = partial(torch.compile, fullgraph=True, dynamic=False)
else:
def compile_func(fn, **kwargs): # pylint: disable=unused-argument
return fn