diff --git a/CHANGELOG.md b/CHANGELOG.md index cf6acaff2..fb797f9e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,9 @@ ## Update for 2026-06-20 - **Features** - - **SDNQ** support for NPU during quantization and inference + - **SDNQ** support for NPU during quantization and inference + - add option: force dtype on load + use to force model components to override loading with desired dtype regardless of component config - **Fixes** - `sdnq` warn instead of error for triton - `insightface` missing dependencies diff --git a/data/reference-quant.json b/data/reference-quant.json index a12cc7f95..596959ee8 100644 --- a/data/reference-quant.json +++ b/data/reference-quant.json @@ -129,16 +129,6 @@ "size": 17.27, "date": "2025 December" }, - "Qwen-Image-2512 sdnq-dynamic-uint4": { - "path": "Disty0/Qwen-Image-2512-SDNQ-4bit-dynamic", - "preview": "Disty0--Qwen-Image-2512-SDNQ-uint4-svd-r32.jpg", - "desc": "Quantization of Qwen/Qwen-Image-2512 using SDNQ: sdnq-dynamic 4-bit uint", - "skip": true, - "tags": "quantized", - "extras": "", - "size": 18.51, - "date": "2026 January" - }, "Qwen-Image-Edit sdnq-svd-uint4": { "path": "Disty0/Qwen-Image-Edit-SDNQ-uint4-svd-r32", "preview": "Qwen--Qwen-Image-Edit.jpg", diff --git a/modules/devices.py b/modules/devices.py index 438c12028..43dc25888 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -377,7 +377,7 @@ def test_bf16(): else: from modules.zluda_installer import default_agent agent = default_agent - if agent is not None and agent.gfx_version < 0x1100 and agent.arch != rocm.MicroArchitecture.CDNA: # all cards before RDNA 3 except for CDNA cards + if (agent is not None) and (agent.gfx_version < 0x1100) and (agent.arch != rocm.MicroArchitecture.CDNA): # all cards before RDNA 3 except for CDNA cards bf16_ok = False return bf16_ok try: diff --git a/modules/sd_hijack_te.py b/modules/sd_hijack_te.py index c1163eabb..172b3f863 100644 --- a/modules/sd_hijack_te.py +++ b/modules/sd_hijack_te.py @@ -40,6 +40,8 @@ def hijack_encode_prompt(*args, **kwargs): timer.process.add('te', t1-t0) shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) shared.state.end(jobid) + from modules import memstats + log.debug(f'Encode: memory={memstats.memory_stats()}') return res diff --git a/modules/ui_definitions.py b/modules/ui_definitions.py index 679bcb4fc..27f29970a 100644 --- a/modules/ui_definitions.py +++ b/modules/ui_definitions.py @@ -220,6 +220,7 @@ def create_settings(cmd_opts): "math_sep": OptionInfo("

Execution Precision

", "", gr.HTML), "precision": OptionInfo("Autocast", "Precision type", gr.Radio, {"choices": ["Autocast", "Full"], "visible": False}), "cuda_dtype": OptionInfo("Auto", "Device precision type", gr.Radio, {"choices": ["Auto", "FP32", "FP16", "BF16"]}), + "force_dtype": OptionInfo(False, "Force dtype on load", None, None, None), "no_half": OptionInfo(False, "Force full precision (--no-half)", None, None, None), "upcast_sampling": OptionInfo(False if sys.platform != "darwin" else True, "Upcast sampling", gr.Checkbox, {"visible": False}), diff --git a/pipelines/generic_text_encoder.py b/pipelines/generic_text_encoder.py index 2bb94e58d..8da8039d5 100644 --- a/pipelines/generic_text_encoder.py +++ b/pipelines/generic_text_encoder.py @@ -1,5 +1,6 @@ import os import json +import torch import transformers from modules import shared, devices, errors, sd_models, sd_offload, model_quant from modules.logger import log @@ -147,4 +148,14 @@ def load_text_encoder(repo_id, cls_name, load_config=None, subfolder="text_encod module_memory = sd_offload.get_module_memory(text_encoder) log.debug(f'Load model: text_encoder="{repo_id}" quant="{quant_type}" size={module_size:.3f} params={param_num:.3f} memory={module_memory}') + try: + actual_dtype = text_encoder.dtype + if isinstance(actual_dtype, torch.dtype) and isinstance(dtype, torch.dtype) and actual_dtype != dtype: + force = shared.opts.force_dtype + log.warning(f'Load model: text_encoder="{repo_id}" dtype desired={dtype} actual={actual_dtype} force={force}') + if force: + text_encoder = text_encoder.to(dtype) + except Exception: + pass + return text_encoder diff --git a/pipelines/generic_transformer.py b/pipelines/generic_transformer.py index f7ee42c8d..8cc177cb8 100644 --- a/pipelines/generic_transformer.py +++ b/pipelines/generic_transformer.py @@ -1,4 +1,5 @@ import os +import torch from modules import shared, devices, errors, sd_models, sd_offload, model_quant from modules.logger import log from pipelines.generic_util import get_loader @@ -133,4 +134,14 @@ def load_transformer(repo_id, cls_name, load_config=None, subfolder="transformer module_memory = sd_offload.get_module_memory(transformer) log.debug(f'Load model: transformer="{repo_id}" quant="{quant_type}" size={module_size:.3f} params={param_num:.3f} memory={module_memory}') + try: + actual_dtype = transformer.dtype + if isinstance(actual_dtype, torch.dtype) and isinstance(dtype, torch.dtype) and actual_dtype != dtype: + force = shared.opts.force_dtype + log.warning(f'Load model: transformer="{repo_id}" dtype desired={dtype} actual={actual_dtype} force={force}') + if force: + transformer = transformer.to(dtype) + except Exception: + pass + return transformer