mirror of
https://github.com/vladmandic/automatic
synced 2026-09-20 01:31:13 +02:00
Remove sdnq_use_quantized_matmul and use sdnq_quantize_matmul_mode instead
This commit is contained in:
Disty0
@@ -237,7 +237,6 @@ all_mm_backends = ["torch", "triton"]
|
||||
recommend_error_cap = 2.0 # a faster config is not recommended when it multiplies measured output error beyond this
|
||||
|
||||
atten_settings = [
|
||||
("sdnq_attention_use_quantized_matmul", "Use Quantized MatMul"),
|
||||
("sdnq_attention_matmul_type", "MatMul type"),
|
||||
("sdnq_attention_pv_matmul_type", "PV MatMul type"),
|
||||
("sdnq_attention_smooth_k", "Use Smooth K"),
|
||||
@@ -2356,7 +2355,7 @@ def bench_block_section(iters, warmup, config_timeout=300, selected=None):
|
||||
notes.append(f"four stacked blocks: error grows x{sum(growth) / len(growth):.2f} avg, residuals dampen compounding")
|
||||
weights_mode = str(getattr(shared.opts, "sdnq_quantize_weights_mode", ""))
|
||||
current_id = None
|
||||
if weights_mode == "int8" and getattr(shared.opts, "sdnq_use_quantized_matmul", False):
|
||||
if weights_mode == "int8" and getattr(shared.opts, "sdnq_quantize_matmul_mode", "disabled") != "disabled":
|
||||
current_id = "int8-mm-atten" if "SDNQ attention" in shared.opts.sdp_overrides else "int8-mm"
|
||||
if current_id and results.get(current_id, {}).get("ms"):
|
||||
notes.append(f"current config runs the {results[current_id]['label']} row for int8-quantized models")
|
||||
@@ -2418,12 +2417,12 @@ def build_recommendations(all_results, fp8_result, prep_status):
|
||||
quant_reason += f", error {int8_err:.5f} vs {noquant_err:.5f}; smooth k and hadamard below buy error back"
|
||||
else:
|
||||
quant_reason = f"unquantized sdnq measured x{int8_ms / noquant_ms:.2f} vs int8 qk with lower error; quantization prep outweighs the kernel gain on this gpu"
|
||||
rows.append(("Use Quantized MatMul", current("sdnq_attention_use_quantized_matmul"), str(use_quantized), quant_reason))
|
||||
rows.append(("Use Quantized MatMul", (current("sdnq_attention_matmul_type") != "disabled"), str(use_quantized), quant_reason))
|
||||
elif int8_ms and base_ms:
|
||||
use_quantized = base_ms / int8_ms >= 1.10
|
||||
rows.append(("Use Quantized MatMul", current("sdnq_attention_use_quantized_matmul"), str(use_quantized), f"int8 qk measured x{base_ms / int8_ms:.2f} vs torch sdpa; unquantized sdnq row unavailable"))
|
||||
rows.append(("Use Quantized MatMul", (current("sdnq_attention_matmul_type") != "disabled"), str(use_quantized), f"int8 qk measured x{base_ms / int8_ms:.2f} vs torch sdpa; unquantized sdnq row unavailable"))
|
||||
else:
|
||||
rows.append(("Use Quantized MatMul", current("sdnq_attention_use_quantized_matmul"), "False", "int8 qk failed to run"))
|
||||
rows.append(("Use Quantized MatMul", (current("sdnq_attention_matmul_type") != "disabled"), "False", "int8 qk failed to run"))
|
||||
|
||||
qk_choice = "auto"
|
||||
qk_reason = "resolves to int8; uint8 remaps to int8"
|
||||
@@ -2639,7 +2638,7 @@ def build_dequant_recommendations(dequant_results, weight_dequant_result, varian
|
||||
mm_reason += f"; measured with MatMul type {best_sel}" if mm_entry.get("mm_ms") else f"; needs MatMul type {best_sel}, auto (float8_e4m3fn) failed on this gpu"
|
||||
if weights_mode not in mode_ids:
|
||||
mm_reason += f"; current quantization type {weights_mode} was not benchmarked, judged on int8"
|
||||
rows.append(("Use quantized MatMul", current("sdnq_use_quantized_matmul"), str(recommend_mm), mm_reason))
|
||||
rows.append(("Use quantized MatMul", current("sdnq_quantize_matmul_mode"), str(recommend_mm), mm_reason))
|
||||
|
||||
type_parts = []
|
||||
for sel, resolved, ms, err in mm_candidates:
|
||||
@@ -2655,7 +2654,7 @@ def build_dequant_recommendations(dequant_results, weight_dequant_result, varian
|
||||
|
||||
sweeps = sweep_results or {}
|
||||
sweeps_at_reference = reference == dequant_shapes[0][0]
|
||||
mm_on = str(getattr(shared.opts, "sdnq_use_quantized_matmul", False)) == "True"
|
||||
mm_on = getattr(shared.opts, "sdnq_quantize_matmul_mode", "disabled") != "disabled"
|
||||
|
||||
# Group size: judged on the path the current config runs (mm cells when quantized matmul
|
||||
# is on); an explicit size must cut error meaningfully without real speed or size cost
|
||||
|
||||
@@ -38,7 +38,6 @@ def set_sdnq_attention():
|
||||
smooth_k=shared.opts.sdnq_attention_smooth_k,
|
||||
use_hadamard=shared.opts.sdnq_attention_use_hadamard,
|
||||
hadamard_group_size=shared.opts.sdnq_attention_hadamard_group_size,
|
||||
do_quantize=shared.opts.sdnq_attention_use_quantized_matmul,
|
||||
)
|
||||
else:
|
||||
if enable_gqa:
|
||||
|
||||
+13
-11
@@ -112,7 +112,10 @@ def create_sdnq_config(kwargs = None, allow: bool = True, module: str = 'Model',
|
||||
quantized_matmul_dtype = shared.opts.sdnq_quantize_matmul_mode_te
|
||||
else:
|
||||
quantized_matmul_dtype = shared.opts.sdnq_quantize_matmul_mode
|
||||
if quantized_matmul_dtype == "auto":
|
||||
|
||||
use_quantized_matmul = quantized_matmul_dtype != "disabled"
|
||||
quantized_matmul_dtype_log = quantized_matmul_dtype
|
||||
if quantized_matmul_dtype in {"enabled", "disabled"}:
|
||||
quantized_matmul_dtype = None
|
||||
|
||||
if modules_to_not_convert is None:
|
||||
@@ -154,7 +157,7 @@ def create_sdnq_config(kwargs = None, allow: bool = True, module: str = 'Model',
|
||||
use_hadamard=shared.opts.sdnq_use_hadamard,
|
||||
quant_conv=shared.opts.sdnq_quantize_conv_layers,
|
||||
quant_embedding=shared.opts.sdnq_quantize_embedding_layers,
|
||||
use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul,
|
||||
use_quantized_matmul=use_quantized_matmul,
|
||||
use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv,
|
||||
use_dynamic_quantization=shared.opts.sdnq_use_dynamic_quantization,
|
||||
dequantize_fp32=shared.opts.sdnq_dequantize_fp32,
|
||||
@@ -164,11 +167,9 @@ def create_sdnq_config(kwargs = None, allow: bool = True, module: str = 'Model',
|
||||
modules_to_not_convert=modules_to_not_convert,
|
||||
modules_dtype_dict=modules_dtype_dict.copy(),
|
||||
)
|
||||
if quantized_matmul_dtype is None:
|
||||
quantized_matmul_dtype = "auto" # set for logging
|
||||
svd = f'{shared.opts.sdnq_use_svd} rank={shared.opts.sdnq_svd_rank} steps={shared.opts.sdnq_svd_steps}' if shared.opts.sdnq_use_svd else f'{shared.opts.sdnq_use_svd}'
|
||||
hadamard = f'{shared.opts.sdnq_use_hadamard} group={shared.opts.sdnq_hadamard_group_size}' if shared.opts.sdnq_use_hadamard else f'{shared.opts.sdnq_use_hadamard}'
|
||||
log.debug(f'Quantization: module="{module}" type=sdnq mode=pre dtype={weights_dtype} svd={svd} hadamard={hadamard} dynamic={shared.opts.sdnq_use_dynamic_quantization} group={shared.opts.sdnq_group_size} loss={shared.opts.sdnq_dynamic_loss_threshold} matmul_dtype={quantized_matmul_dtype} matmul_quant={shared.opts.sdnq_use_quantized_matmul} matmul_conv={shared.opts.sdnq_use_quantized_matmul_conv} quant_conv={shared.opts.sdnq_quantize_conv_layers} quant_embed={shared.opts.sdnq_quantize_embedding_layers} fp32={shared.opts.sdnq_dequantize_fp32} device={quantization_device} return={return_device} gpu={shared.opts.sdnq_quantize_with_gpu} map={shared.opts.device_map}')
|
||||
log.debug(f'Quantization: module="{module}" type=sdnq mode=pre dtype={weights_dtype} svd={svd} hadamard={hadamard} dynamic={shared.opts.sdnq_use_dynamic_quantization} group={shared.opts.sdnq_group_size} loss={shared.opts.sdnq_dynamic_loss_threshold} matmul_dtype={quantized_matmul_dtype_log} matmul_quant={use_quantized_matmul} matmul_conv={shared.opts.sdnq_use_quantized_matmul_conv} quant_conv={shared.opts.sdnq_quantize_conv_layers} quant_embed={shared.opts.sdnq_quantize_embedding_layers} fp32={shared.opts.sdnq_dequantize_fp32} device={quantization_device} return={return_device} gpu={shared.opts.sdnq_quantize_with_gpu} map={shared.opts.device_map}')
|
||||
if len(modules_to_not_convert) > 0 or modules_dtype_dict:
|
||||
log.debug(f'Quantization: module={module} type=sdnq skip_modules={modules_to_not_convert} modules_dtype_dict={modules_dtype_dict}')
|
||||
if kwargs is None:
|
||||
@@ -371,9 +372,13 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc: bool = True, weigh
|
||||
quantized_matmul_dtype = shared.opts.sdnq_quantize_matmul_mode_te
|
||||
else:
|
||||
quantized_matmul_dtype = shared.opts.sdnq_quantize_matmul_mode
|
||||
if quantized_matmul_dtype == "auto":
|
||||
|
||||
use_quantized_matmul = quantized_matmul_dtype not in {"no", "disabled"}
|
||||
quantized_matmul_dtype_log = quantized_matmul_dtype
|
||||
if quantized_matmul_dtype in {"enabled", "disabled"}:
|
||||
quantized_matmul_dtype = None
|
||||
|
||||
|
||||
quantization_device, return_device = get_sdnq_devices(mode="post")
|
||||
|
||||
if modules_to_not_convert is None:
|
||||
@@ -416,7 +421,7 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc: bool = True, weigh
|
||||
use_hadamard=shared.opts.sdnq_use_hadamard,
|
||||
quant_conv=shared.opts.sdnq_quantize_conv_layers,
|
||||
quant_embedding=shared.opts.sdnq_quantize_embedding_layers,
|
||||
use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul,
|
||||
use_quantized_matmul=use_quantized_matmul,
|
||||
use_quantized_matmul_conv=shared.opts.sdnq_use_quantized_matmul_conv,
|
||||
use_dynamic_quantization=shared.opts.sdnq_use_dynamic_quantization,
|
||||
dequantize_fp32=shared.opts.sdnq_dequantize_fp32,
|
||||
@@ -457,10 +462,7 @@ def sdnq_quantize_model(model, op=None, sd_model=None, do_gc: bool = True, weigh
|
||||
model = model.to(devices.cpu)
|
||||
if do_gc:
|
||||
devices.torch_gc(force=True, reason='sdnq')
|
||||
|
||||
if quantized_matmul_dtype is None:
|
||||
quantized_matmul_dtype = "auto" # set for logging
|
||||
log.debug(f'Quantization: module="{op if op is not None else model.__class__}" type=sdnq mode=post dtype={weights_dtype} matmul_dtype={quantized_matmul_dtype} matmul={shared.opts.sdnq_use_quantized_matmul} svd={shared.opts.sdnq_use_svd} hadamard={shared.opts.sdnq_use_hadamard} dynamic={shared.opts.sdnq_use_dynamic_quantization}:group={shared.opts.sdnq_group_size}:hadamard_group={shared.opts.sdnq_hadamard_group_size}:rank={shared.opts.sdnq_svd_rank}:steps={shared.opts.sdnq_svd_steps}:loss={shared.opts.sdnq_dynamic_loss_threshold} matmul_conv={shared.opts.sdnq_use_quantized_matmul_conv} quant_conv={shared.opts.sdnq_quantize_conv_layers} quant_embedding={shared.opts.sdnq_quantize_embedding_layers} fp32={shared.opts.sdnq_dequantize_fp32} gpu={shared.opts.sdnq_quantize_with_gpu} device={quantization_device} return={return_device} map={shared.opts.device_map} non_blocking={shared.opts.diffusers_offload_nonblocking} modules_skip={modules_to_not_convert} modules_dtype={modules_dtype_dict}')
|
||||
log.debug(f'Quantization: module="{op if op is not None else model.__class__}" type=sdnq mode=post dtype={weights_dtype} matmul_dtype={quantized_matmul_dtype_log} matmul={use_quantized_matmul} svd={shared.opts.sdnq_use_svd} hadamard={shared.opts.sdnq_use_hadamard} dynamic={shared.opts.sdnq_use_dynamic_quantization}:group={shared.opts.sdnq_group_size}:hadamard_group={shared.opts.sdnq_hadamard_group_size}:rank={shared.opts.sdnq_svd_rank}:steps={shared.opts.sdnq_svd_steps}:loss={shared.opts.sdnq_dynamic_loss_threshold} matmul_conv={shared.opts.sdnq_use_quantized_matmul_conv} quant_conv={shared.opts.sdnq_quantize_conv_layers} quant_embedding={shared.opts.sdnq_quantize_embedding_layers} fp32={shared.opts.sdnq_dequantize_fp32} gpu={shared.opts.sdnq_quantize_with_gpu} device={quantization_device} return={return_device} map={shared.opts.device_map} non_blocking={shared.opts.diffusers_offload_nonblocking} modules_skip={modules_to_not_convert} modules_dtype={modules_dtype_dict}')
|
||||
return model
|
||||
|
||||
|
||||
|
||||
+10
-3
@@ -194,10 +194,17 @@ def set_diffuser_options(sd_model, vae=None, op:str='model', offload:bool=True,
|
||||
for module_name in get_module_names(sd_model):
|
||||
module = getattr(sd_model, module_name, None)
|
||||
if hasattr(module, "quantization_config") and getattr(module.quantization_config, "quant_method", None) == "sdnq":
|
||||
if module.quantization_config.use_quantized_matmul != shared.opts.sdnq_use_quantized_matmul:
|
||||
if module_name.startswith("text_encoder"):
|
||||
if shared.opts.sdnq_quantize_matmul_mode_te == "Same as model":
|
||||
sdnq_use_quantized_matmul = shared.opts.sdnq_quantize_matmul_mode != "disabled"
|
||||
else:
|
||||
sdnq_use_quantized_matmul = shared.opts.sdnq_quantize_matmul_mode_te != "disabled"
|
||||
else:
|
||||
sdnq_use_quantized_matmul = shared.opts.sdnq_quantize_matmul_mode != "disabled"
|
||||
if module.quantization_config.use_quantized_matmul != sdnq_use_quantized_matmul:
|
||||
from modules.sdnq.loader import apply_sdnq_options_to_model
|
||||
# log.debug(f'Setting {op} {module_name}: sdnq_use_quantized_matmul={shared.opts.sdnq_use_quantized_matmul}')
|
||||
module = apply_sdnq_options_to_model(module, use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul)
|
||||
# log.debug(f'Setting {op} {module_name}: sdnq_use_quantized_matmul={sdnq_use_quantized_matmul}')
|
||||
module = apply_sdnq_options_to_model(module, use_quantized_matmul=sdnq_use_quantized_matmul)
|
||||
setattr(sd_model, module_name, module)
|
||||
|
||||
if offload:
|
||||
|
||||
@@ -201,7 +201,7 @@ def quantize_attn(
|
||||
matmul_dtype: str = "int8",
|
||||
pv_matmul_dtype: str | None = None,
|
||||
) -> tuple[torch.Tensor]:
|
||||
if matmul_dtype in {"auto", "uint8"}:
|
||||
if matmul_dtype in {"auto", "enabled", "uint8"}:
|
||||
matmul_dtype = "int8"
|
||||
if pv_matmul_dtype == "uint8":
|
||||
pv_matmul_dtype = "int8"
|
||||
@@ -213,7 +213,7 @@ def quantize_attn(
|
||||
k = k.sub_(k.mean(dim=2, keepdim=True))
|
||||
else:
|
||||
k = k.sub(k.mean(dim=2, keepdim=True))
|
||||
if matmul_dtype not in {None, "none", "no"}:
|
||||
if matmul_dtype not in {None, "none", "no", "disabled"}:
|
||||
if hadamard is not None:
|
||||
q, use_hadamard, hadamard_group_size = apply_hadamard(q, group_size=hadamard_group_size, hadamard=hadamard, layer_class_name="Linear")
|
||||
if use_hadamard:
|
||||
@@ -228,7 +228,7 @@ def quantize_attn(
|
||||
k_q = k.contiguous().to(dtype=q.dtype)
|
||||
q_scale = None
|
||||
k_scale = None
|
||||
if pv_matmul_dtype not in {None, "auto", "none", "no"}:
|
||||
if pv_matmul_dtype not in {None, "auto", "none", "no", "disabled"}:
|
||||
quantize_mm_func_pv = quantize_int_mm if pv_matmul_dtype.startswith("int") else quantize_fp_mm
|
||||
v_q, v_scale = quantize_mm_func_pv(v.contiguous().to(dtype=torch.float32), dim=-1, matmul_dtype=pv_matmul_dtype)
|
||||
v_scale = v_scale.squeeze(-1)
|
||||
@@ -280,8 +280,8 @@ def get_attn_inputs(
|
||||
smooth_k=smooth_k,
|
||||
hadamard=hadamard,
|
||||
hadamard_group_size=hadamard_group_size,
|
||||
matmul_dtype=matmul_dtype if do_quantize else "no",
|
||||
pv_matmul_dtype=pv_matmul_dtype if do_quantize else "no",
|
||||
matmul_dtype=matmul_dtype if do_quantize else "disabled",
|
||||
pv_matmul_dtype=pv_matmul_dtype if do_quantize else "disabled",
|
||||
)
|
||||
return query, query_scale, key, key_scale, value, value_scale, attn_mask, scale, out_dtype
|
||||
|
||||
@@ -308,7 +308,7 @@ def sdnq_triton_atten(
|
||||
_, _, VN, VHD = value.shape
|
||||
|
||||
hadamard = None
|
||||
if use_hadamard and do_quantize and matmul_dtype not in {None, "none", "no"}:
|
||||
if use_hadamard and do_quantize and matmul_dtype not in {None, "none", "no", "disabled"}:
|
||||
hadamard_channel_size = next_power_of_2(min(QHD, KHD))
|
||||
hadamard_group_size = min(hadamard_group_size, hadamard_channel_size)
|
||||
use_hadamard, hadamard_group_size = get_hadamard_group_size(hadamard_channel_size, hadamard_group_size)
|
||||
|
||||
@@ -180,4 +180,4 @@ def get_repo(model):
|
||||
|
||||
|
||||
sdnq_quant_modes = ["int8", "uint8", "int6", "uint6", "uint5", "uint4", "uint3", "uint2", "float8_e4m3fn", "float8_e3m4fn", "float6_e3m2fn", "float5_e2m2fn", "float4_e2m1fn", "float3_e1m1fn", "float2_e1m0fn", "int16", "uint16", "float16"]
|
||||
sdnq_matmul_modes = ["auto", "int8", "uint8", "float8_e4m3fn", "float16"]
|
||||
sdnq_matmul_modes = ["disabled", "enabled", "int8", "uint8", "float8_e4m3fn", "float16"]
|
||||
|
||||
@@ -174,9 +174,9 @@ def create_settings(cmd_opts):
|
||||
"sdnq_quantize_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "TE", "LLM", "Control", "VAE"]}),
|
||||
"sdnq_quantize_mode": OptionInfo("auto", "Quantization mode", gr.Dropdown, {"choices": ["auto", "pre", "post"]}),
|
||||
"sdnq_quantize_weights_mode": OptionInfo("int8", "Quantization type", gr.Dropdown, {"choices": sdnq_quant_modes}),
|
||||
"sdnq_quantize_matmul_mode": OptionInfo("auto", "Quantized MatMul type", gr.Dropdown, {"choices": sdnq_matmul_modes}),
|
||||
"sdnq_quantize_matmul_mode": OptionInfo("disabled", "Quantized MatMul type", gr.Dropdown, {"choices": sdnq_matmul_modes}),
|
||||
"sdnq_quantize_weights_mode_te": OptionInfo("Same as model", "Quantization type for Text Encoders", gr.Dropdown, {"choices": ['Same as model'] + sdnq_quant_modes}),
|
||||
"sdnq_quantize_matmul_mode_te": OptionInfo("Same as model", "Quantized MatMul type for Text Encoders", gr.Dropdown, {"choices": ['Same as model'] + sdnq_matmul_modes}),
|
||||
"sdnq_quantize_matmul_mode_te": OptionInfo("disabled", "Quantized MatMul type for Text Encoders", gr.Dropdown, {"choices": ['Same as model'] + sdnq_matmul_modes}),
|
||||
"sdnq_modules_to_not_convert": OptionInfo("", "Modules to not convert"),
|
||||
"sdnq_modules_dtype_dict": OptionInfo("{}", "Modules dtype dict"),
|
||||
"sdnq_group_size": OptionInfo(0, "Group size", gr.Slider, {"minimum": -1, "maximum": 4096, "step": 1}),
|
||||
@@ -190,7 +190,6 @@ def create_settings(cmd_opts):
|
||||
"sdnq_quantize_conv_layers": OptionInfo(False, "Quantize convolutional layers", gr.Checkbox),
|
||||
"sdnq_quantize_embedding_layers": OptionInfo(False, "Quantize embedding layers", gr.Checkbox),
|
||||
"sdnq_dequantize_compile": OptionInfo(devices.has_triton(early=True), "Dequantize using torch.compile", gr.Checkbox),
|
||||
"sdnq_use_quantized_matmul": OptionInfo(False, "Use quantized MatMul", gr.Checkbox),
|
||||
"sdnq_use_quantized_matmul_conv": OptionInfo(False, "Use quantized MatMul with conv", gr.Checkbox),
|
||||
"sdnq_quantize_with_gpu": OptionInfo(True, "Quantize using GPU", gr.Checkbox),
|
||||
"sdnq_dequantize_fp32": OptionInfo(True, "Dequantize using full precision", gr.Checkbox),
|
||||
@@ -260,9 +259,8 @@ def create_settings(cmd_opts):
|
||||
"sdnq_attention_sep": OptionInfo("<h2>SDNQ Attention</h2>", "", gr.HTML),
|
||||
"sdnq_attention_smooth_k": OptionInfo(False, "SDNQ Attention use Smooth K", gr.Checkbox),
|
||||
"sdnq_attention_use_hadamard": OptionInfo(False, "SDNQ Attention use Hadamard", gr.Checkbox),
|
||||
"sdnq_attention_use_quantized_matmul": OptionInfo(True, "SDNQ Attention use Quantized MatMul", gr.Checkbox),
|
||||
"sdnq_attention_matmul_type": OptionInfo("auto", "SDNQ Attention MatMul type", gr.Radio, {"choices": sdnq_matmul_modes}),
|
||||
"sdnq_attention_pv_matmul_type": OptionInfo("auto", "SDNQ Attention PV MatMul type", gr.Radio, {"choices": sdnq_matmul_modes}),
|
||||
"sdnq_attention_matmul_type": OptionInfo("enabled", "SDNQ Attention MatMul type", gr.Radio, {"choices": sdnq_matmul_modes}),
|
||||
"sdnq_attention_pv_matmul_type": OptionInfo("disabled", "SDNQ Attention PV MatMul type", gr.Radio, {"choices": sdnq_matmul_modes}),
|
||||
"sdnq_attention_hadamard_group_size": OptionInfo(256, "SDNQ Attention Hadamard Group Size", gr.Slider, {"minimum": 4, "maximum": 1024, "step": 1}),
|
||||
|
||||
"hf_attention_sep": OptionInfo("<h2>Attention Dispatcher</h2>", "", gr.HTML),
|
||||
|
||||
@@ -867,7 +867,7 @@ def build_component_prequantized(
|
||||
weights_dtype=weights_dtype,
|
||||
quantized_matmul_dtype=matmul_dtype,
|
||||
group_size=NVFP4_GROUP_SIZE if is_nvfp4 else -1,
|
||||
use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul,
|
||||
use_quantized_matmul=(shared.opts.sdnq_quantize_matmul_mode != "disabled"),
|
||||
dequantize_fp32=shared.opts.sdnq_dequantize_fp32,
|
||||
add_skip_keys=False,
|
||||
modules_to_not_convert=[],
|
||||
@@ -976,7 +976,7 @@ def build_component_prequantized(
|
||||
component,
|
||||
dtype=target_dtype,
|
||||
dequantize_fp32=shared.opts.sdnq_dequantize_fp32,
|
||||
use_quantized_matmul=shared.opts.sdnq_use_quantized_matmul,
|
||||
use_quantized_matmul=(shared.opts.sdnq_quantize_matmul_mode != "disabled"),
|
||||
)
|
||||
return component
|
||||
|
||||
|
||||
@@ -1387,7 +1387,7 @@ class ComfyTestEnv:
|
||||
sdnq_common.is_fp8_compile_supported = self.fp8_compile_supported
|
||||
sdnq_common.check_torch_compile = lambda: not self.fp8_compile_supported
|
||||
self.orig_opts = {
|
||||
'sdnq_use_quantized_matmul': shared.opts.sdnq_use_quantized_matmul,
|
||||
'sdnq_use_quantized_matmul': (shared.opts.sdnq_quantize_matmul_mode != "disabled"),
|
||||
'sdnq_dequantize_fp32': shared.opts.sdnq_dequantize_fp32,
|
||||
'diffusers_offload_mode': shared.opts.diffusers_offload_mode,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user