Public/automatic
1
0
Fork 0
mirror of https://github.com/vladmandic/automatic synced 2026-09-19 17:24:32 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add FP16 Accumulation option to SDNQ Atten

Browse Source
This commit is contained in:
Dity0
2026-08-18 20:44:05 +03:00
parent 05eef5e919
commit 0218a0aadc
3 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/attention.py
+2 -1
View File
@@ -38,6 +38,7 @@ def set_sdnq_attention():
smooth_k=shared.opts.sdnq_attention_smooth_k,
use_hadamard=shared.opts.sdnq_attention_use_hadamard,
hadamard_group_size=shared.opts.sdnq_attention_hadamard_group_size,
use_fp16_accum=shared.opts.sdnq_attention_use_fp16_accum,
)
else:
if enable_gqa:
@@ -45,7 +46,7 @@ def set_sdnq_attention():
return sdpa_pre_sdnq_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs)
torch.nn.functional.scaled_dot_product_attention = sdpa_sdnq_atten
torch_info.set(attention='sdnq')
log.debug(f'Torch attention: type="SDNQ attention" matmul={shared.opts.sdnq_attention_matmul_type}:{shared.opts.sdnq_attention_pv_matmul_type} smooth={shared.opts.sdnq_attention_smooth_k} hadamard={shared.opts.sdnq_attention_use_hadamard}')
log.debug(f'Torch attention: type="SDNQ attention" matmul={shared.opts.sdnq_attention_matmul_type}:{shared.opts.sdnq_attention_pv_matmul_type} smooth={shared.opts.sdnq_attention_smooth_k} hadamard={shared.opts.sdnq_attention_use_hadamard} fp16_accum={shared.opts.sdnq_attention_use_fp16_accum}')
except Exception as err:
log.error(f'Torch attention: type="SDNQ attention" {err}')
modules/ui_definitions.py
+1
View File
@@ -257,6 +257,7 @@ def create_settings(cmd_opts):
"sdnq_attention_sep": OptionInfo("<h2>SDNQ Attention</h2>", "", gr.HTML),
"sdnq_attention_smooth_k": OptionInfo(True, "SDNQ Attention use Smooth K", gr.Checkbox),
"sdnq_attention_use_hadamard": OptionInfo(False, "SDNQ Attention use Hadamard", gr.Checkbox),
"sdnq_attention_use_fp16_accum": OptionInfo(False, "SDNQ Attention use FP16 Accumulation", gr.Checkbox),
"sdnq_attention_matmul_type": OptionInfo("enabled", "SDNQ Attention MatMul type", gr.Radio, {"choices": sdnq_matmul_modes}),
"sdnq_attention_pv_matmul_type": OptionInfo("disabled", "SDNQ Attention PV MatMul type", gr.Radio, {"choices": sdnq_matmul_modes}),
"sdnq_attention_hadamard_group_size": OptionInfo(256, "SDNQ Attention Hadamard Group Size", gr.Slider, {"minimum": 4, "maximum": 1024, "step": 1}),