diff --git a/extensions-builtin/sdnq b/extensions-builtin/sdnq index cc94d3e58..d3ccffcdd 160000 --- a/extensions-builtin/sdnq +++ b/extensions-builtin/sdnq @@ -1 +1 @@ -Subproject commit cc94d3e5803da3beb556022a121e315d203fd63c +Subproject commit d3ccffcdd6cf7ba831167dc1f8f8d5684044df1c diff --git a/modules/attention.py b/modules/attention.py index e823375be..1aad29692 100644 --- a/modules/attention.py +++ b/modules/attention.py @@ -38,6 +38,7 @@ def set_sdnq_attention(): smooth_k=shared.opts.sdnq_attention_smooth_k, use_hadamard=shared.opts.sdnq_attention_use_hadamard, hadamard_group_size=shared.opts.sdnq_attention_hadamard_group_size, + use_fp16_accum=shared.opts.sdnq_attention_use_fp16_accum, ) else: if enable_gqa: @@ -45,7 +46,7 @@ def set_sdnq_attention(): return sdpa_pre_sdnq_atten(query=query, key=key, value=value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, scale=scale, **kwargs) torch.nn.functional.scaled_dot_product_attention = sdpa_sdnq_atten torch_info.set(attention='sdnq') - log.debug(f'Torch attention: type="SDNQ attention" matmul={shared.opts.sdnq_attention_matmul_type}:{shared.opts.sdnq_attention_pv_matmul_type} smooth={shared.opts.sdnq_attention_smooth_k} hadamard={shared.opts.sdnq_attention_use_hadamard}') + log.debug(f'Torch attention: type="SDNQ attention" matmul={shared.opts.sdnq_attention_matmul_type}:{shared.opts.sdnq_attention_pv_matmul_type} smooth={shared.opts.sdnq_attention_smooth_k} hadamard={shared.opts.sdnq_attention_use_hadamard} fp16_accum={shared.opts.sdnq_attention_use_fp16_accum}') except Exception as err: log.error(f'Torch attention: type="SDNQ attention" {err}') diff --git a/modules/ui_definitions.py b/modules/ui_definitions.py index 2335f5c1c..65cdfbc5e 100644 --- a/modules/ui_definitions.py +++ b/modules/ui_definitions.py @@ -257,6 +257,7 @@ def create_settings(cmd_opts): "sdnq_attention_sep": OptionInfo("

SDNQ Attention

", "", gr.HTML), "sdnq_attention_smooth_k": OptionInfo(True, "SDNQ Attention use Smooth K", gr.Checkbox), "sdnq_attention_use_hadamard": OptionInfo(False, "SDNQ Attention use Hadamard", gr.Checkbox), + "sdnq_attention_use_fp16_accum": OptionInfo(False, "SDNQ Attention use FP16 Accumulation", gr.Checkbox), "sdnq_attention_matmul_type": OptionInfo("enabled", "SDNQ Attention MatMul type", gr.Radio, {"choices": sdnq_matmul_modes}), "sdnq_attention_pv_matmul_type": OptionInfo("disabled", "SDNQ Attention PV MatMul type", gr.Radio, {"choices": sdnq_matmul_modes}), "sdnq_attention_hadamard_group_size": OptionInfo(256, "SDNQ Attention Hadamard Group Size", gr.Slider, {"minimum": 4, "maximum": 1024, "step": 1}),