From 304caf8c063d3aa1d284a1dc68e44c06b37f80cf Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 10 Feb 2024 10:45:10 -0500 Subject: [PATCH] update cross-attention --- CHANGELOG.md | 13 +++++++++---- modules/sd_models.py | 13 ++++++------- modules/shared.py | 2 +- modules/shared_items.py | 6 +++--- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07bea87ad..baa7bc66e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,19 +7,24 @@ - [DeepCache](https://github.com/horseee/DeepCache) model acceleration it can produce massive speedups (2x-5x) with no overhead, but with some loss of quality *settings -> compute -> model compile -> deep-cache* and *settings -> compute -> model compile -> cache interval* - - *Dynamic Attention Slicing* - dynamically slices attention queries in order to save vram based on query size and slice rate in GB - slicing gets only triggered if the query size is larger than the slice rate to gain performance - *settings -> diffusers settings -> dynamic attention slicing* - **Control** units now have extra option to re-use current preview image as processor input - improved `clip-skip` value handling in diffusers, thanks @AI-Casanova & @Disty0 now clip-skip range is 0-12 where previously lowest value was 1 (default is still 1) values can also be decimal to interpolate between different layers, for example `clip-skip: 1.5`, thanks @AI-Casanova + - **Cross-attention** refactored cross-attention methods, thanks @Disty0 + - for backend:original, its unchanged: SDP, xFormers, Doggettxs, InvokeAI, Sub-quadratic, Split attention + - for backend:diffuers, list is now: SDP, xFormers, Batch matrix-matrix, Split attention, Dynamic Attention BMM, Dynamic Attention SDP + note: you may need to update your settings! if you were previously using split-attention, closest match is batch-matrix-matrix + - **Dynamic Attention Slicing** + dynamically slices attention queries in order to save vram based on query size and slice rate in GB + slicing gets only triggered if the query size is larger than the slice rate to gain performance + *settings -> diffusers settings -> dynamic attention slicing* - **ONNX**: - allow specify onnx default provider and cpu fallback *settings -> diffusers* - allow manual install of specific onnx flavor *settings -> onnx* + - better handling of `fp16` models/vae, thanks @lshqqytiger - **OpenVINO** - update to `torch 2.2.0` - add `--theme` cli param to force theme on startup diff --git a/modules/sd_models.py b/modules/sd_models.py index 79f782c3e..724dca68e 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -699,7 +699,7 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model'): sd_model.enable_attention_slicing() elif shared.opts.cross_attention_optimization == "xFormers" and hasattr(sd_model, 'enable_xformers_memory_efficient_attention'): sd_model.enable_xformers_memory_efficient_attention() - elif shared.opts.cross_attention_optimization == "Torch BMM": + elif shared.opts.cross_attention_optimization == "Batch matrix-matrix": from diffusers.models.attention_processor import AttnProcessor set_diffusers_attention(sd_model, AttnProcessor()) elif shared.opts.cross_attention_optimization == "Dynamic Attention BMM": @@ -1161,12 +1161,11 @@ def set_diffuser_pipe(pipe, new_pipe_type): def set_diffusers_attention(pipe, attention): - module_names, _ = pipe._get_signature_keys(pipe) - modules = [getattr(pipe, n, None) for n in module_names] - modules = [m for m in modules if isinstance(m, torch.nn.Module) and hasattr(m, "set_attn_processor")] - - for module in modules: - module.set_attn_processor(attention) + module_names, _ = pipe._get_signature_keys(pipe) # pylint: disable=protected-access + modules = [getattr(pipe, n, None) for n in module_names] + modules = [m for m in modules if isinstance(m, torch.nn.Module) and hasattr(m, "set_attn_processor")] + for module in modules: + module.set_attn_processor(attention) def get_native(pipe: diffusers.DiffusionPipeline): diff --git a/modules/shared.py b/modules/shared.py index 0a2ef3c48..0d53d312c 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -356,7 +356,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), { "cross_attention_sep": OptionInfo("

Attention

", "", gr.HTML), "cross_attention_optimization": OptionInfo(cross_attention_optimization_default, "Attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention(diffusers=backend == Backend.DIFFUSERS) }), - "cross_attention_options": OptionInfo([], "Attention advanced options", gr.CheckboxGroup, {"choices": ['xFormers enable flash Attention', 'SDP disable memory attention']}), + "cross_attention_options": OptionInfo([], "Attention advanced options", gr.CheckboxGroup, {"choices": ['xFormers enable flash Attention', 'SDP disable memory attention'], "visible": False }), "dynamic_attention_slice_rate": OptionInfo(4, "Slicing rate for Dynamic Attention Slicing in GB", gr.Slider, {"minimum": 0.1, "maximum": 16, "step": 0.1, "visible": backend == Backend.DIFFUSERS}), "sub_quad_sep": OptionInfo("

Sub-quadratic options

", "", gr.HTML, {"visible": backend == Backend.ORIGINAL}), "sub_quad_q_chunk_size": OptionInfo(512, "Attention query chunk size", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8, "visible": backend == Backend.ORIGINAL}), diff --git a/modules/shared_items.py b/modules/shared_items.py index 721ee0c16..24bf66caa 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -17,9 +17,9 @@ def list_crossattention(diffusers=False): if diffusers: return [ "Disabled", - "xFormers", "Scaled-Dot-Product", - "Torch BMM", + "xFormers", + "Batch matrix-matrix", "Split attention", "Dynamic Attention BMM", "Dynamic Attention SDP" @@ -27,8 +27,8 @@ def list_crossattention(diffusers=False): else: return [ "Disabled", - "xFormers", "Scaled-Dot-Product", + "xFormers", "Doggettx's", "InvokeAI's", "Sub-quadratic",