update cross-attention

CHANGELOG.md

@@ -7,19 +7,24 @@
   - [DeepCache](https://github.com/horseee/DeepCache) model acceleration  
     it can produce massive speedups (2x-5x) with no overhead, but with some loss of quality  
       *settings -> compute -> model compile -> deep-cache* and *settings -> compute -> model compile -> cache interval*  
-  - *Dynamic Attention Slicing*  
-    dynamically slices attention queries in order to save vram based on query size and slice rate in GB  
-    slicing gets only triggered if the query size is larger than the slice rate to gain performance  
-    *settings -> diffusers settings -> dynamic attention slicing*  
   - **Control** units now have extra option to re-use current preview image as processor input  
   - improved `clip-skip` value handling in diffusers, thanks @AI-Casanova & @Disty0  
     now clip-skip range is 0-12 where previously lowest value was 1 (default is still 1)  
     values can also be decimal to interpolate between different layers, for example `clip-skip: 1.5`, thanks @AI-Casanova  
+  - **Cross-attention** refactored cross-attention methods, thanks @Disty0  
+    - for backend:original, its unchanged: SDP, xFormers, Doggettxs, InvokeAI, Sub-quadratic, Split attention  
+    - for backend:diffuers, list is now: SDP, xFormers, Batch matrix-matrix, Split attention, Dynamic Attention BMM, Dynamic Attention SDP  
+      note: you may need to update your settings! if you were previously using split-attention, closest match is batch-matrix-matrix  
+  - **Dynamic Attention Slicing**  
+    dynamically slices attention queries in order to save vram based on query size and slice rate in GB  
+    slicing gets only triggered if the query size is larger than the slice rate to gain performance  
+    *settings -> diffusers settings -> dynamic attention slicing*  
   - **ONNX**:  
     - allow specify onnx default provider and cpu fallback  
       *settings -> diffusers*  
     - allow manual install of specific onnx flavor  
       *settings -> onnx*  
+    - better handling of `fp16` models/vae, thanks @lshqqytiger  
   - **OpenVINO**  
     - update to `torch 2.2.0`  
   - add `--theme` cli param to force theme on startup  

modules/sd_models.py

@@ -699,7 +699,7 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model'):
         sd_model.enable_attention_slicing()
     elif shared.opts.cross_attention_optimization == "xFormers" and hasattr(sd_model, 'enable_xformers_memory_efficient_attention'):
         sd_model.enable_xformers_memory_efficient_attention()
-    elif shared.opts.cross_attention_optimization == "Torch BMM":
+    elif shared.opts.cross_attention_optimization == "Batch matrix-matrix":
         from diffusers.models.attention_processor import AttnProcessor
         set_diffusers_attention(sd_model, AttnProcessor())
     elif shared.opts.cross_attention_optimization == "Dynamic Attention BMM":
@@ -1161,12 +1161,11 @@ def set_diffuser_pipe(pipe, new_pipe_type):
 
 
 def set_diffusers_attention(pipe, attention):
-        module_names, _ = pipe._get_signature_keys(pipe)
-        modules = [getattr(pipe, n, None) for n in module_names]
-        modules = [m for m in modules if isinstance(m, torch.nn.Module) and hasattr(m, "set_attn_processor")]
-
-        for module in modules:
-            module.set_attn_processor(attention)
+    module_names, _ = pipe._get_signature_keys(pipe) # pylint: disable=protected-access
+    modules = [getattr(pipe, n, None) for n in module_names]
+    modules = [m for m in modules if isinstance(m, torch.nn.Module) and hasattr(m, "set_attn_processor")]
+    for module in modules:
+        module.set_attn_processor(attention)
 
 
 def get_native(pipe: diffusers.DiffusionPipeline):

modules/shared.py

@@ -356,7 +356,7 @@ options_templates.update(options_section(('cuda', "Compute Settings"), {
 
     "cross_attention_sep": OptionInfo("<h2>Attention</h2>", "", gr.HTML),
     "cross_attention_optimization": OptionInfo(cross_attention_optimization_default, "Attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention(diffusers=backend == Backend.DIFFUSERS) }),
-    "cross_attention_options": OptionInfo([], "Attention advanced options", gr.CheckboxGroup, {"choices": ['xFormers enable flash Attention', 'SDP disable memory attention']}),
+    "cross_attention_options": OptionInfo([], "Attention advanced options", gr.CheckboxGroup, {"choices": ['xFormers enable flash Attention', 'SDP disable memory attention'], "visible": False }),
     "dynamic_attention_slice_rate": OptionInfo(4, "Slicing rate for Dynamic Attention Slicing in GB", gr.Slider, {"minimum": 0.1, "maximum": 16, "step": 0.1, "visible": backend == Backend.DIFFUSERS}),
     "sub_quad_sep": OptionInfo("<h3>Sub-quadratic options</h3>", "", gr.HTML, {"visible": backend == Backend.ORIGINAL}),
     "sub_quad_q_chunk_size": OptionInfo(512, "Attention query chunk size", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8, "visible": backend == Backend.ORIGINAL}),

modules/shared_items.py

@@ -17,9 +17,9 @@ def list_crossattention(diffusers=False):
     if diffusers:
         return [
             "Disabled",
-            "xFormers",
             "Scaled-Dot-Product",
-            "Torch BMM",
+            "xFormers",
+            "Batch matrix-matrix",
             "Split attention",
             "Dynamic Attention BMM",
             "Dynamic Attention SDP"
@@ -27,8 +27,8 @@ def list_crossattention(diffusers=False):
     else:
         return [
             "Disabled",
-            "xFormers",
             "Scaled-Dot-Product",
+            "xFormers",
             "Doggettx's",
             "InvokeAI's",
             "Sub-quadratic",