From 3f8da51eac6b1ffb543615ed5edbfe93647a09e0 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 11 Feb 2024 13:44:39 +0300 Subject: [PATCH 1/5] Set default atten for ROCm to SDP --- CHANGELOG.md | 5 +++-- modules/sd_hijack_dynamic_atten.py | 8 ++++++-- modules/shared.py | 4 +--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1486b7e3e..b98347037 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,9 +23,10 @@ - **Cross-attention** refactored cross-attention methods, thanks @Disty0 - for backend:original, its unchanged: SDP, xFormers, Doggettxs, InvokeAI, Sub-quadratic, Split attention - for backend:diffuers, list is now: SDP, xFormers, Batch matrix-matrix, Split attention, Dynamic Attention BMM, Dynamic Attention SDP - note: you may need to update your settings! if you were previously using split-attention, closest match is batch-matrix-matrix + note: you may need to update your settings! Attention Slicing is renamed to Split attention + - for ROCm, updated default cross-attention to Scaled Dot Product - **Dynamic Attention Slicing**, thanks @Disty0 - - dynamically slices attention queries in order to save vram based on query size and slice rate in GB + - dynamically slices attention queries in order to keep them under the slice rate slicing gets only triggered if the query size is larger than the slice rate to gain performance *Dynamic Attention Slicing BMM* uses *Batch matrix-matrix* *Dynamic Attention Slicing SDP* uses *Scaled Dot Product* diff --git a/modules/sd_hijack_dynamic_atten.py b/modules/sd_hijack_dynamic_atten.py index d9881beb7..2ccc81e77 100644 --- a/modules/sd_hijack_dynamic_atten.py +++ b/modules/sd_hijack_dynamic_atten.py @@ -99,8 +99,10 @@ def sliced_scaled_dot_product_attention(query, key, value, attn_mask=None, dropo class DynamicAttnProcessorSDP: r""" - dynamically slices attention queries based on query size and slice rate in GB + dynamically slices attention queries in order to keep them under the slice rate slicing will not get triggered if the query size is smaller than the slice rate to gain performance + + slice rate is in GB based on AttnProcessor V2 """ @@ -181,8 +183,10 @@ class DynamicAttnProcessorSDP: class DynamicAttnProcessorBMM: r""" - dynamically slices attention queries based on query size and slice rate in GB + dynamically slices attention queries in order to keep them under the slice rate slicing will not get triggered if the query size is smaller than the slice rate to gain performance + + slice rate is in GB based on AttnProcessor V1 """ diff --git a/modules/shared.py b/modules/shared.py index 86818f290..7082be669 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -318,9 +318,7 @@ elif devices.backend == "mps": cross_attention_optimization_default = "Scaled-Dot-Product" if backend == Backend.DIFFUSERS else "Doggettx's" elif devices.backend == "directml": cross_attention_optimization_default = "Dynamic Attention BMM" if backend == Backend.DIFFUSERS else "Sub-quadratic" -elif devices.backend == "rocm": - cross_attention_optimization_default = "Dynamic Attention BMM" if backend == Backend.DIFFUSERS else "Sub-quadratic" -else: # cuda and ipex +else: # cuda, rocm, ipex cross_attention_optimization_default ="Scaled-Dot-Product" From 3c9a4d55bb6650dedf4b03cbd965a9c9c5d01526 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 11 Feb 2024 17:23:47 -0500 Subject: [PATCH 2/5] init values for ip --- TODO.md | 1 + modules/processing_class.py | 6 +++--- modules/sd_models.py | 2 -- wiki | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/TODO.md b/TODO.md index 81f34ab89..655e6dce2 100644 --- a/TODO.md +++ b/TODO.md @@ -12,3 +12,4 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - masking api - preprocess api - bind panZoom to control input +- onediff: diff --git a/modules/processing_class.py b/modules/processing_class.py index 63e082091..fb228b2fe 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -115,9 +115,9 @@ class StableDiffusionProcessing: self.script_args_value: list = field(default=None, init=False) self.scripts_setup_complete: bool = field(default=False, init=False) # ip adapter - self.ip_adapter_names = None - self.ip_adapter_scales = 0.0 - self.ip_adapter_images = None + self.ip_adapter_names = [] + self.ip_adapter_scales = [0.0] + self.ip_adapter_images = [] # hdr self.hdr_mode=hdr_mode self.hdr_brightness=hdr_brightness diff --git a/modules/sd_models.py b/modules/sd_models.py index 44c20d3cf..28992fe5e 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -901,8 +901,6 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No else: modules.sd_hijack_accelerate.restore_accelerate() sd_model = pipeline.from_single_file(checkpoint_info.path, **diffusers_load_config) - if shared.opts.diffusers_to_gpu: - shared.log.debug(f'Model load: move={modules.sd_hijack_accelerate.tensor_to_timer:.2f}') if sd_model is not None and hasattr(sd_model, 'unet') and hasattr(sd_model.unet, 'config') and 'inpainting' in checkpoint_info.path.lower(): shared.log.debug('Model patch: type=inpaint') sd_model.unet.config.in_channels = 9 diff --git a/wiki b/wiki index f294ce941..102ab490f 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit f294ce94114ef3663dcf0a09e5d4de1065d46bf0 +Subproject commit 102ab490f3bfd443e8c9f8e74a86612e699e313a From c9e01b4a324946c9f2fc1a003a9f63d2f4725bdb Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 12 Feb 2024 01:57:27 +0300 Subject: [PATCH 3/5] OpenVINO fix --device-id with single GPU --- modules/intel/ipex/attention.py | 12 ++++++------ modules/intel/openvino/__init__.py | 2 ++ modules/sd_hijack_dynamic_atten.py | 10 +++++----- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/modules/intel/ipex/attention.py b/modules/intel/ipex/attention.py index 45c0525e6..1bc27323b 100644 --- a/modules/intel/ipex/attention.py +++ b/modules/intel/ipex/attention.py @@ -128,9 +128,9 @@ def torch_bmm_32_bit(input, mat2, *, out=None): return hidden_states original_scaled_dot_product_attention = torch.nn.functional.scaled_dot_product_attention -def scaled_dot_product_attention_32_bit(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False): +def scaled_dot_product_attention_32_bit(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, **kwargs): if query.device.type != "xpu": - return original_scaled_dot_product_attention(query, key, value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal) + return original_scaled_dot_product_attention(query, key, value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, **kwargs) do_split, do_split_2, do_split_3, split_slice_size, split_2_slice_size, split_3_slice_size = find_sdpa_slice_sizes(query.shape, query.element_size()) # Slice SDPA @@ -153,7 +153,7 @@ def scaled_dot_product_attention_32_bit(query, key, value, attn_mask=None, dropo key[start_idx:end_idx, start_idx_2:end_idx_2, start_idx_3:end_idx_3], value[start_idx:end_idx, start_idx_2:end_idx_2, start_idx_3:end_idx_3], attn_mask=attn_mask[start_idx:end_idx, start_idx_2:end_idx_2, start_idx_3:end_idx_3] if attn_mask is not None else attn_mask, - dropout_p=dropout_p, is_causal=is_causal + dropout_p=dropout_p, is_causal=is_causal, **kwargs ) else: hidden_states[start_idx:end_idx, start_idx_2:end_idx_2] = original_scaled_dot_product_attention( @@ -161,7 +161,7 @@ def scaled_dot_product_attention_32_bit(query, key, value, attn_mask=None, dropo key[start_idx:end_idx, start_idx_2:end_idx_2], value[start_idx:end_idx, start_idx_2:end_idx_2], attn_mask=attn_mask[start_idx:end_idx, start_idx_2:end_idx_2] if attn_mask is not None else attn_mask, - dropout_p=dropout_p, is_causal=is_causal + dropout_p=dropout_p, is_causal=is_causal, **kwargs ) else: hidden_states[start_idx:end_idx] = original_scaled_dot_product_attention( @@ -169,9 +169,9 @@ def scaled_dot_product_attention_32_bit(query, key, value, attn_mask=None, dropo key[start_idx:end_idx], value[start_idx:end_idx], attn_mask=attn_mask[start_idx:end_idx] if attn_mask is not None else attn_mask, - dropout_p=dropout_p, is_causal=is_causal + dropout_p=dropout_p, is_causal=is_causal, **kwargs ) torch.xpu.synchronize(query.device) else: - return original_scaled_dot_product_attention(query, key, value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal) + return original_scaled_dot_product_attention(query, key, value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, **kwargs) return hidden_states diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index bb4f1f1c5..0e05bc410 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -91,6 +91,8 @@ def get_device(): device = "CPU" elif shared.cmd_opts.device_id is not None: device = f"GPU.{shared.cmd_opts.device_id}" + if device not in available_devices: + device = "GPU.0" if "GPU.0" in available_devices else "GPU" if "GPU" in available_devices else "CPU" elif "GPU" in core.available_devices: device = "GPU" elif "GPU.1" in core.available_devices: diff --git a/modules/sd_hijack_dynamic_atten.py b/modules/sd_hijack_dynamic_atten.py index 2ccc81e77..5ba9dc6e4 100644 --- a/modules/sd_hijack_dynamic_atten.py +++ b/modules/sd_hijack_dynamic_atten.py @@ -49,7 +49,7 @@ def find_slice_sizes(query_shape, query_element_size, slice_rate=4): return do_split, do_split_2, do_split_3, split_slice_size, split_2_slice_size, split_3_slice_size -def sliced_scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False): +def sliced_scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, **kwargs): do_split, do_split_2, do_split_3, split_slice_size, split_2_slice_size, split_3_slice_size = find_slice_sizes(query.shape, query.element_size(), slice_rate=shared.opts.dynamic_attention_slice_rate) # Slice SDPA @@ -72,7 +72,7 @@ def sliced_scaled_dot_product_attention(query, key, value, attn_mask=None, dropo key[start_idx:end_idx, start_idx_2:end_idx_2, start_idx_3:end_idx_3], value[start_idx:end_idx, start_idx_2:end_idx_2, start_idx_3:end_idx_3], attn_mask=attn_mask[start_idx:end_idx, start_idx_2:end_idx_2, start_idx_3:end_idx_3] if attn_mask is not None else attn_mask, - dropout_p=dropout_p, is_causal=is_causal + dropout_p=dropout_p, is_causal=is_causal, **kwargs ) else: hidden_states[start_idx:end_idx, start_idx_2:end_idx_2] = F.scaled_dot_product_attention( @@ -80,7 +80,7 @@ def sliced_scaled_dot_product_attention(query, key, value, attn_mask=None, dropo key[start_idx:end_idx, start_idx_2:end_idx_2], value[start_idx:end_idx, start_idx_2:end_idx_2], attn_mask=attn_mask[start_idx:end_idx, start_idx_2:end_idx_2] if attn_mask is not None else attn_mask, - dropout_p=dropout_p, is_causal=is_causal + dropout_p=dropout_p, is_causal=is_causal, **kwargs ) else: hidden_states[start_idx:end_idx] = F.scaled_dot_product_attention( @@ -88,12 +88,12 @@ def sliced_scaled_dot_product_attention(query, key, value, attn_mask=None, dropo key[start_idx:end_idx], value[start_idx:end_idx], attn_mask=attn_mask[start_idx:end_idx] if attn_mask is not None else attn_mask, - dropout_p=dropout_p, is_causal=is_causal + dropout_p=dropout_p, is_causal=is_causal, **kwargs ) if devices.backend != "directml": getattr(torch, query.device.type).synchronize() else: - return F.scaled_dot_product_attention(query, key, value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal) + return F.scaled_dot_product_attention(query, key, value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal, **kwargs) return hidden_states From a675347287e184588596e56bb51698830fbf8de4 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 12 Feb 2024 01:58:20 +0300 Subject: [PATCH 4/5] Cleanup --- modules/intel/openvino/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index 0e05bc410..38b9fe35c 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -91,8 +91,8 @@ def get_device(): device = "CPU" elif shared.cmd_opts.device_id is not None: device = f"GPU.{shared.cmd_opts.device_id}" - if device not in available_devices: - device = "GPU.0" if "GPU.0" in available_devices else "GPU" if "GPU" in available_devices else "CPU" + if device not in core.available_devices: + device = "GPU.0" if "GPU.0" in core.available_devices else "GPU" if "GPU" in core.available_devices else "CPU" elif "GPU" in core.available_devices: device = "GPU" elif "GPU.1" in core.available_devices: From baf42fbc0a304ecdbb8196cbe2d473852c650e51 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 12 Feb 2024 05:15:17 +0300 Subject: [PATCH 5/5] Fix dtype mismatch with vae decode --- modules/processing_vae.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/processing_vae.py b/modules/processing_vae.py index e2c78b15c..bb0a11cb5 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -44,8 +44,8 @@ def full_vae_decode(latents, model): upcast = (model.vae.dtype == torch.float16) and getattr(model.vae.config, 'force_upcast', False) and hasattr(model, 'upcast_vae') if upcast: # this is done by diffusers automatically if output_type != 'latent' model.upcast_vae() - latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) + latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) decoded = model.vae.decode(latents / model.vae.config.scaling_factor, return_dict=False)[0] # Delete PyTorch VAE after OpenVINO compile