From 3253ec9c995f371fa5ddf944cd0108e6728a8fad Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Fri, 21 Feb 2025 23:54:14 +0900 Subject: [PATCH] zluda flash attention 2 --- modules/shared.py | 2 +- modules/zluda.py | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index d99ae3482..a348ad05d 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -453,7 +453,7 @@ def get_default_modes(): default_cross_attention = "Scaled-Dot-Product" if devices.backend == "zluda": - default_sdp_options = ['Math attention', 'Dynamic attention'] + default_sdp_options = ['Math attention', 'Flash attention', 'Dynamic attention'] else: default_sdp_options = ['Flash attention', 'Memory attention', 'Math attention'] diff --git a/modules/zluda.py b/modules/zluda.py index d0a690375..0203a6398 100644 --- a/modules/zluda.py +++ b/modules/zluda.py @@ -34,15 +34,13 @@ def initialize_zluda(): from modules import zluda_installer torch.backends.cudnn.enabled = zluda_installer.MIOpen_available - torch.backends.cuda.enable_flash_sdp(False) - torch.backends.cuda.enable_flash_sdp = do_nothing - torch.backends.cuda.enable_math_sdp(True) - torch.backends.cuda.enable_math_sdp = do_nothing - torch.backends.cuda.enable_mem_efficient_sdp(False) - torch.backends.cuda.enable_mem_efficient_sdp = do_nothing - if hasattr(torch.backends.cuda, "enable_cudnn_sdp"): + if not zluda_installer.MIOpen_available: torch.backends.cuda.enable_cudnn_sdp(False) torch.backends.cuda.enable_cudnn_sdp = do_nothing + torch.backends.cuda.enable_flash_sdp(False) + torch.backends.cuda.enable_flash_sdp = torch.backends.cuda.enable_cudnn_sdp + torch.backends.cuda.enable_mem_efficient_sdp(False) + torch.backends.cuda.enable_mem_efficient_sdp = do_nothing # ONNX Runtime is not supported ort.capi._pybind_state.get_available_providers = lambda: [v for v in available_execution_providers if v != ExecutionProvider.CUDA] # pylint: disable=protected-access