From 67f495454c23aec6a6e34ee7faa5d1285095c95b Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 6 Mar 2025 22:55:03 +0300 Subject: [PATCH 1/7] IPEX fix torch.cuda.device --- modules/intel/ipex/hijacks.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/intel/ipex/hijacks.py b/modules/intel/ipex/hijacks.py index 5191e324a..cdce4d073 100644 --- a/modules/intel/ipex/hijacks.py +++ b/modules/intel/ipex/hijacks.py @@ -343,6 +343,13 @@ def torch_cuda_synchronize(device=None): else: return torch.xpu.synchronize(device) +@wraps(torch.cuda.device) +def torch_cuda_device(device): + if check_cuda(device): + return torch.xpu.device(return_xpu(device)) + else: + return torch.xpu.device(device) + # Hijack Functions: def ipex_hijacks(legacy=True): @@ -367,6 +374,7 @@ def ipex_hijacks(legacy=True): torch.load = torch_load torch.Generator = torch_Generator torch.cuda.synchronize = torch_cuda_synchronize + torch.cuda.device = torch_cuda_device torch.backends.cuda.sdp_kernel = return_null_context torch.nn.DataParallel = DummyDataParallel From 953a15a218b16362daf16b22d4845dd03ad45347 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 11 Mar 2025 16:35:48 +0300 Subject: [PATCH 2/7] IPEX fix PyTorch 2.7 compatibility --- CHANGELOG.md | 3 +- modules/intel/ipex/__init__.py | 103 +++++++++++++++++---------------- modules/intel/ipex/hijacks.py | 13 +++-- 3 files changed, 64 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffb84d8a2..b59ef0c17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,10 @@ # Change Log for SD.Next -## Update for 2025-03-06 +## Update for 2025-03-11 - fix installer not starting when older version of rich is installed - ipex, fix untyped_storage and torch.eye +- ipex, fix torch 2.7 compatibility ## Update for 2025-02-28 diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py index a36664bb3..cd272af70 100644 --- a/modules/intel/ipex/__init__.py +++ b/modules/intel/ipex/__init__.py @@ -4,11 +4,13 @@ import contextlib import torch try: import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import - legacy = True + has_ipex = True except Exception: - legacy = False + has_ipex = False from .hijacks import ipex_hijacks +torch_version = float(torch.__version__[:3]) + # pylint: disable=protected-access, missing-function-docstring, line-too-long def ipex_init(): # pylint: disable=too-many-statements @@ -45,7 +47,6 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.Optional = torch.xpu.Optional torch.cuda.__cached__ = torch.xpu.__cached__ torch.cuda.__loader__ = torch.xpu.__loader__ - torch.cuda.Tuple = torch.xpu.Tuple torch.cuda.streams = torch.xpu.streams torch.cuda.Any = torch.xpu.Any torch.cuda.__doc__ = torch.xpu.__doc__ @@ -58,7 +59,6 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.__annotations__ = torch.xpu.__annotations__ torch.cuda.__package__ = torch.xpu.__package__ torch.cuda.__builtins__ = torch.xpu.__builtins__ - torch.cuda.List = torch.xpu.List torch.cuda._lazy_init = torch.xpu._lazy_init torch.cuda.StreamContext = torch.xpu.StreamContext torch.cuda._lazy_call = torch.xpu._lazy_call @@ -70,47 +70,40 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.__file__ = torch.xpu.__file__ # torch.cuda.is_current_stream_capturing = torch.xpu.is_current_stream_capturing - if legacy: - torch.cuda.os = torch.xpu.os - torch.cuda.Device = torch.xpu.Device - torch.cuda.warnings = torch.xpu.warnings - torch.cuda.classproperty = torch.xpu.classproperty - torch.UntypedStorage.cuda = torch.UntypedStorage.xpu - if float(ipex.__version__[:3]) < 2.3: - torch.cuda._initialization_lock = torch.xpu.lazy_init._initialization_lock - torch.cuda._initialized = torch.xpu.lazy_init._initialized - torch.cuda._is_in_bad_fork = torch.xpu.lazy_init._is_in_bad_fork - torch.cuda._lazy_seed_tracker = torch.xpu.lazy_init._lazy_seed_tracker - torch.cuda._queued_calls = torch.xpu.lazy_init._queued_calls - torch.cuda._tls = torch.xpu.lazy_init._tls - torch.cuda.threading = torch.xpu.lazy_init.threading - torch.cuda.traceback = torch.xpu.lazy_init.traceback - torch.cuda._lazy_new = torch.xpu._lazy_new + if torch_version < 2.3: + torch.cuda._initialization_lock = torch.xpu.lazy_init._initialization_lock + torch.cuda._initialized = torch.xpu.lazy_init._initialized + torch.cuda._is_in_bad_fork = torch.xpu.lazy_init._is_in_bad_fork + torch.cuda._lazy_seed_tracker = torch.xpu.lazy_init._lazy_seed_tracker + torch.cuda._queued_calls = torch.xpu.lazy_init._queued_calls + torch.cuda._tls = torch.xpu.lazy_init._tls + torch.cuda.threading = torch.xpu.lazy_init.threading + torch.cuda.traceback = torch.xpu.lazy_init.traceback + torch.cuda._lazy_new = torch.xpu._lazy_new - torch.cuda.FloatTensor = torch.xpu.FloatTensor - torch.cuda.FloatStorage = torch.xpu.FloatStorage - torch.cuda.BFloat16Tensor = torch.xpu.BFloat16Tensor - torch.cuda.BFloat16Storage = torch.xpu.BFloat16Storage - torch.cuda.HalfTensor = torch.xpu.HalfTensor - torch.cuda.HalfStorage = torch.xpu.HalfStorage - torch.cuda.ByteTensor = torch.xpu.ByteTensor - torch.cuda.ByteStorage = torch.xpu.ByteStorage - torch.cuda.DoubleTensor = torch.xpu.DoubleTensor - torch.cuda.DoubleStorage = torch.xpu.DoubleStorage - torch.cuda.ShortTensor = torch.xpu.ShortTensor - torch.cuda.ShortStorage = torch.xpu.ShortStorage - torch.cuda.LongTensor = torch.xpu.LongTensor - torch.cuda.LongStorage = torch.xpu.LongStorage - torch.cuda.IntTensor = torch.xpu.IntTensor - torch.cuda.IntStorage = torch.xpu.IntStorage - torch.cuda.CharTensor = torch.xpu.CharTensor - torch.cuda.CharStorage = torch.xpu.CharStorage - torch.cuda.BoolTensor = torch.xpu.BoolTensor - torch.cuda.BoolStorage = torch.xpu.BoolStorage - torch.cuda.ComplexFloatStorage = torch.xpu.ComplexFloatStorage - torch.cuda.ComplexDoubleStorage = torch.xpu.ComplexDoubleStorage - - if not legacy or float(ipex.__version__[:3]) >= 2.3: + torch.cuda.FloatTensor = torch.xpu.FloatTensor + torch.cuda.FloatStorage = torch.xpu.FloatStorage + torch.cuda.BFloat16Tensor = torch.xpu.BFloat16Tensor + torch.cuda.BFloat16Storage = torch.xpu.BFloat16Storage + torch.cuda.HalfTensor = torch.xpu.HalfTensor + torch.cuda.HalfStorage = torch.xpu.HalfStorage + torch.cuda.ByteTensor = torch.xpu.ByteTensor + torch.cuda.ByteStorage = torch.xpu.ByteStorage + torch.cuda.DoubleTensor = torch.xpu.DoubleTensor + torch.cuda.DoubleStorage = torch.xpu.DoubleStorage + torch.cuda.ShortTensor = torch.xpu.ShortTensor + torch.cuda.ShortStorage = torch.xpu.ShortStorage + torch.cuda.LongTensor = torch.xpu.LongTensor + torch.cuda.LongStorage = torch.xpu.LongStorage + torch.cuda.IntTensor = torch.xpu.IntTensor + torch.cuda.IntStorage = torch.xpu.IntStorage + torch.cuda.CharTensor = torch.xpu.CharTensor + torch.cuda.CharStorage = torch.xpu.CharStorage + torch.cuda.BoolTensor = torch.xpu.BoolTensor + torch.cuda.BoolStorage = torch.xpu.BoolStorage + torch.cuda.ComplexFloatStorage = torch.xpu.ComplexFloatStorage + torch.cuda.ComplexDoubleStorage = torch.xpu.ComplexDoubleStorage + else: torch.cuda._initialization_lock = torch.xpu._initialization_lock torch.cuda._initialized = torch.xpu._initialized torch.cuda._is_in_bad_fork = torch.xpu._is_in_bad_fork @@ -120,12 +113,24 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.threading = torch.xpu.threading torch.cuda.traceback = torch.xpu.traceback + if torch_version < 2.5: + torch.cuda.os = torch.xpu.os + torch.cuda.Device = torch.xpu.Device + torch.cuda.warnings = torch.xpu.warnings + torch.cuda.classproperty = torch.xpu.classproperty + torch.UntypedStorage.cuda = torch.UntypedStorage.xpu + + if torch_version < 2.7: + torch.cuda.Tuple = torch.xpu.Tuple + torch.cuda.List = torch.xpu.List + + # Memory: if 'linux' in sys.platform and "WSL2" in os.popen("uname -a").read(): torch.xpu.empty_cache = lambda: None torch.cuda.empty_cache = torch.xpu.empty_cache - if legacy: + if has_ipex: torch.cuda.memory_summary = torch.xpu.memory_summary torch.cuda.memory_snapshot = torch.xpu.memory_snapshot torch.cuda.memory = torch.xpu.memory @@ -154,11 +159,11 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.initial_seed = torch.xpu.initial_seed # AMP: - if legacy: + if has_ipex: torch.xpu.amp.custom_fwd = torch.cuda.amp.custom_fwd torch.xpu.amp.custom_bwd = torch.cuda.amp.custom_bwd torch.cuda.amp = torch.xpu.amp - if float(ipex.__version__[:3]) < 2.3: + if torch_version < 2.3: torch.is_autocast_enabled = torch.xpu.is_autocast_xpu_enabled torch.get_autocast_gpu_dtype = torch.xpu.get_autocast_xpu_dtype @@ -177,7 +182,7 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.amp.GradScaler = ipex.cpu.autocast._grad_scaler.GradScaler # C - if legacy and float(ipex.__version__[:3]) < 2.3: + if torch_version < 2.3: torch._C._cuda_getCurrentRawStream = ipex._C._getCurrentRawStream ipex._C._DeviceProperties.multi_processor_count = ipex._C._DeviceProperties.gpu_subslice_count ipex._C._DeviceProperties.major = 12 @@ -206,7 +211,7 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.ipc_collect = lambda *args, **kwargs: None torch.cuda.utilization = lambda *args, **kwargs: 0 - device_supports_fp64, can_allocate_plus_4gb = ipex_hijacks(legacy=legacy) + device_supports_fp64, can_allocate_plus_4gb = ipex_hijacks() try: from .diffusers import ipex_diffusers ipex_diffusers(device_supports_fp64=device_supports_fp64, can_allocate_plus_4gb=can_allocate_plus_4gb) diff --git a/modules/intel/ipex/hijacks.py b/modules/intel/ipex/hijacks.py index cdce4d073..91a256aed 100644 --- a/modules/intel/ipex/hijacks.py +++ b/modules/intel/ipex/hijacks.py @@ -5,7 +5,10 @@ import torch import numpy as np from modules import devices, errors + +torch_version = float(torch.__version__[:3]) device_supports_fp64 = torch.xpu.has_fp64_dtype() if hasattr(torch.xpu, "has_fp64_dtype") else torch.xpu.get_device_properties(devices.device).has_fp64 + if os.environ.get('IPEX_FORCE_ATTENTION_SLICE', '0') == '0' and (torch.xpu.get_device_properties(devices.device).total_memory / 1024 / 1024 / 1024) > 4.1: try: x = torch.ones((33000,33000), dtype=torch.float32, device=devices.device) @@ -57,6 +60,7 @@ def autocast_init(self, device_type, dtype=None, enabled=True, cache_enabled=Non return original_autocast_init(self, device_type=device_type, dtype=dtype, enabled=enabled, cache_enabled=cache_enabled) # Latent Antialias CPU Offload: +# IPEX 2.5 and above has partial support but doesn't really work most of the time. original_interpolate = torch.nn.functional.interpolate @wraps(torch.nn.functional.interpolate) def interpolate(tensor, size=None, scale_factor=None, mode='nearest', align_corners=None, recompute_scale_factor=None, antialias=False): # pylint: disable=too-many-arguments @@ -245,7 +249,7 @@ def UntypedStorage_init(*args, device=None, **kwargs): else: return original_UntypedStorage_init(*args, device=device, **kwargs) -if float(torch.__version__[:3]) >= 2.4: +if torch_version >= 2.4: original_UntypedStorage_to = torch.UntypedStorage.to @wraps(torch.UntypedStorage.to) def UntypedStorage_to(self, *args, device=None, **kwargs): @@ -352,13 +356,11 @@ def torch_cuda_device(device): # Hijack Functions: -def ipex_hijacks(legacy=True): +def ipex_hijacks(): global device_supports_fp64, can_allocate_plus_4gb - if float(torch.__version__[:3]) >= 2.4: + if torch_version >= 2.4: torch.UntypedStorage.cuda = UntypedStorage_cuda torch.UntypedStorage.to = UntypedStorage_to - else: # ipex 2.3 and below - torch.nn.functional.interpolate = interpolate torch.tensor = torch_tensor torch.Tensor.to = Tensor_to torch.Tensor.cuda = Tensor_cuda @@ -381,6 +383,7 @@ def ipex_hijacks(legacy=True): torch.UntypedStorage.is_cuda = is_cuda torch.amp.autocast_mode.autocast.__init__ = autocast_init + torch.nn.functional.interpolate = interpolate torch.nn.functional.scaled_dot_product_attention = scaled_dot_product_attention torch.nn.functional.group_norm = functional_group_norm torch.nn.functional.layer_norm = functional_layer_norm From 30afbd036d2ce2612a2133926c355eab12dabeb0 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 11 Mar 2025 17:03:33 +0300 Subject: [PATCH 3/7] Fix circular import between sd_models.py and shared.py --- modules/interrogate/openclip.py | 5 +++-- modules/interrogate/vqa.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/interrogate/openclip.py b/modules/interrogate/openclip.py index 6d3656fab..761c0fa39 100644 --- a/modules/interrogate/openclip.py +++ b/modules/interrogate/openclip.py @@ -10,7 +10,7 @@ import gradio as gr from PIL import Image from torchvision import transforms from torchvision.transforms.functional import InterpolationMode -from modules import devices, paths, shared, lowvram, errors, sd_models +from modules import devices, paths, shared, lowvram, errors caption_models = { @@ -328,7 +328,8 @@ def interrogate_image(image, clip_model, blip_model, mode): lowvram.send_everything_to_cpu() devices.torch_gc() if shared.native and shared.sd_loaded: - sd_models.apply_balanced_offload(shared.sd_model) + from modules.sd_models import apply_balanced_offload # prevent circular import + apply_balanced_offload(shared.sd_model) load_interrogator(clip_model, blip_model) image = image.convert('RGB') prompt = interrogate(image, mode) diff --git a/modules/interrogate/vqa.py b/modules/interrogate/vqa.py index f0fb752d8..afe5aac09 100644 --- a/modules/interrogate/vqa.py +++ b/modules/interrogate/vqa.py @@ -7,7 +7,7 @@ import torch import transformers import transformers.dynamic_module_utils from PIL import Image -from modules import shared, devices, errors, sd_models +from modules import shared, devices, errors processor = None model = None @@ -429,7 +429,8 @@ def interrogate(question, prompt, image, model_name, quiet:bool=False): if len(question) < 2: question = "Describe the image." if shared.native and shared.sd_loaded: - sd_models.apply_balanced_offload(shared.sd_model) + from modules.sd_models import apply_balanced_offload # prevent circular import + apply_balanced_offload(shared.sd_model) from modules import modelloader modelloader.hf_login() try: From 9b7bb5f213d70ee73f2135b64c07b69e373bd828 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 11 Mar 2025 17:43:48 +0300 Subject: [PATCH 4/7] Add XPU to profiler --- modules/processing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/processing.py b/modules/processing.py index 02eee0206..3a49f99ba 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -188,6 +188,8 @@ def process_images(p: StableDiffusionProcessing) -> Processed: activities=[torch.profiler.ProfilerActivity.CPU] if torch.cuda.is_available(): activities.append(torch.profiler.ProfilerActivity.CUDA) + if devices.has_xpu() and hasattr(torch.profiler.ProfilerActivity, "XPU"): + activities.append(torch.profiler.ProfilerActivity.XPU) shared.log.debug(f'Torch profile: activities={activities}') if shared.profiler is None: profile_args = { From 2be555dc821eaba011fc2c8fe50da831014ac6fc Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Wed, 12 Mar 2025 00:50:27 +0900 Subject: [PATCH 5/7] fix directml backend --- modules/dml/hijack/torch.py | 7 ++++++- modules/sd_offload.py | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/modules/dml/hijack/torch.py b/modules/dml/hijack/torch.py index e216de46f..1786a1499 100644 --- a/modules/dml/hijack/torch.py +++ b/modules/dml/hijack/torch.py @@ -1,12 +1,17 @@ import torch - from modules.sd_hijack_utils import CondFunc + CondFunc('torchsde._brownian.brownian_interval._randn', lambda _, size, dtype, device, seed: torch.randn(size, dtype=dtype, device=torch.device("cpu"), generator=torch.Generator(torch.device("cpu")).manual_seed(int(seed))).to(device), lambda _, size, dtype, device, seed: device.type == 'privateuseone') # https://github.com/microsoft/DirectML/issues/400 CondFunc('torch.Tensor.new', lambda orig, self, *args, **kwargs: orig(self.cpu(), *args, **kwargs).to(self.device), lambda orig, self, *args, **kwargs: torch.dml.is_directml_device(self.device)) +def cuda(self: torch.Tensor): + return self.to(torch.dml.current_device()) +torch.Tensor.cuda = cuda + + # https://github.com/lshqqytiger/stable-diffusion-webui-directml/issues/436 _pow_ = torch.Tensor.pow_ def pow_(self: torch.Tensor, *args, **kwargs): diff --git a/modules/sd_offload.py b/modules/sd_offload.py index b200bed1a..7e603f4ec 100644 --- a/modules/sd_offload.py +++ b/modules/sd_offload.py @@ -162,6 +162,12 @@ class OffloadHook(accelerate.hooks.ModelHook): if device_map is None or max_memory != getattr(module, "balanced_offload_max_memory", None): device_map = accelerate.infer_auto_device_map(module, max_memory=max_memory) offload_dir = getattr(module, "offload_dir", os.path.join(shared.opts.accelerate_offload_path, module.__class__.__name__)) + keys = device_map.keys() + for v in keys: + if isinstance(device_map[v], int): + # int implies CUDA device, but it will break DirectML backend. + # Therefore, the type of device should be added. + device_map[v] = f"{devices.device.type}:{device_map[v]}" module = accelerate.dispatch_model(module, device_map=device_map, offload_dir=offload_dir) module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access module.balanced_offload_device_map = device_map From 1914ad901128155c5fccc9e0e035bc9b7343f313 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 11 Mar 2025 18:59:38 +0300 Subject: [PATCH 6/7] IPEX fix performance with balanced offload --- CHANGELOG.md | 7 +++++-- modules/sd_offload.py | 16 +++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b59ef0c17..29f6d8e68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,11 @@ ## Update for 2025-03-11 - fix installer not starting when older version of rich is installed -- ipex, fix untyped_storage and torch.eye -- ipex, fix torch 2.7 compatibility +- fix cuda errors with directml +- **ipex** + - fix untyped_storage and torch.eye + - fix torch 2.7 compatibility + - fix performance with balanced offload ## Update for 2025-02-28 diff --git a/modules/sd_offload.py b/modules/sd_offload.py index 7e603f4ec..33ce1b2c0 100644 --- a/modules/sd_offload.py +++ b/modules/sd_offload.py @@ -162,12 +162,13 @@ class OffloadHook(accelerate.hooks.ModelHook): if device_map is None or max_memory != getattr(module, "balanced_offload_max_memory", None): device_map = accelerate.infer_auto_device_map(module, max_memory=max_memory) offload_dir = getattr(module, "offload_dir", os.path.join(shared.opts.accelerate_offload_path, module.__class__.__name__)) - keys = device_map.keys() - for v in keys: - if isinstance(device_map[v], int): - # int implies CUDA device, but it will break DirectML backend. - # Therefore, the type of device should be added. - device_map[v] = f"{devices.device.type}:{device_map[v]}" + if devices.backend == "directml": + keys = device_map.keys() + for v in keys: + if isinstance(device_map[v], int): + # int implies CUDA or XPU device, but it will break DirectML backend. + # Therefore, the type of device should be added. + device_map[v] = f"{devices.device.type}:{device_map[v]}" module = accelerate.dispatch_model(module, device_map=device_map, offload_dir=offload_dir) module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access module.balanced_offload_device_map = device_map @@ -250,7 +251,8 @@ def apply_balanced_offload(sd_model, exclude=[]): prev_gpu = used_gpu do_offload = (perc_gpu > shared.opts.diffusers_offload_min_gpu_memory) and (module.device != devices.cpu) if do_offload: - module = module.to(devices.cpu, non_blocking=True) + non_blocking = devices.backend != "ipex" # non_blocking on ipex causes 2x slowdown + module = module.to(devices.cpu, non_blocking=non_blocking) used_gpu -= module_size cls = module.__class__.__name__ quant = getattr(module, "quantization_method", None) From 3226df25292d673a547f164a2e339841d17d780c Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 11 Mar 2025 19:25:43 +0300 Subject: [PATCH 7/7] Update changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29f6d8e68..6b7bac782 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,11 @@ ## Update for 2025-03-11 - fix installer not starting when older version of rich is installed +- fix circular imports when debug flags are enabled - fix cuda errors with directml - **ipex** - - fix untyped_storage and torch.eye + - add xpu to profiler + - fix untyped_storage, torch.eye and torch.cuda.device ops - fix torch 2.7 compatibility - fix performance with balanced offload