diff --git a/extensions-builtin/LDSR/ldsr_model_arch.py b/extensions-builtin/LDSR/ldsr_model_arch.py index 41d97d071..f0a90c657 100644 --- a/extensions-builtin/LDSR/ldsr_model_arch.py +++ b/extensions-builtin/LDSR/ldsr_model_arch.py @@ -12,7 +12,7 @@ import safetensors.torch from ldm.models.diffusion.ddim import DDIMSampler from ldm.util import instantiate_from_config, ismap -from modules import shared, sd_hijack +from modules import devices, shared, sd_hijack cached_ldsr_model: torch.nn.Module = None @@ -113,7 +113,7 @@ class LDSR: gc.collect() if torch.cuda.is_available: torch.cuda.empty_cache() - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': torch.xpu.empty_cache() im_og = image diff --git a/extensions-builtin/SwinIR/scripts/swinir_model.py b/extensions-builtin/SwinIR/scripts/swinir_model.py index cd8ddb08c..f8cd77e69 100644 --- a/extensions-builtin/SwinIR/scripts/swinir_model.py +++ b/extensions-builtin/SwinIR/scripts/swinir_model.py @@ -7,7 +7,7 @@ from tqdm.rich import tqdm from swinir_model_arch import SwinIR as net from swinir_model_arch_v2 import Swin2SR as net2 from modules import modelloader, devices, script_callbacks, shared -from modules.shared import cmd_opts, opts, state +from modules.shared import opts, state from modules.upscaler import Upscaler, UpscalerData @@ -41,11 +41,12 @@ class UpscalerSwinIR(Upscaler): model = model.to(device_swinir, dtype=devices.dtype) img = upscale(img, model) try: - torch.cuda.empty_cache() + if devices.backend == 'ipex': + torch.xpu.empty_cache() + else: + torch.cuda.empty_cache() except Exception: pass - if cmd_opts.use_ipex: - torch.xpu.empty_cache() return img def load_model(self, path, scale=4): diff --git a/installer.py b/installer.py index 4e9a3b805..c01b88fe0 100644 --- a/installer.py +++ b/installer.py @@ -295,8 +295,11 @@ def check_torch(): os.environ.setdefault('PYTORCH_HIP_ALLOC_CONF', 'garbage_collection_threshold:0.8,max_split_size_mb:512') torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.0.1 torchvision==0.15.2 --index-url https://download.pytorch.org/whl/rocm5.4.2') xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') - elif allow_ipex and args.use_ipex and shutil.which('sycl-ls') is not None: + elif allow_ipex and (args.use_ipex or shutil.which('sycl-ls') is not None or os.environ.get('ONEAPI_ROOT') is not None or os.path.exists('/opt/intel/oneapi')): + args.use_ipex = True log.info('Intel OneAPI Toolkit detected') + if shutil.which('sycl-ls') is None: + log.error('Intel OneAPI Toolkit is not activated! Start the WebUI with --use-ipex or activate OneAPI manually') torch_command = os.environ.get('TORCH_COMMAND', 'torch==1.13.0a0 torchvision==0.14.1a0 intel_extension_for_pytorch==1.13.120+xpu -f https://developer.intel.com/ipex-whl-stable-xpu') xformers_package = os.environ.get('XFORMERS_PACKAGE', 'none') else: diff --git a/modules/api/api.py b/modules/api/api.py index 2505b0891..82853117f 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -605,7 +605,7 @@ class Api: ram = { 'error': f'{err}' } try: import torch - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': system = { 'free': (torch.xpu.get_device_properties(shared.device).total_memory - torch.xpu.memory_allocated()), 'used': torch.xpu.memory_allocated(), 'total': torch.xpu.get_device_properties(shared.device).total_memory } s = dict(torch.xpu.memory_stats()) allocated = { 'current': s['allocated_bytes.all.current'], 'peak': s['allocated_bytes.all.peak'] } diff --git a/modules/codeformer_model.py b/modules/codeformer_model.py index a57e1ee49..92763e078 100644 --- a/modules/codeformer_model.py +++ b/modules/codeformer_model.py @@ -101,7 +101,7 @@ def setup_model(dirname): output = self.net(cropped_face_t, w=w if w is not None else shared.opts.code_former_weight, adain=True)[0] restored_face = tensor2img(output, rgb2bgr=True, min_max=(-1, 1)) del output - if cmd_opts.use_ipex: + if devices.backend == 'ipex': torch.xpu.empty_cache() else: torch.cuda.empty_cache() diff --git a/modules/devices.py b/modules/devices.py index 64852d109..561c73df3 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -23,7 +23,7 @@ def extract_device_id(args, name): # pylint: disable=redefined-outer-name def get_cuda_device_string(): - if shared.cmd_opts.use_ipex: + if backend == 'ipex': if shared.cmd_opts.device_id is not None: return f"xpu:{shared.cmd_opts.device_id}" return "xpu" @@ -34,7 +34,7 @@ def get_cuda_device_string(): def get_optimal_device_name(): - if (cuda_ok or shared.cmd_opts.use_ipex) and not shared.cmd_opts.use_directml: + if (cuda_ok or backend == 'ipex') and not shared.cmd_opts.use_directml: return get_cuda_device_string() if has_mps(): return "mps" @@ -64,7 +64,7 @@ def torch_gc(force=False): if shared.opts.disable_gc and not force: return collected = gc.collect() - if shared.cmd_opts.use_ipex: + if backend == 'ipex': try: with torch.xpu.device(get_cuda_device_string()): torch.xpu.empty_cache() @@ -161,7 +161,20 @@ def set_cuda_params(): args = cmd_args.parser.parse_args() -if args.use_ipex: +if args.use_ipex or (hasattr(torch, 'xpu') and torch.xpu.is_available()): + backend = 'ipex' +elif args.use_directml: + backend = 'directml' +elif torch.cuda.is_available() and torch.version.cuda: + backend = 'cuda' +elif torch.cuda.is_available() and torch.version.hip: + backend = 'rocm' +elif sys.platform == 'darwin': + backend = 'mps' +else: + backend = 'cpu' + +if backend == 'ipex': #Fix broken function in ipex 1.13.120+xpu from modules.sd_hijack_utils import CondFunc #Functions with dtype errors: @@ -206,19 +219,6 @@ dtype = torch.float16 dtype_vae = torch.float16 dtype_unet = torch.float16 unet_needs_upcast = False -if args.use_ipex: - backend = 'ipex' -elif args.use_directml: - backend = 'directml' -elif torch.cuda.is_available() and torch.version.cuda: - backend = 'cuda' -elif torch.cuda.is_available() and torch.version.hip: - backend = 'rocm' -elif sys.platform == 'darwin': - backend = 'mps' -else: - backend = 'cpu' - def cond_cast_unet(tensor): @@ -231,7 +231,7 @@ def cond_cast_float(tensor): def randn(seed, shape): torch.manual_seed(seed) - if shared.cmd_opts.use_ipex: + if backend == 'ipex': torch.xpu.manual_seed_all(seed) if device.type == 'mps': return torch.randn(shape, device=cpu).to(device) @@ -251,7 +251,7 @@ def autocast(disable=False): return contextlib.nullcontext() if shared.cmd_opts.use_directml: return torch.dml.amp.autocast(dtype) - if shared.cmd_opts.use_ipex: + if backend == 'ipex': return torch.xpu.amp.autocast(enabled=True, dtype=dtype) if cuda_ok: return torch.autocast("cuda") @@ -264,7 +264,7 @@ def without_autocast(disable=False): return contextlib.nullcontext() if shared.cmd_opts.use_directml: return torch.dml.amp.autocast(enabled=False) if torch.is_autocast_enabled() else contextlib.nullcontext() - if shared.cmd_opts.use_ipex: + if backend == 'ipex': return torch.xpu.amp.autocast(enabled=False) if torch.is_autocast_enabled() else contextlib.nullcontext() if cuda_ok: return torch.autocast("cuda", enabled=False) if torch.is_autocast_enabled() else contextlib.nullcontext() diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py index 18e58abfc..471e75322 100644 --- a/modules/hypernetworks/hypernetwork.py +++ b/modules/hypernetworks/hypernetwork.py @@ -589,7 +589,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi print("Cannot resume from saved optimizer!") print(e) - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': scaler = torch.xpu.amp.GradScaler() else: scaler = torch.cuda.amp.GradScaler() @@ -706,7 +706,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi hypernetwork.eval() rng_state = torch.get_rng_state() cuda_rng_state = None - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': cuda_rng_state = torch.xpu.get_rng_state_all() elif torch.cuda.is_available(): cuda_rng_state = torch.cuda.get_rng_state_all() @@ -745,7 +745,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi shared.sd_model.cond_stage_model.to(devices.cpu) shared.sd_model.first_stage_model.to(devices.cpu) torch.set_rng_state(rng_state) - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': torch.xpu.set_rng_state_all(cuda_rng_state) elif torch.cuda.is_available(): torch.cuda.set_rng_state_all(cuda_rng_state) diff --git a/modules/memmon.py b/modules/memmon.py index 3cf30f5c7..929451875 100644 --- a/modules/memmon.py +++ b/modules/memmon.py @@ -2,7 +2,7 @@ import threading import time from collections import defaultdict import torch -from modules import shared +from modules import devices class MemUsageMonitor(threading.Thread): @@ -22,7 +22,7 @@ class MemUsageMonitor(threading.Thread): self.data = defaultdict(int) if not torch.cuda.is_available(): #torch.cuda.is_available() reports False when using IPEX. - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': self.cuda_mem_get_info() torch.xpu.memory_stats(self.device) else: @@ -35,7 +35,7 @@ class MemUsageMonitor(threading.Thread): self.disabled = True def cuda_mem_get_info(self): - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': index = self.device.index if self.device.index is not None else torch.xpu.current_device() return [(torch.xpu.get_device_properties(index).total_memory - torch.xpu.memory_allocated(index)), torch.xpu.get_device_properties(index).total_memory] else: @@ -47,7 +47,7 @@ class MemUsageMonitor(threading.Thread): return while True: self.run_flag.wait() - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': torch.xpu.reset_peak_memory_stats() else: torch.cuda.reset_peak_memory_stats() @@ -70,7 +70,7 @@ class MemUsageMonitor(threading.Thread): self.data["free"] = free self.data["total"] = total try: - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': torch_stats = torch.xpu.memory_stats(self.device) else: torch_stats = torch.cuda.memory_stats(self.device) diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 420fbcf18..53d899f62 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -175,7 +175,7 @@ class StableDiffusionModelHijack: if opts.cuda_compile and opts.cuda_compile_mode == 'ipex': import logging - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': shared.log.info("Model compile enabled: IPEX Optimize Graph Mode") else: shared.log.warning("Model compile skipped: IPEX Method is for Intel GPU's with OneAPI") diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py index d32c35a29..a757a3755 100644 --- a/modules/sd_hijack_optimizations.py +++ b/modules/sd_hijack_optimizations.py @@ -29,7 +29,7 @@ else: def get_available_vram(): - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': stats = torch.xpu.memory_stats(shared.device) mem_active = stats['active_bytes.all.current'] mem_reserved = stats['reserved_bytes.all.current'] @@ -190,7 +190,7 @@ def einsum_op_tensor_mem(q, k, v, max_tensor_mb): return einsum_op_slice_1(q, k, v, max(q.shape[1] // div, 1)) def einsum_op_cuda(q, k, v): - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': stats = torch.xpu.memory_stats(q.device) mem_active = stats['active_bytes.all.current'] mem_reserved = stats['reserved_bytes.all.current'] @@ -218,10 +218,7 @@ def einsum_op_dml(q, k, v): return einsum_op_tensor_mem(q, k, v, (mem_reserved - mem_active) if mem_reserved > mem_active else 1) def einsum_op(q, k, v): - if shared.cmd_opts.use_ipex: - return einsum_op_cuda(q, k, v) - - if q.device.type == 'cuda': + if q.device.type == 'cuda' or devices.backend == 'ipex': return einsum_op_cuda(q, k, v) if q.device.type == 'mps': @@ -413,7 +410,7 @@ def scaled_dot_product_attention_forward(self, x, context=None, mask=None): return hidden_states def scaled_dot_product_no_mem_attention_forward(self, x, context=None, mask=None): - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': with torch.backends.xpu.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False): return scaled_dot_product_attention_forward(self, x, context, mask) else: @@ -522,7 +519,7 @@ def sdp_attnblock_forward(self, x): return x + out def sdp_no_mem_attnblock_forward(self, x): - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': with torch.backends.xpu.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=False): return sdp_attnblock_forward(self, x) else: diff --git a/modules/sd_hijack_unet.py b/modules/sd_hijack_unet.py index c7fee64b5..e5e9ded1d 100644 --- a/modules/sd_hijack_unet.py +++ b/modules/sd_hijack_unet.py @@ -3,7 +3,6 @@ from packaging import version from modules import devices from modules.sd_hijack_utils import CondFunc -from modules import shared class TorchHijackForUnet: @@ -68,7 +67,7 @@ def hijack_ddpm_edit(): unet_needs_upcast = lambda *args, **kwargs: devices.unet_needs_upcast # pylint: disable=unnecessary-lambda-assignment CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.apply_model', apply_model, unet_needs_upcast) CondFunc('ldm.modules.diffusionmodules.openaimodel.timestep_embedding', lambda orig_func, timesteps, *args, **kwargs: orig_func(timesteps, *args, **kwargs).to(torch.float32 if timesteps.dtype == torch.int64 else devices.dtype_unet), unet_needs_upcast) -if version.parse(torch.__version__) <= version.parse("1.13.2") or torch.cuda.is_available() or shared.cmd_opts.use_ipex: +if version.parse(torch.__version__) <= version.parse("1.13.2") or torch.cuda.is_available() or devices.backend == 'ipex': CondFunc('ldm.modules.diffusionmodules.util.GroupNorm32.forward', lambda orig_func, self, *args, **kwargs: orig_func(self.float(), *args, **kwargs), unet_needs_upcast) CondFunc('ldm.modules.attention.GEGLU.forward', lambda orig_func, self, x: orig_func(self.float(), x.float()).to(devices.dtype_unet), unet_needs_upcast) CondFunc('open_clip.transformer.ResidualAttentionBlock.__init__', lambda orig_func, *args, **kwargs: kwargs.update({'act_layer': GELUHijack}) and False or orig_func(*args, **kwargs), lambda _, *args, **kwargs: kwargs.get('act_layer') is None or kwargs['act_layer'] == torch.nn.GELU) diff --git a/modules/sd_models.py b/modules/sd_models.py index 30d5dfdb8..eb8c5d48f 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -585,7 +585,7 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None) sd_hijack.model_hijack.hijack(sd_model) timer.record("hijack") sd_model.eval() - if shared.cmd_opts.use_ipex and not (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): + if devices.backend == 'ipex' and not (shared.cmd_opts.lowvram or shared.cmd_opts.medvram): sd_model = torch.xpu.optimize(sd_model, dtype=devices.dtype, auto_kernel_selection=True, optimize_lstm=True, graph_mode=True if shared.opts.cuda_compile and shared.opts.cuda_compile_mode == 'ipex' else False) shared.log.info("Applied IPEX Optimize") diff --git a/modules/sd_samplers_kdiffusion.py b/modules/sd_samplers_kdiffusion.py index 2393b2ec2..2dc6c701f 100644 --- a/modules/sd_samplers_kdiffusion.py +++ b/modules/sd_samplers_kdiffusion.py @@ -4,7 +4,7 @@ import torch import k_diffusion.sampling from modules import prompt_parser, devices, sd_samplers_common -from modules.shared import opts, state, cmd_opts +from modules.shared import opts, state import modules.shared as shared from modules.script_callbacks import CFGDenoiserParams, cfg_denoiser_callback from modules.script_callbacks import CFGDenoisedParams, cfg_denoised_callback @@ -326,11 +326,11 @@ class KDiffusionSampler: sigma_max = sigmas.max() current_iter_seeds = p.all_seeds[p.iteration * p.batch_size:(p.iteration + 1) * p.batch_size] - if cmd_opts.use_ipex: #Remove this after Intel adds support for torch.Generator() + if devices.backend == 'ipex': #Remove this after Intel adds support for torch.Generator() try: return BrownianTreeNoiseSampler(x.to("cpu"), sigma_min, sigma_max, seed=current_iter_seeds, transform=lambda x: x.to("cpu"), transform_last=lambda x: x.to(shared.device)) # pylint: disable=E1123 except Exception: - print("ERROR Please apply this patch to repositories/k-diffusion/k_diffusion/sampling.py: https://github.com/crowsonkb/k-diffusion/pull/68/files") + shared.log.error("Please apply this patch to repositories/k-diffusion/k_diffusion/sampling.py: https://github.com/crowsonkb/k-diffusion/pull/68/files") return None else: return BrownianTreeNoiseSampler(x, sigma_min, sigma_max, seed=current_iter_seeds) diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index 2fdcb03d9..ddb0fbe6d 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -429,7 +429,7 @@ def train_embedding(id_task, embedding_name, learn_rate, batch_size, gradient_st else: shared.log.info("No saved optimizer exists in checkpoint") - if shared.cmd_opts.use_ipex: + if devices.backend == 'ipex': scaler = torch.xpu.amp.GradScaler() else: scaler = torch.cuda.amp.GradScaler() diff --git a/webui.sh b/webui.sh index 2abf44a67..dee18931f 100755 --- a/webui.sh +++ b/webui.sh @@ -82,7 +82,7 @@ else fi #Set OneAPI environmet if it's not set by the user -if [[ "$@" == *"--use-ipex"* ]] && ! [ -x "$(command -v sycl-ls)" ] +if ([[ "$@" == *"--use-ipex"* ]] || [[ -d "/opt/intel/oneapi" ]] || [[ ! -z "$ONEAPI_ROOT" ]]) && [ ! -x "$(command -v sycl-ls)" ] then echo "Setting OneAPI environment" if [[ -z "$ONEAPI_ROOT" ]] @@ -96,7 +96,7 @@ if [[ ! -z "${ACCELERATE}" ]] && [ ${ACCELERATE}="True" ] && [ -x "$(command -v then echo "Launching accelerate launch.py..." exec accelerate launch --num_cpu_threads_per_process=6 launch.py "$@" -elif [[ -z "${first_launch}" ]] && [ -x "$(command -v ipexrun)" ] && [ -x "$(command -v numactl)" ] && [[ "$@" == *"--use-ipex"* ]] +elif [[ -z "${first_launch}" ]] && [ -x "$(command -v ipexrun)" ] && [ -x "$(command -v numactl)" ] && [ -x "$(command -v sycl-ls)" ] then echo "Launching ipexrun launch.py..." exec ipexrun launch.py "$@"